Examples with InMemoryRecordSet io.trino.spi.connector.InMemoryRecordSet used on opensource projects

Example 6 with InMemoryRecordSet

use of io.trino.spi.connector.InMemoryRecordSet in project trino by trinodb.

the class TestThriftIndexPageSource method testGetNextPageTwoConcurrentRequests.

@Test
public void testGetNextPageTwoConcurrentRequests() throws Exception {
    final int splits = 3;
    final int lookupRequestsConcurrency = 2;
    final int rowsPerSplit = 1;
    List<SettableFuture<TrinoThriftPageResult>> futures = IntStream.range(0, splits).mapToObj(i -> SettableFuture.<TrinoThriftPageResult>create()).collect(toImmutableList());
    List<CountDownLatch> signals = IntStream.range(0, splits).mapToObj(i -> new CountDownLatch(1)).collect(toImmutableList());
    TestingThriftService client = new TestingThriftService(rowsPerSplit, false, false) {

        @Override
        public ListenableFuture<TrinoThriftPageResult> getRows(TrinoThriftId splitId, List<String> columns, long maxBytes, TrinoThriftNullableToken nextToken) {
            int key = Ints.fromByteArray(splitId.getId());
            signals.get(key).countDown();
            return futures.get(key);
        }
    };
    ThriftConnectorStats stats = new ThriftConnectorStats();
    long pageSizeReceived = 0;
    ThriftIndexPageSource pageSource = new ThriftIndexPageSource((context, headers) -> client, ImmutableMap.of(), stats, new ThriftIndexHandle(new SchemaTableName("default", "table1"), TupleDomain.all()), ImmutableList.of(column("a", INTEGER)), ImmutableList.of(column("b", INTEGER)), new InMemoryRecordSet(ImmutableList.of(INTEGER), generateKeys(0, splits)), MAX_BYTES_PER_RESPONSE, lookupRequestsConcurrency);
    assertNull(pageSource.getNextPage());
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), 0);
    signals.get(0).await(1, SECONDS);
    signals.get(1).await(1, SECONDS);
    signals.get(2).await(1, SECONDS);
    assertEquals(signals.get(0).getCount(), 0, "first request wasn't sent");
    assertEquals(signals.get(1).getCount(), 0, "second request wasn't sent");
    assertEquals(signals.get(2).getCount(), 1, "third request shouldn't be sent");
    // at this point first two requests were sent
    assertFalse(pageSource.isFinished());
    assertNull(pageSource.getNextPage());
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), 0);
    // completing the second request
    futures.get(1).set(pageResult(20, null));
    Page page = pageSource.getNextPage();
    pageSizeReceived += page.getSizeInBytes();
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), pageSizeReceived);
    assertNotNull(page);
    assertEquals(page.getPositionCount(), 1);
    assertEquals(page.getBlock(0).getInt(0, 0), 20);
    // not complete yet
    assertFalse(pageSource.isFinished());
    // once one of the requests completes the next one should be sent
    signals.get(2).await(1, SECONDS);
    assertEquals(signals.get(2).getCount(), 0, "third request wasn't sent");
    // completing the first request
    futures.get(0).set(pageResult(10, null));
    page = pageSource.getNextPage();
    assertNotNull(page);
    pageSizeReceived += page.getSizeInBytes();
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), pageSizeReceived);
    assertEquals(page.getPositionCount(), 1);
    assertEquals(page.getBlock(0).getInt(0, 0), 10);
    // still not complete
    assertFalse(pageSource.isFinished());
    // completing the third request
    futures.get(2).set(pageResult(30, null));
    page = pageSource.getNextPage();
    assertNotNull(page);
    pageSizeReceived += page.getSizeInBytes();
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), pageSizeReceived);
    assertEquals(page.getPositionCount(), 1);
    assertEquals(page.getBlock(0).getInt(0, 0), 30);
    // finished now
    assertTrue(pageSource.isFinished());
    // after completion
    assertNull(pageSource.getNextPage());
    pageSource.close();
}

Example 7 with InMemoryRecordSet

use of io.trino.spi.connector.InMemoryRecordSet in project trino by trinodb.

the class TestThriftIndexPageSource method runGeneralTest.

private static void runGeneralTest(int splits, int lookupRequestsConcurrency, int rowsPerSplit, boolean twoSplitBatches) throws Exception {
    TestingThriftService client = new TestingThriftService(rowsPerSplit, true, twoSplitBatches);
    ThriftIndexPageSource pageSource = new ThriftIndexPageSource((context, headers) -> client, ImmutableMap.of(), new ThriftConnectorStats(), new ThriftIndexHandle(new SchemaTableName("default", "table1"), TupleDomain.all()), ImmutableList.of(column("a", INTEGER)), ImmutableList.of(column("b", INTEGER)), new InMemoryRecordSet(ImmutableList.of(INTEGER), generateKeys(1, splits + 1)), MAX_BYTES_PER_RESPONSE, lookupRequestsConcurrency);
    List<Integer> actual = new ArrayList<>();
    while (!pageSource.isFinished()) {
        CompletableFuture<?> blocked = pageSource.isBlocked();
        blocked.get(1, SECONDS);
        Page page = pageSource.getNextPage();
        if (page != null) {
            Block block = page.getBlock(0);
            for (int position = 0; position < block.getPositionCount(); position++) {
                actual.add(block.getInt(position, 0));
            }
        }
    }
    Collections.sort(actual);
    List<Integer> expected = new ArrayList<>(splits * rowsPerSplit);
    for (int split = 1; split <= splits; split++) {
        for (int row = 0; row < rowsPerSplit; row++) {
            expected.add(split * 10 + row);
        }
    }
    assertEquals(actual, expected);
    // must be null after finish
    assertNull(pageSource.getNextPage());
    pageSource.close();
}