Search in sources :

Example 1 with PrestoThriftNullableToken

use of com.facebook.presto.thrift.api.connector.PrestoThriftNullableToken in project presto by prestodb.

the class TestThriftIndexPageSource method testGetNextPageTwoConcurrentRequests.

@Test
public void testGetNextPageTwoConcurrentRequests() throws Exception {
    final int splits = 3;
    final int lookupRequestsConcurrency = 2;
    final int rowsPerSplit = 1;
    List<SettableFuture<PrestoThriftPageResult>> futures = IntStream.range(0, splits).mapToObj(i -> SettableFuture.<PrestoThriftPageResult>create()).collect(toImmutableList());
    List<CountDownLatch> signals = IntStream.range(0, splits).mapToObj(i -> new CountDownLatch(1)).collect(toImmutableList());
    TestingThriftService client = new TestingThriftService(rowsPerSplit, false, false) {

        @Override
        public ListenableFuture<PrestoThriftPageResult> getRows(PrestoThriftId splitId, List<String> columns, long maxBytes, PrestoThriftNullableToken nextToken) {
            int key = Ints.fromByteArray(splitId.getId());
            signals.get(key).countDown();
            return futures.get(key);
        }
    };
    ThriftConnectorStats stats = new ThriftConnectorStats();
    long pageSizeReceived = 0;
    ThriftIndexPageSource pageSource = new ThriftIndexPageSource((context, headers) -> client, ImmutableMap.of(), stats, new ThriftIndexHandle(new SchemaTableName("default", "table1"), TupleDomain.all()), ImmutableList.of(column("a", INTEGER)), ImmutableList.of(column("b", INTEGER)), new InMemoryRecordSet(ImmutableList.of(INTEGER), generateKeys(0, splits)), MAX_BYTES_PER_RESPONSE, lookupRequestsConcurrency);
    assertNull(pageSource.getNextPage());
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), 0);
    signals.get(0).await(1, SECONDS);
    signals.get(1).await(1, SECONDS);
    signals.get(2).await(1, SECONDS);
    assertEquals(signals.get(0).getCount(), 0, "first request wasn't sent");
    assertEquals(signals.get(1).getCount(), 0, "second request wasn't sent");
    assertEquals(signals.get(2).getCount(), 1, "third request shouldn't be sent");
    // at this point first two requests were sent
    assertFalse(pageSource.isFinished());
    assertNull(pageSource.getNextPage());
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), 0);
    // completing the second request
    futures.get(1).set(pageResult(20, null));
    Page page = pageSource.getNextPage();
    pageSizeReceived += page.getSizeInBytes();
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), pageSizeReceived);
    assertNotNull(page);
    assertEquals(page.getPositionCount(), 1);
    assertEquals(page.getBlock(0).getInt(0), 20);
    // not complete yet
    assertFalse(pageSource.isFinished());
    // once one of the requests completes the next one should be sent
    signals.get(2).await(1, SECONDS);
    assertEquals(signals.get(2).getCount(), 0, "third request wasn't sent");
    // completing the first request
    futures.get(0).set(pageResult(10, null));
    page = pageSource.getNextPage();
    assertNotNull(page);
    pageSizeReceived += page.getSizeInBytes();
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), pageSizeReceived);
    assertEquals(page.getPositionCount(), 1);
    assertEquals(page.getBlock(0).getInt(0), 10);
    // still not complete
    assertFalse(pageSource.isFinished());
    // completing the third request
    futures.get(2).set(pageResult(30, null));
    page = pageSource.getNextPage();
    assertNotNull(page);
    pageSizeReceived += page.getSizeInBytes();
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), pageSizeReceived);
    assertEquals(page.getPositionCount(), 1);
    assertEquals(page.getBlock(0).getInt(0), 30);
    // finished now
    assertTrue(pageSource.isFinished());
    // after completion
    assertNull(pageSource.getNextPage());
    pageSource.close();
}
Also used : SettableFuture(com.google.common.util.concurrent.SettableFuture) IntStream(java.util.stream.IntStream) Collections.shuffle(java.util.Collections.shuffle) PrestoThriftNullableSchemaName(com.facebook.presto.thrift.api.connector.PrestoThriftNullableSchemaName) Page(com.facebook.presto.common.Page) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) Assert.assertNull(org.testng.Assert.assertNull) PrestoThriftNullableTableMetadata(com.facebook.presto.thrift.api.connector.PrestoThriftNullableTableMetadata) PrestoThriftNullableColumnSet(com.facebook.presto.thrift.api.connector.PrestoThriftNullableColumnSet) PrestoThriftBlock.integerData(com.facebook.presto.thrift.api.datatypes.PrestoThriftBlock.integerData) Assert.assertEquals(org.testng.Assert.assertEquals) PrestoThriftSplitBatch(com.facebook.presto.thrift.api.connector.PrestoThriftSplitBatch) Test(org.testng.annotations.Test) CompletableFuture(java.util.concurrent.CompletableFuture) SettableFuture(com.google.common.util.concurrent.SettableFuture) PrestoThriftPageResult(com.facebook.presto.thrift.api.connector.PrestoThriftPageResult) InMemoryRecordSet(com.facebook.presto.spi.InMemoryRecordSet) ArrayList(java.util.ArrayList) SchemaTableName(com.facebook.presto.spi.SchemaTableName) ImmutableList(com.google.common.collect.ImmutableList) PrestoThriftSchemaTableName(com.facebook.presto.thrift.api.connector.PrestoThriftSchemaTableName) PrestoThriftNullableToken(com.facebook.presto.thrift.api.connector.PrestoThriftNullableToken) PrestoThriftTupleDomain(com.facebook.presto.thrift.api.connector.PrestoThriftTupleDomain) Assert.assertFalse(org.testng.Assert.assertFalse) Type(com.facebook.presto.common.type.Type) Futures.immediateFuture(com.google.common.util.concurrent.Futures.immediateFuture) PrestoThriftService(com.facebook.presto.thrift.api.connector.PrestoThriftService) PrestoThriftServiceException(com.facebook.presto.thrift.api.connector.PrestoThriftServiceException) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Assert.assertNotNull(org.testng.Assert.assertNotNull) PrestoThriftId(com.facebook.presto.thrift.api.connector.PrestoThriftId) PrestoThriftSplit(com.facebook.presto.thrift.api.connector.PrestoThriftSplit) Ints(com.google.common.primitives.Ints) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) INTEGER(com.facebook.presto.common.type.IntegerType.INTEGER) PrestoThriftInteger(com.facebook.presto.thrift.api.datatypes.PrestoThriftInteger) Assert.assertTrue(org.testng.Assert.assertTrue) Block(com.facebook.presto.common.block.Block) Collections(java.util.Collections) SECONDS(java.util.concurrent.TimeUnit.SECONDS) PrestoThriftId(com.facebook.presto.thrift.api.connector.PrestoThriftId) PrestoThriftNullableToken(com.facebook.presto.thrift.api.connector.PrestoThriftNullableToken) PrestoThriftPageResult(com.facebook.presto.thrift.api.connector.PrestoThriftPageResult) Page(com.facebook.presto.common.Page) CountDownLatch(java.util.concurrent.CountDownLatch) SchemaTableName(com.facebook.presto.spi.SchemaTableName) PrestoThriftSchemaTableName(com.facebook.presto.thrift.api.connector.PrestoThriftSchemaTableName) InMemoryRecordSet(com.facebook.presto.spi.InMemoryRecordSet) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) List(java.util.List) Test(org.testng.annotations.Test)

Example 2 with PrestoThriftNullableToken

use of com.facebook.presto.thrift.api.connector.PrestoThriftNullableToken in project presto by prestodb.

the class ThriftIndexPageSource method sendSplitRequest.

private ListenableFuture<PrestoThriftSplitBatch> sendSplitRequest(@Nullable PrestoThriftId nextToken) {
    long start = System.nanoTime();
    ListenableFuture<PrestoThriftSplitBatch> future = client.get(thriftHeaders).getIndexSplits(schemaTableName, lookupColumnNames, outputColumnNames, keys, outputConstraint, MAX_SPLIT_COUNT, new PrestoThriftNullableToken(nextToken));
    future = catchingThriftException(future);
    future.addListener(() -> readTimeNanos.addAndGet(System.nanoTime() - start), directExecutor());
    return future;
}
Also used : PrestoThriftSplitBatch(com.facebook.presto.thrift.api.connector.PrestoThriftSplitBatch) PrestoThriftNullableToken(com.facebook.presto.thrift.api.connector.PrestoThriftNullableToken)

Example 3 with PrestoThriftNullableToken

use of com.facebook.presto.thrift.api.connector.PrestoThriftNullableToken in project presto by prestodb.

the class ThriftIndexPageSource method sendDataRequest.

private void sendDataRequest(RunningSplitContext context, @Nullable PrestoThriftId nextToken) {
    long start = System.nanoTime();
    ListenableFuture<PrestoThriftPageResult> future = context.getClient().getRows(context.getSplit().getSplitId(), outputColumnNames, maxBytesPerResponse, new PrestoThriftNullableToken(nextToken));
    future = catchingThriftException(future);
    future.addListener(() -> readTimeNanos.addAndGet(System.nanoTime() - start), directExecutor());
    dataRequests.add(future);
    contexts.put(future, context);
}
Also used : PrestoThriftNullableToken(com.facebook.presto.thrift.api.connector.PrestoThriftNullableToken) PrestoThriftPageResult(com.facebook.presto.thrift.api.connector.PrestoThriftPageResult)

Example 4 with PrestoThriftNullableToken

use of com.facebook.presto.thrift.api.connector.PrestoThriftNullableToken in project presto by prestodb.

the class ThriftPageSource method sendDataRequestInternal.

private CompletableFuture<PrestoThriftPageResult> sendDataRequestInternal() {
    long start = System.nanoTime();
    ListenableFuture<PrestoThriftPageResult> rowsBatchFuture = client.getRows(splitId, columnNames, maxBytesPerResponse, new PrestoThriftNullableToken(nextToken));
    rowsBatchFuture = catchingThriftException(rowsBatchFuture);
    rowsBatchFuture.addListener(() -> readTimeNanos.addAndGet(System.nanoTime() - start), directExecutor());
    return toCompletableFuture(nonCancellationPropagating(rowsBatchFuture));
}
Also used : PrestoThriftNullableToken(com.facebook.presto.thrift.api.connector.PrestoThriftNullableToken) PrestoThriftPageResult(com.facebook.presto.thrift.api.connector.PrestoThriftPageResult)

Aggregations

PrestoThriftNullableToken (com.facebook.presto.thrift.api.connector.PrestoThriftNullableToken)4 PrestoThriftPageResult (com.facebook.presto.thrift.api.connector.PrestoThriftPageResult)3 PrestoThriftSplitBatch (com.facebook.presto.thrift.api.connector.PrestoThriftSplitBatch)2 Page (com.facebook.presto.common.Page)1 Block (com.facebook.presto.common.block.Block)1 TupleDomain (com.facebook.presto.common.predicate.TupleDomain)1 INTEGER (com.facebook.presto.common.type.IntegerType.INTEGER)1 Type (com.facebook.presto.common.type.Type)1 InMemoryRecordSet (com.facebook.presto.spi.InMemoryRecordSet)1 SchemaTableName (com.facebook.presto.spi.SchemaTableName)1 PrestoThriftId (com.facebook.presto.thrift.api.connector.PrestoThriftId)1 PrestoThriftNullableColumnSet (com.facebook.presto.thrift.api.connector.PrestoThriftNullableColumnSet)1 PrestoThriftNullableSchemaName (com.facebook.presto.thrift.api.connector.PrestoThriftNullableSchemaName)1 PrestoThriftNullableTableMetadata (com.facebook.presto.thrift.api.connector.PrestoThriftNullableTableMetadata)1 PrestoThriftSchemaTableName (com.facebook.presto.thrift.api.connector.PrestoThriftSchemaTableName)1 PrestoThriftService (com.facebook.presto.thrift.api.connector.PrestoThriftService)1 PrestoThriftServiceException (com.facebook.presto.thrift.api.connector.PrestoThriftServiceException)1 PrestoThriftSplit (com.facebook.presto.thrift.api.connector.PrestoThriftSplit)1 PrestoThriftTupleDomain (com.facebook.presto.thrift.api.connector.PrestoThriftTupleDomain)1 PrestoThriftBlock.integerData (com.facebook.presto.thrift.api.datatypes.PrestoThriftBlock.integerData)1