Examples with Page com.facebook.presto.common.Page used on opensource projects

Example 76 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class ThriftTpchService method getRowsInternal.

private static PrestoThriftPageResult getRowsInternal(ConnectorPageSource pageSource, String tableName, List<String> columnNames, @Nullable PrestoThriftId nextToken) {
    // very inefficient implementation as it needs to re-generate all previous results to get the next page
    int skipPages = nextToken != null ? Ints.fromByteArray(nextToken.getId()) : 0;
    skipPages(pageSource, skipPages);
    Page page = null;
    while (!pageSource.isFinished() && page == null) {
        page = pageSource.getNextPage();
        skipPages++;
    }
    PrestoThriftId newNextToken = pageSource.isFinished() ? null : new PrestoThriftId(Ints.toByteArray(skipPages));
    return toThriftPage(page, types(tableName, columnNames), newNextToken);
}

Example 77 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestThriftIndexPageSource method testGetNextPageTwoConcurrentRequests.

@Test
public void testGetNextPageTwoConcurrentRequests() throws Exception {
    final int splits = 3;
    final int lookupRequestsConcurrency = 2;
    final int rowsPerSplit = 1;
    List<SettableFuture<PrestoThriftPageResult>> futures = IntStream.range(0, splits).mapToObj(i -> SettableFuture.<PrestoThriftPageResult>create()).collect(toImmutableList());
    List<CountDownLatch> signals = IntStream.range(0, splits).mapToObj(i -> new CountDownLatch(1)).collect(toImmutableList());
    TestingThriftService client = new TestingThriftService(rowsPerSplit, false, false) {

        @Override
        public ListenableFuture<PrestoThriftPageResult> getRows(PrestoThriftId splitId, List<String> columns, long maxBytes, PrestoThriftNullableToken nextToken) {
            int key = Ints.fromByteArray(splitId.getId());
            signals.get(key).countDown();
            return futures.get(key);
        }
    };
    ThriftConnectorStats stats = new ThriftConnectorStats();
    long pageSizeReceived = 0;
    ThriftIndexPageSource pageSource = new ThriftIndexPageSource((context, headers) -> client, ImmutableMap.of(), stats, new ThriftIndexHandle(new SchemaTableName("default", "table1"), TupleDomain.all()), ImmutableList.of(column("a", INTEGER)), ImmutableList.of(column("b", INTEGER)), new InMemoryRecordSet(ImmutableList.of(INTEGER), generateKeys(0, splits)), MAX_BYTES_PER_RESPONSE, lookupRequestsConcurrency);
    assertNull(pageSource.getNextPage());
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), 0);
    signals.get(0).await(1, SECONDS);
    signals.get(1).await(1, SECONDS);
    signals.get(2).await(1, SECONDS);
    assertEquals(signals.get(0).getCount(), 0, "first request wasn't sent");
    assertEquals(signals.get(1).getCount(), 0, "second request wasn't sent");
    assertEquals(signals.get(2).getCount(), 1, "third request shouldn't be sent");
    // at this point first two requests were sent
    assertFalse(pageSource.isFinished());
    assertNull(pageSource.getNextPage());
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), 0);
    // completing the second request
    futures.get(1).set(pageResult(20, null));
    Page page = pageSource.getNextPage();
    pageSizeReceived += page.getSizeInBytes();
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), pageSizeReceived);
    assertNotNull(page);
    assertEquals(page.getPositionCount(), 1);
    assertEquals(page.getBlock(0).getInt(0), 20);
    // not complete yet
    assertFalse(pageSource.isFinished());
    // once one of the requests completes the next one should be sent
    signals.get(2).await(1, SECONDS);
    assertEquals(signals.get(2).getCount(), 0, "third request wasn't sent");
    // completing the first request
    futures.get(0).set(pageResult(10, null));
    page = pageSource.getNextPage();
    assertNotNull(page);
    pageSizeReceived += page.getSizeInBytes();
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), pageSizeReceived);
    assertEquals(page.getPositionCount(), 1);
    assertEquals(page.getBlock(0).getInt(0), 10);
    // still not complete
    assertFalse(pageSource.isFinished());
    // completing the third request
    futures.get(2).set(pageResult(30, null));
    page = pageSource.getNextPage();
    assertNotNull(page);
    pageSizeReceived += page.getSizeInBytes();
    assertEquals((long) stats.getIndexPageSize().getAllTime().getTotal(), pageSizeReceived);
    assertEquals(page.getPositionCount(), 1);
    assertEquals(page.getBlock(0).getInt(0), 30);
    // finished now
    assertTrue(pageSource.isFinished());
    // after completion
    assertNull(pageSource.getNextPage());
    pageSource.close();
}

Example 78 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class ThriftIndexPageSource method getNextPage.

@Override
public Page getNextPage() {
    if (finished) {
        return null;
    }
    if (!loadAllSplits()) {
        return null;
    }
    // check if any data requests were started
    if (dataSignalFuture == null) {
        // no data requests were started, start a number of initial requests
        checkState(contexts.isEmpty() && dataRequests.isEmpty(), "some splits are already started");
        if (splits.isEmpty()) {
            // all done: no splits
            finished = true;
            return null;
        }
        for (int i = 0; i < min(lookupRequestsConcurrency, splits.size()); i++) {
            startDataFetchForNextSplit();
        }
        updateSignalAndStatusFutures();
    }
    // check if any data request is finished
    if (!dataSignalFuture.isDone()) {
        // not finished yet
        return null;
    }
    // at least one of data requests completed
    ListenableFuture<PrestoThriftPageResult> resultFuture = getAndRemoveNextCompletedRequest();
    RunningSplitContext resultContext = contexts.remove(resultFuture);
    checkState(resultContext != null, "no associated context for the request");
    PrestoThriftPageResult pageResult = getFutureValue(resultFuture);
    Page page = pageResult.toPage(outputColumnTypes);
    if (page != null) {
        long pageSize = page.getSizeInBytes();
        completedBytes += pageSize;
        completedPositions += page.getPositionCount();
        stats.addIndexPageSize(pageSize);
    } else {
        stats.addIndexPageSize(0);
    }
    if (pageResult.getNextToken() != null) {
        // can get more data
        sendDataRequest(resultContext, pageResult.getNextToken());
        updateSignalAndStatusFutures();
        return page;
    }
    // are there more splits available
    if (splitIndex < splits.size()) {
        // can send data request for a new split
        startDataFetchForNextSplit();
        updateSignalAndStatusFutures();
    } else if (!dataRequests.isEmpty()) {
        // no more new splits, but some requests are still in progress, wait for them
        updateSignalAndStatusFutures();
    } else {
        // all done: no more new splits, no requests in progress
        dataSignalFuture = null;
        statusFuture = null;
        finished = true;
    }
    return page;
}

Example 79 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TempFileReader method computeNext.

@Override
protected Page computeNext() {
    try {
        if (Thread.currentThread().isInterrupted()) {
            throw new InterruptedIOException();
        }
        int batchSize = reader.nextBatch();
        if (batchSize <= 0) {
            return endOfData();
        }
        Block[] blocks = new Block[columnCount];
        for (int i = 0; i < columnCount; i++) {
            blocks[i] = reader.readBlock(i).getLoadedBlock();
        }
        return new Page(batchSize, blocks);
    } catch (IOException e) {
        throw new PrestoException(HIVE_WRITER_DATA_ERROR, "Failed to read temporary data");
    }
}

Example 80 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class AbstractTestHiveFileFormats method assertFileStatistics.

private static void assertFileStatistics(Optional<Page> fileStatistics, long writtenBytes, HiveStorageFormat storageFormat) {
    if (storageFormat == ORC || storageFormat == DWRF) {
        assertTrue(fileStatistics.isPresent());
        Page statisticsPage = fileStatistics.get();
        assertEquals(statisticsPage.getPositionCount(), 1);
        assertEquals(writtenBytes, getFileSize(statisticsPage, 0));
    }
}