Examples with LazyBlock com.facebook.presto.common.block.LazyBlock used on opensource projects

Example 1 with LazyBlock

use of com.facebook.presto.common.block.LazyBlock in project presto by prestodb.

the class DruidSegmentPageSource method getNextPage.

@Override
public Page getNextPage() {
    batchId++;
    int batchSize = segmentReader.nextBatch();
    if (batchSize <= 0) {
        close();
        return null;
    }
    Block[] blocks = new Block[columns.size()];
    for (int i = 0; i < blocks.length; i++) {
        DruidColumnHandle columnHandle = (DruidColumnHandle) columns.get(i);
        blocks[i] = new LazyBlock(batchSize, new SegmentBlockLoader(columnHandle.getColumnType(), columnHandle.getColumnName()));
    }
    Page page = new Page(batchSize, blocks);
    completedBytes += page.getSizeInBytes();
    completedPositions += page.getPositionCount();
    return page;
}

Example 2 with LazyBlock

use of com.facebook.presto.common.block.LazyBlock in project presto by prestodb.

the class ScanFilterAndProjectOperator method recordProcessedInput.

private Page recordProcessedInput(Page page) {
    long blockSizeSum = 0L;
    Block[] blocks = null;
    for (int i = 0; i < page.getChannelCount(); ++i) {
        Block block = page.getBlock(i);
        // account processed bytes from lazy blocks only when they are loaded
        if (block instanceof LazyBlock && !((LazyBlock) block).isLoaded()) {
            if (blocks == null) {
                blocks = copyOfPageBlocks(page);
            }
            blocks[i] = new LazyBlock(page.getPositionCount(), new RecordingLazyBlockLoader((LazyBlock) block));
        } else {
            blockSizeSum += block.getSizeInBytes();
        }
    }
    return (blocks == null) ? page : new Page(page.getPositionCount(), blocks);
}

Example 3 with LazyBlock

use of com.facebook.presto.common.block.LazyBlock in project presto by prestodb.

the class TestScanFilterAndProjectOperator method testPageSourceLazyLoad.

@Test
public void testPageSourceLazyLoad() {
    Block inputBlock = BlockAssertions.createLongSequenceBlock(0, 100);
    // If column 1 is loaded, test will fail
    Page input = new Page(100, inputBlock, new LazyBlock(100, lazyBlock -> {
        throw new AssertionError("Lazy block should not be loaded");
    }));
    DriverContext driverContext = newDriverContext();
    List<RowExpression> projections = ImmutableList.of(field(0, VARCHAR));
    Supplier<CursorProcessor> cursorProcessor = expressionCompiler.compileCursorProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections, "key");
    PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new PageProjectionWithOutputs(new LazyPagePageProjection(), new int[] { 0 })));
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns) -> new SinglePagePageSource(input), cursorProcessor, () -> pageProcessor, TESTING_TABLE_HANDLE, ImmutableList.of(), ImmutableList.of(BIGINT), Optional.empty(), new DataSize(0, BYTE), 0);
    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit()));
    operator.noMoreSplits();
    MaterializedResult expected = toMaterializedResult(driverContext.getSession(), ImmutableList.of(BIGINT), ImmutableList.of(new Page(inputBlock)));
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), ImmutableList.of(BIGINT), toPages(operator));
    assertEquals(actual.getRowCount(), expected.getRowCount());
    assertEquals(actual, expected);
}

Example 4 with LazyBlock

use of com.facebook.presto.common.block.LazyBlock in project presto by prestodb.

the class TestScanFilterAndProjectOperator method testPageSourceLazyBlock.

@Test
public void testPageSourceLazyBlock() {
    // Tests that a page containing a LazyBlock is loaded and its bytes are counted by the operator.
    DriverContext driverContext = newDriverContext();
    List<RowExpression> projections = ImmutableList.of(field(0, BIGINT));
    Supplier<CursorProcessor> cursorProcessor = expressionCompiler.compileCursorProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections, "key");
    Supplier<PageProcessor> pageProcessor = expressionCompiler.compilePageProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections);
    // This Block will be wrapped in a LazyBlock inside the operator on call to getNextPage().
    Block inputBlock = BlockAssertions.createLongSequenceBlock(0, 10);
    CountingLazyPageSource pageSource = new CountingLazyPageSource(ImmutableList.of(new Page(inputBlock)));
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns) -> pageSource, cursorProcessor, pageProcessor, TESTING_TABLE_HANDLE, ImmutableList.of(), ImmutableList.of(BIGINT), Optional.empty(), new DataSize(0, BYTE), 0);
    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit()));
    operator.noMoreSplits();
    MaterializedResult expected = toMaterializedResult(driverContext.getSession(), ImmutableList.of(BIGINT), ImmutableList.of(new Page(inputBlock)));
    Page expectedPage = expected.toPage();
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), ImmutableList.of(BIGINT), toPages(operator));
    Page actualPage = actual.toPage();
    // Assert expected page and actual page are equal.
    assertPageEquals(actual.getTypes(), actualPage, expectedPage);
    // PageSource counting isn't flawed, assert on the test implementation
    assertEquals(pageSource.getCompletedBytes(), expectedPage.getSizeInBytes());
    assertEquals(pageSource.getCompletedPositions(), expectedPage.getPositionCount());
    // Assert operator stats match the expected values
    assertEquals(operator.getOperatorContext().getOperatorStats().getRawInputDataSize().toBytes(), expectedPage.getSizeInBytes());
    assertEquals(operator.getOperatorContext().getOperatorStats().getInputPositions(), expected.getRowCount());
}

Example 5 with LazyBlock

use of com.facebook.presto.common.block.LazyBlock in project presto by prestodb.

the class HivePageSource method getNextPage.

@Override
public Page getNextPage() {
    try {
        Page dataPage = delegate.getNextPage();
        if (dataPage == null) {
            return null;
        }
        if (bucketAdapter.isPresent()) {
            dataPage = bucketAdapter.get().filterPageToEligibleRowsOrDiscard(dataPage);
            if (dataPage == null) {
                return null;
            }
        }
        int batchSize = dataPage.getPositionCount();
        List<Block> blocks = new ArrayList<>();
        for (int fieldId = 0; fieldId < columnMappings.size(); fieldId++) {
            ColumnMapping columnMapping = columnMappings.get(fieldId);
            switch(columnMapping.getKind()) {
                case PREFILLED:
                    blocks.add(RunLengthEncodedBlock.create(types[fieldId], prefilledValues[fieldId], batchSize));
                    break;
                case REGULAR:
                    Block block = dataPage.getBlock(columnMapping.getIndex());
                    if (coercers[fieldId] != null) {
                        block = new LazyBlock(batchSize, new CoercionLazyBlockLoader(block, coercers[fieldId]));
                    }
                    blocks.add(block);
                    break;
                case INTERIM:
                    // interim columns don't show up in output
                    break;
                case AGGREGATED:
                    // do not require data read and do not show up in output
                    break;
                default:
                    throw new UnsupportedOperationException();
            }
        }
        return new Page(batchSize, blocks.toArray(new Block[0]));
    } catch (PrestoException e) {
        closeWithSuppression(e);
        throw e;
    } catch (RuntimeException e) {
        closeWithSuppression(e);
        throw new PrestoException(HIVE_CURSOR_ERROR, e);
    }
}