Examples with PageProcessor com.facebook.presto.operator.project.PageProcessor used on opensource projects

Example 6 with PageProcessor

use of com.facebook.presto.operator.project.PageProcessor in project presto by prestodb.

the class TestScanFilterAndProjectOperator method testPageSourceLazyBlock.

@Test
public void testPageSourceLazyBlock() {
    // Tests that a page containing a LazyBlock is loaded and its bytes are counted by the operator.
    DriverContext driverContext = newDriverContext();
    List<RowExpression> projections = ImmutableList.of(field(0, BIGINT));
    Supplier<CursorProcessor> cursorProcessor = expressionCompiler.compileCursorProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections, "key");
    Supplier<PageProcessor> pageProcessor = expressionCompiler.compilePageProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections);
    // This Block will be wrapped in a LazyBlock inside the operator on call to getNextPage().
    Block inputBlock = BlockAssertions.createLongSequenceBlock(0, 10);
    CountingLazyPageSource pageSource = new CountingLazyPageSource(ImmutableList.of(new Page(inputBlock)));
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns) -> pageSource, cursorProcessor, pageProcessor, TESTING_TABLE_HANDLE, ImmutableList.of(), ImmutableList.of(BIGINT), Optional.empty(), new DataSize(0, BYTE), 0);
    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit()));
    operator.noMoreSplits();
    MaterializedResult expected = toMaterializedResult(driverContext.getSession(), ImmutableList.of(BIGINT), ImmutableList.of(new Page(inputBlock)));
    Page expectedPage = expected.toPage();
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), ImmutableList.of(BIGINT), toPages(operator));
    Page actualPage = actual.toPage();
    // Assert expected page and actual page are equal.
    assertPageEquals(actual.getTypes(), actualPage, expectedPage);
    // PageSource counting isn't flawed, assert on the test implementation
    assertEquals(pageSource.getCompletedBytes(), expectedPage.getSizeInBytes());
    assertEquals(pageSource.getCompletedPositions(), expectedPage.getPositionCount());
    // Assert operator stats match the expected values
    assertEquals(operator.getOperatorContext().getOperatorStats().getRawInputDataSize().toBytes(), expectedPage.getSizeInBytes());
    assertEquals(operator.getOperatorContext().getOperatorStats().getInputPositions(), expected.getRowCount());
}

Example 7 with PageProcessor

use of com.facebook.presto.operator.project.PageProcessor in project presto by prestodb.

the class TestFilterAndProjectOperator method testMergeOutput.

@Test
public void testMergeOutput() {
    List<Page> input = rowPagesBuilder(VARCHAR, BIGINT).addSequencePage(100, 0, 0).addSequencePage(100, 0, 0).addSequencePage(100, 0, 0).addSequencePage(100, 0, 0).build();
    MetadataManager metadata = createTestMetadataManager();
    RowExpression filter = call(EQUAL.name(), metadata.getFunctionAndTypeManager().resolveOperator(EQUAL, fromTypes(BIGINT, BIGINT)), BOOLEAN, field(1, BIGINT), constant(10L, BIGINT));
    ExpressionCompiler compiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0));
    Supplier<PageProcessor> processor = compiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.of(filter), ImmutableList.of(field(1, BIGINT)));
    OperatorFactory operatorFactory = new FilterAndProjectOperator.FilterAndProjectOperatorFactory(0, new PlanNodeId("test"), processor, ImmutableList.of(BIGINT), new DataSize(64, KILOBYTE), 2);
    List<Page> expected = rowPagesBuilder(BIGINT).row(10L).row(10L).row(10L).row(10L).build();
    assertOperatorEquals(operatorFactory, ImmutableList.of(BIGINT), driverContext, input, expected);
}

Example 8 with PageProcessor

use of com.facebook.presto.operator.project.PageProcessor in project presto by prestodb.

the class FunctionAssertions method compileScanFilterProject.

private static SourceOperatorFactory compileScanFilterProject(SqlFunctionProperties sqlFunctionProperties, Optional<RowExpression> filter, RowExpression projection, ExpressionCompiler compiler) {
    try {
        Supplier<CursorProcessor> cursorProcessor = compiler.compileCursorProcessor(sqlFunctionProperties, filter, ImmutableList.of(projection), SOURCE_ID);
        Supplier<PageProcessor> pageProcessor = compiler.compilePageProcessor(sqlFunctionProperties, filter, ImmutableList.of(projection));
        return new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), SOURCE_ID, PAGE_SOURCE_PROVIDER, cursorProcessor, pageProcessor, new TableHandle(new ConnectorId("test"), new ConnectorTableHandle() {
        }, new ConnectorTransactionHandle() {
        }, Optional.empty()), ImmutableList.of(), ImmutableList.of(projection.getType()), Optional.empty(), new DataSize(0, BYTE), 0);
    } catch (Throwable e) {
        if (e instanceof UncheckedExecutionException) {
            e = e.getCause();
        }
        throw new RuntimeException("Error compiling filter " + filter + ": " + e.getMessage(), e);
    }
}

Example 9 with PageProcessor

use of com.facebook.presto.operator.project.PageProcessor in project presto by prestodb.

the class TestPageProcessorCompiler method testSanityFilterOnDictionary.

@Test
public void testSanityFilterOnDictionary() {
    FunctionAndTypeManager functionAndTypeManager = createTestMetadataManager().getFunctionAndTypeManager();
    CallExpression lengthVarchar = new CallExpression("length", functionAndTypeManager.lookupFunction("length", fromTypes(VARCHAR)), BIGINT, ImmutableList.of(field(0, VARCHAR)));
    FunctionHandle lessThan = functionAndTypeManager.resolveOperator(LESS_THAN, fromTypes(BIGINT, BIGINT));
    CallExpression filter = new CallExpression(LESS_THAN.name(), lessThan, BOOLEAN, ImmutableList.of(lengthVarchar, constant(10L, BIGINT)));
    PageProcessor processor = compiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.of(filter), ImmutableList.of(field(0, VARCHAR)), false, MAX_BATCH_SIZE).get();
    Page page = new Page(createDictionaryBlock(createExpectedValues(10), 100));
    Page outputPage = getOnlyElement(processor.process(null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertEquals(outputPage.getPositionCount(), 100);
    assertTrue(outputPage.getBlock(0) instanceof DictionaryBlock);
    DictionaryBlock dictionaryBlock = (DictionaryBlock) outputPage.getBlock(0);
    assertEquals(dictionaryBlock.getDictionary().getPositionCount(), 10);
    // test filter caching
    Page outputPage2 = getOnlyElement(processor.process(null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertEquals(outputPage2.getPositionCount(), 100);
    assertTrue(outputPage2.getBlock(0) instanceof DictionaryBlock);
    DictionaryBlock dictionaryBlock2 = (DictionaryBlock) outputPage2.getBlock(0);
    // both output pages must have the same dictionary
    assertEquals(dictionaryBlock2.getDictionary(), dictionaryBlock.getDictionary());
}

Example 10 with PageProcessor

use of com.facebook.presto.operator.project.PageProcessor in project presto by prestodb.

the class TestPageProcessorCompiler method testSanityColumnarDictionary.

@Test
public void testSanityColumnarDictionary() {
    PageProcessor processor = compiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.empty(), ImmutableList.of(field(0, VARCHAR)), false, MAX_BATCH_SIZE).get();
    Page page = new Page(createDictionaryBlock(createExpectedValues(10), 100));
    Page outputPage = getOnlyElement(processor.process(null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertEquals(outputPage.getPositionCount(), 100);
    assertTrue(outputPage.getBlock(0) instanceof DictionaryBlock);
    DictionaryBlock dictionaryBlock = (DictionaryBlock) outputPage.getBlock(0);
    assertEquals(dictionaryBlock.getDictionary().getPositionCount(), 10);
}