Examples with PageProcessor io.prestosql.operator.project.PageProcessor used on opensource projects

Example 11 with PageProcessor

use of io.prestosql.operator.project.PageProcessor in project hetu-core by openlookeng.

the class TestScanFilterAndProjectOperator method testPageSource.

@Test
public void testPageSource() {
    final Page input = SequencePageBuilder.createSequencePage(ImmutableList.of(VARCHAR), 10_000, 0);
    DriverContext driverContext = newDriverContext();
    List<RowExpression> projections = ImmutableList.of(field(0, VARCHAR));
    Supplier<CursorProcessor> cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.empty(), projections, "key");
    Supplier<PageProcessor> pageProcessor = expressionCompiler.compilePageProcessor(Optional.empty(), projections);
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns, dynamicFilter) -> new FixedPageSource(ImmutableList.of(input)), cursorProcessor, pageProcessor, TEST_TABLE_HANDLE, ImmutableList.of(), null, ImmutableList.of(VARCHAR), new DataSize(0, BYTE), 0, ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), false, Optional.empty(), 0, 0);
    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new CatalogName("test"), TestingSplit.createLocalSplit(), Lifespan.taskWide()));
    operator.noMoreSplits();
    MaterializedResult expected = toMaterializedResult(driverContext.getSession(), ImmutableList.of(VARCHAR), ImmutableList.of(input));
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), ImmutableList.of(VARCHAR), toPages(operator));
    assertEquals(actual.getRowCount(), expected.getRowCount());
    assertEquals(actual, expected);
}

Example 12 with PageProcessor

use of io.prestosql.operator.project.PageProcessor in project hetu-core by openlookeng.

the class TestScanFilterAndProjectOperator method testPageSourceMergeOutput.

@Test
public void testPageSourceMergeOutput() {
    List<Page> input = rowPagesBuilder(BIGINT).addSequencePage(100, 0).addSequencePage(100, 0).addSequencePage(100, 0).addSequencePage(100, 0).build();
    RowExpression filter = call(EQUAL.getFunctionName().toString(), new BuiltInFunctionHandle(Signature.internalOperator(EQUAL, BOOLEAN.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature(), BIGINT.getTypeSignature()))), BOOLEAN, field(0, BIGINT), constant(10L, BIGINT));
    List<RowExpression> projections = ImmutableList.of(field(0, BIGINT));
    Supplier<CursorProcessor> cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.of(filter), projections, "key");
    Supplier<PageProcessor> pageProcessor = expressionCompiler.compilePageProcessor(Optional.of(filter), projections);
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns, dynamicFilter) -> new FixedPageSource(input), cursorProcessor, pageProcessor, TEST_TABLE_HANDLE, ImmutableList.of(), null, ImmutableList.of(BIGINT), new DataSize(64, KILOBYTE), 2, ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), false, Optional.empty(), 0, 0);
    SourceOperator operator = factory.createOperator(newDriverContext());
    operator.addSplit(new Split(new CatalogName("test"), TestingSplit.createLocalSplit(), Lifespan.taskWide()));
    operator.noMoreSplits();
    List<Page> actual = toPages(operator);
    assertEquals(actual.size(), 1);
    List<Page> expected = rowPagesBuilder(BIGINT).row(10L).row(10L).row(10L).row(10L).build();
    assertPageEquals(ImmutableList.of(BIGINT), actual.get(0), expected.get(0));
}

Example 13 with PageProcessor

use of io.prestosql.operator.project.PageProcessor in project hetu-core by openlookeng.

the class TestScanFilterAndProjectOperator method testPageSourceLazyLoad.

@Test
public void testPageSourceLazyLoad() {
    Block inputBlock = BlockAssertions.createLongSequenceBlock(0, 100);
    // If column 1 is loaded, test will fail
    Page input = new Page(100, inputBlock, new LazyBlock(100, lazyBlock -> {
        throw new AssertionError("Lazy block should not be loaded");
    }));
    DriverContext driverContext = newDriverContext();
    List<RowExpression> projections = ImmutableList.of(field(0, VARCHAR));
    Supplier<CursorProcessor> cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.empty(), projections, "key");
    PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new LazyPagePageProjection()));
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns, dynamicFilter) -> new SinglePagePageSource(input), cursorProcessor, () -> pageProcessor, TEST_TABLE_HANDLE, ImmutableList.of(), null, ImmutableList.of(BIGINT), new DataSize(0, BYTE), 0, ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), false, Optional.empty(), 0, 0);
    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new CatalogName("test"), TestingSplit.createLocalSplit(), Lifespan.taskWide()));
    operator.noMoreSplits();
    MaterializedResult expected = toMaterializedResult(driverContext.getSession(), ImmutableList.of(BIGINT), ImmutableList.of(new Page(inputBlock)));
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), ImmutableList.of(BIGINT), toPages(operator));
    assertEquals(actual.getRowCount(), expected.getRowCount());
    assertEquals(actual, expected);
}

Example 14 with PageProcessor

use of io.prestosql.operator.project.PageProcessor in project hetu-core by openlookeng.

the class TestScanFilterAndProjectOperator method testPageYield.

@Test
public void testPageYield() {
    int totalRows = 1000;
    Page input = SequencePageBuilder.createSequencePage(ImmutableList.of(BIGINT), totalRows, 1);
    DriverContext driverContext = newDriverContext();
    // 20 columns; each column is associated with a function that will force yield per projection
    int totalColumns = 20;
    ImmutableList.Builder<SqlScalarFunction> functions = ImmutableList.builder();
    for (int i = 0; i < totalColumns; i++) {
        functions.add(new GenericLongFunction("page_col" + i, value -> {
            driverContext.getYieldSignal().forceYieldForTesting();
            return value;
        }));
    }
    Metadata localMetadata = functionAssertions.getMetadata();
    localMetadata.getFunctionAndTypeManager().registerBuiltInFunctions(functions.build());
    // match each column with a projection
    ExpressionCompiler compiler = new ExpressionCompiler(localMetadata, new PageFunctionCompiler(localMetadata, 0));
    ImmutableList.Builder<RowExpression> projections = ImmutableList.builder();
    for (int i = 0; i < totalColumns; i++) {
        projections.add(call(QualifiedObjectName.valueOfDefaultFunction("generic_long_page_col" + i).toString(), new BuiltInFunctionHandle(internalScalarFunction(QualifiedObjectName.valueOfDefaultFunction("generic_long_page_col" + i), BIGINT.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature()))), BIGINT, field(0, BIGINT)));
    }
    Supplier<CursorProcessor> cursorProcessor = compiler.compileCursorProcessor(Optional.empty(), projections.build(), "key");
    Supplier<PageProcessor> pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections.build(), MAX_BATCH_SIZE);
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns, dynamicFilter) -> new FixedPageSource(ImmutableList.of(input)), cursorProcessor, pageProcessor, TEST_TABLE_HANDLE, ImmutableList.of(), null, ImmutableList.of(BIGINT), new DataSize(0, BYTE), 0, ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), false, Optional.empty(), 0, 0);
    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new CatalogName("test"), TestingSplit.createLocalSplit(), Lifespan.taskWide()));
    operator.noMoreSplits();
    // exactly 20 blocks (one for each column) and the PageProcessor will be able to create a Page out of it.
    for (int i = 1; i <= totalRows * totalColumns; i++) {
        driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
        Page page = operator.getOutput();
        if (i == totalColumns) {
            assertNotNull(page);
            assertEquals(page.getPositionCount(), totalRows);
            assertEquals(page.getChannelCount(), totalColumns);
            for (int j = 0; j < totalColumns; j++) {
                assertEquals(toValues(BIGINT, page.getBlock(j)), toValues(BIGINT, input.getBlock(0)));
            }
        } else {
            assertNull(page);
        }
        driverContext.getYieldSignal().reset();
    }
}

Example 15 with PageProcessor

use of io.prestosql.operator.project.PageProcessor in project hetu-core by openlookeng.

the class TestScanFilterAndProjectOperator method createScanFilterAndProjectOperator.

private SourceOperator createScanFilterAndProjectOperator(String sourceId, UUID uuid, int operatorId, Page input, DriverContext driverContext, ReuseExchangeOperator.STRATEGY strategy, boolean spillEnabled, Integer spillerThreshold, Integer consumerTableScanNodeCount) {
    List<RowExpression> projections = ImmutableList.of(field(0, VARCHAR));
    Supplier<CursorProcessor> cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.empty(), projections, "key");
    Supplier<PageProcessor> pageProcessor = expressionCompiler.compilePageProcessor(Optional.empty(), projections);
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(operatorId, new PlanNodeId("test"), new PlanNodeId(sourceId), (session, split, table, columns, dynamicFilter) -> new FixedPageSource(ImmutableList.of(input)), cursorProcessor, pageProcessor, TEST_TABLE_HANDLE, ImmutableList.of(), null, ImmutableList.of(VARCHAR), new DataSize(0, BYTE), 0, strategy, uuid, spillEnabled, Optional.of(new DummySpillerFactory()), spillerThreshold, consumerTableScanNodeCount);
    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new CatalogName("test"), TestingSplit.createLocalSplit(), Lifespan.taskWide()));
    operator.noMoreSplits();
    return operator;
}