Examples with PageProcessor io.trino.operator.project.PageProcessor used on opensource projects

Example 16 with PageProcessor

use of io.trino.operator.project.PageProcessor in project trino by trinodb.

the class TestScanFilterAndProjectOperator method testRecordCursorYield.

@Test
public void testRecordCursorYield() {
    // create a generic long function that yields for projection on every row
    // verify we will yield #row times totally
    // create a table with 15 rows
    int length = 15;
    Page input = SequencePageBuilder.createSequencePage(ImmutableList.of(BIGINT), length, 0);
    DriverContext driverContext = newDriverContext();
    // set up generic long function with a callback to force yield
    functionAssertions.addFunctions(new InternalFunctionBundle(new GenericLongFunction("record_cursor", value -> {
        driverContext.getYieldSignal().forceYieldForTesting();
        return value;
    })));
    FunctionManager functionManager = functionAssertions.getFunctionManager();
    ExpressionCompiler expressionCompiler = new ExpressionCompiler(functionManager, new PageFunctionCompiler(functionManager, 0));
    List<RowExpression> projections = ImmutableList.of(call(functionAssertions.getMetadata().resolveFunction(session, QualifiedName.of("generic_long_record_cursor"), fromTypes(BIGINT)), field(0, BIGINT)));
    Supplier<CursorProcessor> cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.empty(), projections, "key");
    Supplier<PageProcessor> pageProcessor = expressionCompiler.compilePageProcessor(Optional.empty(), projections);
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns, dynamicFilter) -> new RecordPageSource(new PageRecordSet(ImmutableList.of(BIGINT), input)), cursorProcessor, pageProcessor, TEST_TABLE_HANDLE, ImmutableList.of(), DynamicFilter.EMPTY, ImmutableList.of(BIGINT), DataSize.ofBytes(0), 0);
    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new CatalogName("test"), TestingSplit.createLocalSplit(), Lifespan.taskWide()));
    operator.noMoreSplits();
    // start driver; get null value due to yield for the first 15 times
    for (int i = 0; i < length; i++) {
        driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
        assertNull(operator.getOutput());
        driverContext.getYieldSignal().reset();
    }
    // the 16th yield is not going to prevent the operator from producing a page
    driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
    Page output = operator.getOutput();
    driverContext.getYieldSignal().reset();
    assertNotNull(output);
    assertEquals(toValues(BIGINT, output.getBlock(0)), toValues(BIGINT, input.getBlock(0)));
}

Example 17 with PageProcessor

use of io.trino.operator.project.PageProcessor in project trino by trinodb.

the class FunctionAssertions method assertCachedInstanceHasBoundedRetainedSizeInTx.

private void assertCachedInstanceHasBoundedRetainedSizeInTx(String projection, Session session) {
    requireNonNull(projection, "projection is null");
    Expression projectionExpression = createExpression(session, projection, getPlannerContext(), INPUT_TYPES);
    RowExpression projectionRowExpression = toRowExpression(session, projectionExpression);
    PageProcessor processor = runner.getExpressionCompiler().compilePageProcessor(Optional.empty(), ImmutableList.of(projectionRowExpression)).get();
    // This is a heuristic to detect whether the retained size of cachedInstance is bounded.
    // * The test runs at least 1000 iterations.
    // * The test passes if max retained size doesn't refresh after
    // 4x the number of iterations when max was last updated.
    // * The test fails if retained size reaches 1MB.
    // Note that 1MB is arbitrarily chosen and may be increased if a function implementation
    // legitimately needs more.
    long maxRetainedSize = 0;
    int maxIterationCount = 0;
    for (int iterationCount = 0; iterationCount < Math.max(1000, maxIterationCount * 4); iterationCount++) {
        Iterator<Optional<Page>> output = processor.process(session.toConnectorSession(), new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), SOURCE_PAGE);
        // consume the iterator
        @SuppressWarnings("unused") Optional<Page> ignored = Iterators.getOnlyElement(output);
        long retainedSize = processor.getProjections().stream().mapToLong(this::getRetainedSizeOfCachedInstance).sum();
        if (retainedSize > maxRetainedSize) {
            maxRetainedSize = retainedSize;
            maxIterationCount = iterationCount;
        }
        if (maxRetainedSize >= 1048576) {
            fail(format("The retained size of cached instance of function invocation is likely unbounded: %s", projection));
        }
    }
}

Example 18 with PageProcessor

use of io.trino.operator.project.PageProcessor in project trino by trinodb.

the class ExpressionCompiler method compilePageProcessor.

private Supplier<PageProcessor> compilePageProcessor(Optional<RowExpression> filter, List<? extends RowExpression> projections, Optional<String> classNameSuffix, OptionalInt initialBatchSize) {
    Optional<Supplier<PageFilter>> filterFunctionSupplier = filter.map(expression -> pageFunctionCompiler.compileFilter(expression, classNameSuffix));
    List<Supplier<PageProjection>> pageProjectionSuppliers = projections.stream().map(projection -> pageFunctionCompiler.compileProjection(projection, classNameSuffix)).collect(toImmutableList());
    return () -> {
        Optional<PageFilter> filterFunction = filterFunctionSupplier.map(Supplier::get);
        List<PageProjection> pageProjections = pageProjectionSuppliers.stream().map(Supplier::get).collect(toImmutableList());
        return new PageProcessor(filterFunction, pageProjections, initialBatchSize);
    };
}

Example 19 with PageProcessor

use of io.trino.operator.project.PageProcessor in project trino by trinodb.

the class TestColumnarPageProcessor method testProcess.

@Test
public void testProcess() {
    PageProcessor processor = newPageProcessor();
    Page page = createPage(types, false);
    Page outputPage = getOnlyElement(processor.process(SESSION, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertPageEquals(types, outputPage, page);
}

Example 20 with PageProcessor

use of io.trino.operator.project.PageProcessor in project trino by trinodb.

the class TestColumnarPageProcessor method testProcessWithDictionary.

@Test
public void testProcessWithDictionary() {
    PageProcessor processor = newPageProcessor();
    Page page = createPage(types, true);
    Page outputPage = getOnlyElement(processor.process(SESSION, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertPageEquals(types, outputPage, page);
}