Examples with PageProcessor com.facebook.presto.operator.project.PageProcessor used on opensource projects

Example 11 with PageProcessor

use of com.facebook.presto.operator.project.PageProcessor in project presto by prestodb.

the class TestTupleFilterProcessor method testFilter.

@Test
public void testFilter() {
    Page tuplePage = Iterables.getOnlyElement(rowPagesBuilder(BIGINT, VARCHAR, DOUBLE).row(1L, "a", 0.1).build());
    List<Type> outputTypes = ImmutableList.of(VARCHAR, BIGINT, BOOLEAN, DOUBLE, DOUBLE);
    Page inputPage = Iterables.getOnlyElement(rowPagesBuilder(outputTypes).row("a", 1L, true, 0.1, 0.0).row("b", 1L, true, 0.1, 2.0).row("a", 1L, false, 0.1, 2.0).row("a", 0L, false, 0.2, 0.2).build());
    DynamicTupleFilterFactory filterFactory = new DynamicTupleFilterFactory(42, new PlanNodeId("42"), new int[] { 0, 1, 2 }, new int[] { 1, 0, 3 }, outputTypes, SESSION.getSqlFunctionProperties(), SESSION.getSessionFunctions(), new PageFunctionCompiler(createTestMetadataManager(), 0));
    PageProcessor tupleFilterProcessor = filterFactory.createPageProcessor(tuplePage, OptionalInt.of(MAX_BATCH_SIZE)).get();
    Page actualPage = getOnlyElement(tupleFilterProcessor.process(SESSION.getSqlFunctionProperties(), new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), inputPage)).orElseThrow(() -> new AssertionError("page is not present"));
    Page expectedPage = Iterables.getOnlyElement(rowPagesBuilder(outputTypes).row("a", 1L, true, 0.1, 0.0).row("a", 1L, false, 0.1, 2.0).build());
    assertPageEquals(outputTypes, actualPage, expectedPage);
}

Example 12 with PageProcessor

use of com.facebook.presto.operator.project.PageProcessor in project presto by prestodb.

the class AbstractOperatorBenchmark method createHashProjectOperator.

protected final OperatorFactory createHashProjectOperator(int operatorId, PlanNodeId planNodeId, List<Type> types) {
    ImmutableList.Builder<VariableReferenceExpression> variables = ImmutableList.builder();
    ImmutableMap.Builder<VariableReferenceExpression, Integer> variableToInputMapping = ImmutableMap.builder();
    ImmutableList.Builder<PageProjectionWithOutputs> projections = ImmutableList.builder();
    for (int channel = 0; channel < types.size(); channel++) {
        VariableReferenceExpression variable = new VariableReferenceExpression(Optional.empty(), "h" + channel, types.get(channel));
        variables.add(variable);
        variableToInputMapping.put(variable, channel);
        projections.add(new PageProjectionWithOutputs(new InputPageProjection(channel), new int[] { channel }));
    }
    Optional<RowExpression> hashExpression = HashGenerationOptimizer.getHashExpression(localQueryRunner.getMetadata().getFunctionAndTypeManager(), variables.build());
    verify(hashExpression.isPresent());
    RowExpression translatedHashExpression = translate(hashExpression.get(), variableToInputMapping.build());
    PageFunctionCompiler functionCompiler = new PageFunctionCompiler(localQueryRunner.getMetadata(), 0);
    projections.add(new PageProjectionWithOutputs(functionCompiler.compileProjection(session.getSqlFunctionProperties(), translatedHashExpression, Optional.empty()).get(), new int[] { types.size() }));
    return new FilterAndProjectOperator.FilterAndProjectOperatorFactory(operatorId, planNodeId, () -> new PageProcessor(Optional.empty(), projections.build()), ImmutableList.copyOf(Iterables.concat(types, ImmutableList.of(BIGINT))), getFilterAndProjectMinOutputPageSize(session), getFilterAndProjectMinOutputPageRowCount(session));
}

Example 13 with PageProcessor

use of com.facebook.presto.operator.project.PageProcessor in project presto by prestodb.

the class HandTpchQuery6 method createOperatorFactories.

@Override
protected List<? extends OperatorFactory> createOperatorFactories() {
    // select sum(extendedprice * discount) as revenue
    // from lineitem
    // where shipdate >= '1994-01-01'
    // and shipdate < '1995-01-01'
    // and discount >= 0.05
    // and discount <= 0.07
    // and quantity < 24;
    OperatorFactory tableScanOperator = createTableScanOperator(0, new PlanNodeId("test"), "lineitem", "extendedprice", "discount", "shipdate", "quantity");
    List<Supplier<PageProjectionWithOutputs>> projection = new PageFunctionCompiler(localQueryRunner.getMetadata(), 0).compileProjections(session.getSqlFunctionProperties(), session.getSessionFunctions(), ImmutableList.of(field(0, BIGINT)), false, Optional.empty());
    FilterAndProjectOperator.FilterAndProjectOperatorFactory tpchQuery6Operator = new FilterAndProjectOperator.FilterAndProjectOperatorFactory(1, new PlanNodeId("test"), () -> new PageProcessor(Optional.of(new TpchQuery6Filter()), projection.stream().map(Supplier::get).collect(toImmutableList())), ImmutableList.of(DOUBLE), new DataSize(0, BYTE), 0);
    AggregationOperatorFactory aggregationOperator = new AggregationOperatorFactory(2, new PlanNodeId("test"), Step.SINGLE, ImmutableList.of(doubleSum.bind(ImmutableList.of(0), Optional.empty())), false);
    return ImmutableList.of(tableScanOperator, tpchQuery6Operator, aggregationOperator);
}

Example 14 with PageProcessor

use of com.facebook.presto.operator.project.PageProcessor in project presto by prestodb.

the class FunctionAssertions method assertCachedInstanceHasBoundedRetainedSize.

public void assertCachedInstanceHasBoundedRetainedSize(String projection) {
    requireNonNull(projection, "projection is null");
    Expression projectionExpression = createExpression(session, projection, metadata, SYMBOL_TYPES);
    RowExpression projectionRowExpression = toRowExpression(session, projectionExpression);
    PageProcessor processor = compiler.compilePageProcessor(session.getSqlFunctionProperties(), Optional.empty(), ImmutableList.of(projectionRowExpression)).get();
    // This is a heuristic to detect whether the retained size of cachedInstance is bounded.
    // * The test runs at least 1000 iterations.
    // * The test passes if max retained size doesn't refresh after
    // 4x the number of iterations when max was last updated.
    // * The test fails if retained size reaches 1MB.
    // Note that 1MB is arbitrarily chosen and may be increased if a function implementation
    // legitimately needs more.
    long maxRetainedSize = 0;
    int maxIterationCount = 0;
    for (int iterationCount = 0; iterationCount < Math.max(1000, maxIterationCount * 4); iterationCount++) {
        Iterator<Optional<Page>> output = processor.process(session.getSqlFunctionProperties(), new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), SOURCE_PAGE);
        // consume the iterator
        Iterators.getOnlyElement(output);
        long retainedSize = processor.getProjections().stream().mapToLong(this::getRetainedSizeOfCachedInstance).sum();
        if (retainedSize > maxRetainedSize) {
            maxRetainedSize = retainedSize;
            maxIterationCount = iterationCount;
        }
        if (maxRetainedSize >= 1048576) {
            fail(format("The retained size of cached instance of function invocation is likely unbounded: %s", projection));
        }
    }
}

Example 15 with PageProcessor

use of com.facebook.presto.operator.project.PageProcessor in project presto by prestodb.

the class TestPageProcessorCompiler method testSanityFilterOnRLE.

@Test
public void testSanityFilterOnRLE() {
    FunctionAndTypeManager functionAndTypeManager = createTestMetadataManager().getFunctionAndTypeManager();
    FunctionHandle lessThan = functionAndTypeManager.resolveOperator(LESS_THAN, fromTypes(BIGINT, BIGINT));
    CallExpression filter = new CallExpression(LESS_THAN.name(), lessThan, BOOLEAN, ImmutableList.of(field(0, BIGINT), constant(10L, BIGINT)));
    PageProcessor processor = compiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.of(filter), ImmutableList.of(field(0, BIGINT)), false, MAX_BATCH_SIZE).get();
    Page page = new Page(createRLEBlock(5L, 100));
    Page outputPage = getOnlyElement(processor.process(null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertEquals(outputPage.getPositionCount(), 100);
    assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock);
    RunLengthEncodedBlock rle = (RunLengthEncodedBlock) outputPage.getBlock(0);
    assertEquals(BIGINT.getLong(rle.getValue(), 0), 5L);
}