Search in sources :

Example 1 with PageFunctionCompiler

use of io.prestosql.sql.gen.PageFunctionCompiler in project hetu-core by openlookeng.

the class TestScanFilterAndProjectOperator method testRecordCursorYield.

@Test
public void testRecordCursorYield() {
    // create a generic long function that yields for projection on every row
    // verify we will yield #row times totally
    // create a table with 15 rows
    int length = 15;
    Page input = SequencePageBuilder.createSequencePage(ImmutableList.of(BIGINT), length, 0);
    DriverContext driverContext = newDriverContext();
    // set up generic long function with a callback to force yield
    Metadata localMetadata = functionAssertions.getMetadata();
    localMetadata.getFunctionAndTypeManager().registerBuiltInFunctions(ImmutableList.of(new GenericLongFunction("record_cursor", value -> {
        driverContext.getYieldSignal().forceYieldForTesting();
        return value;
    })));
    ExpressionCompiler compiler = new ExpressionCompiler(localMetadata, new PageFunctionCompiler(localMetadata, 0));
    List<RowExpression> projections = ImmutableList.of(call(QualifiedObjectName.valueOfDefaultFunction("generic_long_record_cursor").toString(), new BuiltInFunctionHandle(internalScalarFunction(QualifiedObjectName.valueOfDefaultFunction("generic_long_record_cursor"), BIGINT.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature()))), BIGINT, field(0, BIGINT)));
    Supplier<CursorProcessor> cursorProcessor = compiler.compileCursorProcessor(Optional.empty(), projections, "key");
    Supplier<PageProcessor> pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections);
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns, dynamicFilter) -> new RecordPageSource(new PageRecordSet(ImmutableList.of(BIGINT), input)), cursorProcessor, pageProcessor, TEST_TABLE_HANDLE, ImmutableList.of(), null, ImmutableList.of(BIGINT), new DataSize(0, BYTE), 0, ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), false, Optional.empty(), 0, 0);
    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new CatalogName("test"), TestingSplit.createLocalSplit(), Lifespan.taskWide()));
    operator.noMoreSplits();
    // start driver; get null value due to yield for the first 15 times
    for (int i = 0; i < length; i++) {
        driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
        assertNull(operator.getOutput());
        driverContext.getYieldSignal().reset();
    }
    // the 16th yield is not going to prevent the operator from producing a page
    driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
    Page output = operator.getOutput();
    driverContext.getYieldSignal().reset();
    assertNotNull(output);
    assertEquals(toValues(BIGINT, output.getBlock(0)), toValues(BIGINT, input.getBlock(0)));
}
Also used : PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) CursorProcessor(io.prestosql.operator.project.CursorProcessor) Metadata(io.prestosql.metadata.Metadata) Page(io.prestosql.spi.Page) PageRecordSet(io.prestosql.operator.index.PageRecordSet) RecordPageSource(io.prestosql.spi.connector.RecordPageSource) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) PageProcessor(io.prestosql.operator.project.PageProcessor) DataSize(io.airlift.units.DataSize) UUID(java.util.UUID) RowExpression(io.prestosql.spi.relation.RowExpression) BuiltInFunctionHandle(io.prestosql.spi.function.BuiltInFunctionHandle) ExpressionCompiler(io.prestosql.sql.gen.ExpressionCompiler) CatalogName(io.prestosql.spi.connector.CatalogName) Split(io.prestosql.metadata.Split) TestingSplit(io.prestosql.testing.TestingSplit) Test(org.testng.annotations.Test)

Example 2 with PageFunctionCompiler

use of io.prestosql.sql.gen.PageFunctionCompiler in project hetu-core by openlookeng.

the class BenchmarkEqualsOperator method setup.

@Setup
public void setup() {
    Metadata metadata = createTestMetadataManager();
    ExpressionCompiler expressionCompiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0));
    RowExpression projection = generateComplexComparisonProjection(FIELDS_COUNT, COMPARISONS_COUNT);
    compiledProcessor = expressionCompiler.compilePageProcessor(Optional.empty(), ImmutableList.of(projection)).get();
}
Also used : PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) Metadata(io.prestosql.metadata.Metadata) RowExpression(io.prestosql.spi.relation.RowExpression) ExpressionCompiler(io.prestosql.sql.gen.ExpressionCompiler) Setup(org.openjdk.jmh.annotations.Setup)

Example 3 with PageFunctionCompiler

use of io.prestosql.sql.gen.PageFunctionCompiler in project hetu-core by openlookeng.

the class TestTupleFilterProcessor method testFilter.

@Test
public void testFilter() {
    Page tuplePage = Iterables.getOnlyElement(rowPagesBuilder(BIGINT, VARCHAR, DOUBLE).row(1L, "a", 0.1).build());
    List<Type> outputTypes = ImmutableList.of(VARCHAR, BIGINT, BOOLEAN, DOUBLE, DOUBLE);
    Page inputPage = Iterables.getOnlyElement(rowPagesBuilder(outputTypes).row("a", 1L, true, 0.1, 0.0).row("b", 1L, true, 0.1, 2.0).row("a", 1L, false, 0.1, 2.0).row("a", 0L, false, 0.2, 0.2).build());
    DynamicTupleFilterFactory filterFactory = new DynamicTupleFilterFactory(42, new PlanNodeId("42"), new int[] { 0, 1, 2 }, new int[] { 1, 0, 3 }, outputTypes, new PageFunctionCompiler(createTestMetadataManager(), 0));
    PageProcessor tupleFilterProcessor = filterFactory.createPageProcessor(tuplePage, OptionalInt.of(MAX_BATCH_SIZE)).get();
    Page actualPage = getOnlyElement(tupleFilterProcessor.process(SESSION, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), inputPage)).orElseThrow(() -> new AssertionError("page is not present"));
    Page expectedPage = Iterables.getOnlyElement(rowPagesBuilder(outputTypes).row("a", 1L, true, 0.1, 0.0).row("a", 1L, false, 0.1, 2.0).build());
    assertPageEquals(outputTypes, actualPage, expectedPage);
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Type(io.prestosql.spi.type.Type) PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) PageProcessor(io.prestosql.operator.project.PageProcessor) DriverYieldSignal(io.prestosql.operator.DriverYieldSignal) Page(io.prestosql.spi.Page) Test(org.testng.annotations.Test)

Example 4 with PageFunctionCompiler

use of io.prestosql.sql.gen.PageFunctionCompiler in project hetu-core by openlookeng.

the class HandTpchQuery6 method createOperatorFactories.

@Override
protected List<? extends OperatorFactory> createOperatorFactories() {
    // select sum(extendedprice * discount) as revenue
    // from lineitem
    // where shipdate >= '1994-01-01'
    // and shipdate < '1995-01-01'
    // and discount >= 0.05
    // and discount <= 0.07
    // and quantity < 24;
    OperatorFactory tableScanOperator = createTableScanOperator(0, new PlanNodeId("test"), "lineitem", "extendedprice", "discount", "shipdate", "quantity");
    Supplier<PageProjection> projection = new PageFunctionCompiler(localQueryRunner.getMetadata(), 0).compileProjection(field(0, BIGINT), Optional.empty());
    FilterAndProjectOperator.FilterAndProjectOperatorFactory tpchQuery6Operator = new FilterAndProjectOperator.FilterAndProjectOperatorFactory(1, new PlanNodeId("test"), () -> new PageProcessor(Optional.of(new TpchQuery6Filter()), ImmutableList.of(projection.get())), ImmutableList.of(DOUBLE), new DataSize(0, BYTE), 0);
    AggregationOperatorFactory aggregationOperator = new AggregationOperatorFactory(2, new PlanNodeId("test"), Step.SINGLE, ImmutableList.of(doubleSum.bind(ImmutableList.of(0), Optional.empty())), false);
    return ImmutableList.of(tableScanOperator, tpchQuery6Operator, aggregationOperator);
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) PageProjection(io.prestosql.operator.project.PageProjection) PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) PageProcessor(io.prestosql.operator.project.PageProcessor) OperatorFactory(io.prestosql.operator.OperatorFactory) AggregationOperatorFactory(io.prestosql.operator.AggregationOperator.AggregationOperatorFactory) AggregationOperatorFactory(io.prestosql.operator.AggregationOperator.AggregationOperatorFactory) DataSize(io.airlift.units.DataSize) FilterAndProjectOperator(io.prestosql.operator.FilterAndProjectOperator)

Example 5 with PageFunctionCompiler

use of io.prestosql.sql.gen.PageFunctionCompiler in project hetu-core by openlookeng.

the class AbstractOperatorBenchmark method createHashProjectOperator.

protected final OperatorFactory createHashProjectOperator(int operatorId, PlanNodeId planNodeId, List<Type> types) {
    PlanSymbolAllocator planSymbolAllocator = new PlanSymbolAllocator();
    ImmutableMap.Builder<Symbol, Integer> symbolToInputMapping = ImmutableMap.builder();
    ImmutableList.Builder<PageProjection> projections = ImmutableList.builder();
    for (int channel = 0; channel < types.size(); channel++) {
        Symbol symbol = planSymbolAllocator.newSymbol("h" + channel, types.get(channel));
        symbolToInputMapping.put(symbol, channel);
        projections.add(new InputPageProjection(channel, types.get(channel)));
    }
    Map<Symbol, Type> symbolTypes = planSymbolAllocator.getTypes().allTypes();
    Optional<Expression> hashExpression = HashGenerationOptimizer.getHashExpression(localQueryRunner.getMetadata(), planSymbolAllocator, ImmutableList.copyOf(symbolTypes.keySet()));
    verify(hashExpression.isPresent());
    Map<NodeRef<Expression>, Type> expressionTypes = new TypeAnalyzer(localQueryRunner.getSqlParser(), localQueryRunner.getMetadata()).getTypes(session, TypeProvider.copyOf(symbolTypes), hashExpression.get());
    RowExpression translated = translate(hashExpression.get(), SCALAR, expressionTypes, symbolToInputMapping.build(), localQueryRunner.getMetadata().getFunctionAndTypeManager(), session, false);
    PageFunctionCompiler functionCompiler = new PageFunctionCompiler(localQueryRunner.getMetadata(), 0);
    projections.add(functionCompiler.compileProjection(translated, Optional.empty()).get());
    return new FilterAndProjectOperator.FilterAndProjectOperatorFactory(operatorId, planNodeId, () -> new PageProcessor(Optional.empty(), projections.build()), ImmutableList.copyOf(Iterables.concat(types, ImmutableList.of(BIGINT))), getFilterAndProjectMinOutputPageSize(session), getFilterAndProjectMinOutputPageRowCount(session));
}
Also used : PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) Symbol(io.prestosql.spi.plan.Symbol) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ImmutableList(com.google.common.collect.ImmutableList) RowExpression(io.prestosql.spi.relation.RowExpression) PlanSymbolAllocator(io.prestosql.sql.planner.PlanSymbolAllocator) ImmutableMap(com.google.common.collect.ImmutableMap) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) PageProjection(io.prestosql.operator.project.PageProjection) InputPageProjection(io.prestosql.operator.project.InputPageProjection) NodeRef(io.prestosql.sql.tree.NodeRef) Type(io.prestosql.spi.type.Type) InputPageProjection(io.prestosql.operator.project.InputPageProjection) PageProcessor(io.prestosql.operator.project.PageProcessor) RowExpression(io.prestosql.spi.relation.RowExpression) Expression(io.prestosql.sql.tree.Expression)

Aggregations

PageFunctionCompiler (io.prestosql.sql.gen.PageFunctionCompiler)13 PageProcessor (io.prestosql.operator.project.PageProcessor)9 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)9 ExpressionCompiler (io.prestosql.sql.gen.ExpressionCompiler)9 RowExpression (io.prestosql.spi.relation.RowExpression)8 DataSize (io.airlift.units.DataSize)7 Metadata (io.prestosql.metadata.Metadata)7 Page (io.prestosql.spi.Page)7 Test (org.testng.annotations.Test)7 BuiltInFunctionHandle (io.prestosql.spi.function.BuiltInFunctionHandle)5 ImmutableList (com.google.common.collect.ImmutableList)3 Split (io.prestosql.metadata.Split)3 MaterializedResult (io.prestosql.testing.MaterializedResult)3 ImmutableMap (com.google.common.collect.ImmutableMap)2 TEST_SESSION (io.prestosql.SessionTestUtils.TEST_SESSION)2 MetadataManager.createTestMetadataManager (io.prestosql.metadata.MetadataManager.createTestMetadataManager)2 PageProjection (io.prestosql.operator.project.PageProjection)2 Block (io.prestosql.spi.block.Block)2 LazyBlock (io.prestosql.spi.block.LazyBlock)2 CatalogName (io.prestosql.spi.connector.CatalogName)2