Search in sources :

Example 1 with WindowFunction

use of io.crate.expression.symbol.WindowFunction in project crate by crate.

the class WindowAgg method build.

@Override
public ExecutionPlan build(PlannerContext plannerContext, Set<PlanHint> planHints, ProjectionBuilder projectionBuilder, int limit, int offset, @Nullable OrderBy order, @Nullable Integer pageSizeHint, Row params, SubQueryResults subQueryResults) {
    InputColumns.SourceSymbols sourceSymbols = new InputColumns.SourceSymbols(source.outputs());
    SubQueryAndParamBinder binder = new SubQueryAndParamBinder(params, subQueryResults);
    Function<Symbol, Symbol> toInputCols = binder.andThen(s -> InputColumns.create(s, sourceSymbols));
    List<WindowFunction> boundWindowFunctions = (List<WindowFunction>) (List) Lists2.map(windowFunctions, toInputCols);
    List<Projection> projections = new ArrayList<>();
    WindowAggProjection windowAggProjection = new WindowAggProjection(windowDefinition.map(toInputCols), boundWindowFunctions, InputColumns.create(this.standalone, sourceSymbols));
    projections.add(windowAggProjection);
    ExecutionPlan sourcePlan = source.build(plannerContext, planHints, projectionBuilder, TopN.NO_LIMIT, TopN.NO_OFFSET, null, pageSizeHint, params, subQueryResults);
    ResultDescription resultDescription = sourcePlan.resultDescription();
    boolean executesOnHandler = executesOnHandler(plannerContext.handlerNode(), resultDescription.nodeIds());
    boolean nonDistExecution = windowDefinition.partitions().isEmpty() || resultDescription.hasRemainingLimitOrOffset() || executesOnHandler;
    if (nonDistExecution) {
        sourcePlan = Merge.ensureOnHandler(sourcePlan, plannerContext);
        for (Projection projection : projections) {
            sourcePlan.addProjection(projection);
        }
    } else {
        sourcePlan.setDistributionInfo(new DistributionInfo(DistributionType.MODULO, source.outputs().indexOf(windowDefinition.partitions().iterator().next())));
        MergePhase distWindowAgg = new MergePhase(UUIDs.dirtyUUID(), plannerContext.nextExecutionPhaseId(), "distWindowAgg", resultDescription.nodeIds().size(), resultDescription.numOutputs(), resultDescription.nodeIds(), resultDescription.streamOutputs(), projections, DistributionInfo.DEFAULT_BROADCAST, null);
        return new Merge(sourcePlan, distWindowAgg, TopN.NO_LIMIT, TopN.NO_OFFSET, windowAggProjection.outputs().size(), resultDescription.maxRowsPerNode(), null);
    }
    return sourcePlan;
}
Also used : InputColumns(io.crate.execution.dsl.projection.builder.InputColumns) Symbol(io.crate.expression.symbol.Symbol) ArrayList(java.util.ArrayList) Projection(io.crate.execution.dsl.projection.Projection) WindowAggProjection(io.crate.execution.dsl.projection.WindowAggProjection) DistributionInfo(io.crate.planner.distribution.DistributionInfo) WindowFunction(io.crate.expression.symbol.WindowFunction) ExecutionPlan(io.crate.planner.ExecutionPlan) MergePhase(io.crate.execution.dsl.phases.MergePhase) Merge(io.crate.planner.Merge) ResultDescription(io.crate.planner.ResultDescription) WindowAggProjection(io.crate.execution.dsl.projection.WindowAggProjection) ArrayList(java.util.ArrayList) List(java.util.List)

Example 2 with WindowFunction

use of io.crate.expression.symbol.WindowFunction in project crate by crate.

the class MoveFilterBeneathWindowAgg method apply.

@Override
public LogicalPlan apply(Filter filter, Captures captures, TableStats tableStats, TransactionContext txnCtx, NodeContext nodeCtx) {
    WindowAgg windowAgg = captures.get(windowAggCapture);
    WindowDefinition windowDefinition = windowAgg.windowDefinition();
    List<WindowFunction> windowFunctions = windowAgg.windowFunctions();
    Predicate<Symbol> containsWindowFunction = symbol -> symbol instanceof WindowFunction && windowFunctions.contains(symbol);
    Symbol predicate = filter.query();
    List<Symbol> filterParts = AndOperator.split(predicate);
    ArrayList<Symbol> remainingFilterSymbols = new ArrayList<>();
    ArrayList<Symbol> windowPartitionedBasedFilters = new ArrayList<>();
    for (Symbol part : filterParts) {
        if (SymbolVisitors.any(containsWindowFunction, part) == false && windowDefinition.partitions().containsAll(extractColumns(part))) {
            windowPartitionedBasedFilters.add(part);
        } else {
            remainingFilterSymbols.add(part);
        }
    }
    assert remainingFilterSymbols.size() > 0 || windowPartitionedBasedFilters.size() > 0 : "Splitting the filter symbol must result in at least one group";
    /* SELECT ROW_NUMBER() OVER(PARTITION BY id)
         * WHERE `x = 1`
         *
         * `x` is not the window partition column.
         * We cannot push down the filter as it would change the window aggregation value
         *
         */
    if (windowPartitionedBasedFilters.isEmpty()) {
        return null;
    }
    /* SELECT ROW_NUMBER() OVER(PARTITION BY id)
         * WHERE `id = 1`
         * remainingFilterSymbols:                  []
         * windowPartitionedBasedFilters:           [id = 1]
         *
         * Filter
         *  |
         * WindowsAgg
         *
         * transforms to
         *
         * WindowAgg
         *  |
         * Filter (id = 1)
         */
    if (remainingFilterSymbols.isEmpty()) {
        return transpose(filter, windowAgg);
    }
    /* WHERE `ROW_NUMBER() OVER(PARTITION BY id) = 2 AND id = 1`
         * remainingFilterSymbols:                  [ROW_NUMBER() OVER(PARTITION BY id) = 2]
         * windowPartitionedBasedFilters:           [id = 1]
         *
         * Filter
         *  |
         * WindowsAgg
         *
         * transforms to
         *
         * Filter (ROW_NUMBER() OVER(PARTITION BY id) = 2)
         *  |
         * WindowAgg
         *  |
         * Filter (id = 1)
         */
    LogicalPlan newWindowAgg = windowAgg.replaceSources(List.of(new Filter(windowAgg.source(), AndOperator.join(windowPartitionedBasedFilters))));
    return new Filter(newWindowAgg, AndOperator.join(remainingFilterSymbols));
}
Also used : TransactionContext(io.crate.metadata.TransactionContext) NodeContext(io.crate.metadata.NodeContext) LogicalPlanner.extractColumns(io.crate.planner.operators.LogicalPlanner.extractColumns) LogicalPlan(io.crate.planner.operators.LogicalPlan) Captures(io.crate.planner.optimizer.matcher.Captures) Predicate(java.util.function.Predicate) Util.transpose(io.crate.planner.optimizer.rule.Util.transpose) Pattern(io.crate.planner.optimizer.matcher.Pattern) Rule(io.crate.planner.optimizer.Rule) SymbolVisitors(io.crate.expression.symbol.SymbolVisitors) ArrayList(java.util.ArrayList) List(java.util.List) AndOperator(io.crate.expression.operator.AndOperator) TableStats(io.crate.statistics.TableStats) WindowDefinition(io.crate.analyze.WindowDefinition) Symbol(io.crate.expression.symbol.Symbol) Pattern.typeOf(io.crate.planner.optimizer.matcher.Pattern.typeOf) WindowFunction(io.crate.expression.symbol.WindowFunction) Patterns.source(io.crate.planner.optimizer.matcher.Patterns.source) Filter(io.crate.planner.operators.Filter) WindowAgg(io.crate.planner.operators.WindowAgg) Capture(io.crate.planner.optimizer.matcher.Capture) WindowFunction(io.crate.expression.symbol.WindowFunction) WindowAgg(io.crate.planner.operators.WindowAgg) Filter(io.crate.planner.operators.Filter) Symbol(io.crate.expression.symbol.Symbol) ArrayList(java.util.ArrayList) LogicalPlan(io.crate.planner.operators.LogicalPlan) WindowDefinition(io.crate.analyze.WindowDefinition)

Example 3 with WindowFunction

use of io.crate.expression.symbol.WindowFunction in project crate by crate.

the class InputColumns method visitWindowFunction.

@Override
public Symbol visitWindowFunction(WindowFunction windowFunction, SourceSymbols sourceSymbols) {
    Symbol functionFromSource = getFunctionReplacementOrNull(windowFunction, sourceSymbols);
    if (functionFromSource != null) {
        return functionFromSource;
    }
    ArrayList<Symbol> replacedFunctionArgs = getProcessedArgs(windowFunction.arguments(), sourceSymbols);
    Symbol filterWithReplacedArgs;
    Symbol filter = windowFunction.filter();
    if (filter != null) {
        filterWithReplacedArgs = filter.accept(this, sourceSymbols);
    } else {
        filterWithReplacedArgs = null;
    }
    return new WindowFunction(windowFunction.signature(), replacedFunctionArgs, windowFunction.valueType(), filterWithReplacedArgs, windowFunction.windowDefinition().map(x -> x.accept(this, sourceSymbols)), windowFunction.ignoreNulls());
}
Also used : AliasSymbol(io.crate.expression.symbol.AliasSymbol) InputColumn(io.crate.expression.symbol.InputColumn) SubscriptObjectFunction(io.crate.expression.scalar.SubscriptObjectFunction) ParameterSymbol(io.crate.expression.symbol.ParameterSymbol) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) SelectSymbol(io.crate.expression.symbol.SelectSymbol) FetchReference(io.crate.expression.symbol.FetchReference) ScopedSymbol(io.crate.expression.symbol.ScopedSymbol) DefaultTraversalSymbolVisitor(io.crate.expression.symbol.DefaultTraversalSymbolVisitor) Symbols.lookupValueByColumn(io.crate.expression.symbol.Symbols.lookupValueByColumn) Nullable(javax.annotation.Nullable) IdentityHashMap(java.util.IdentityHashMap) Aggregation(io.crate.expression.symbol.Aggregation) GeneratedReference(io.crate.metadata.GeneratedReference) SymbolType(io.crate.expression.symbol.SymbolType) Collection(java.util.Collection) ColumnIdent(io.crate.metadata.ColumnIdent) Reference(io.crate.metadata.Reference) MatchPredicate(io.crate.expression.symbol.MatchPredicate) DataType(io.crate.types.DataType) FetchMarker(io.crate.expression.symbol.FetchMarker) Function(io.crate.expression.symbol.Function) FetchStub(io.crate.expression.symbol.FetchStub) Objects(java.util.Objects) List(java.util.List) Literal(io.crate.expression.symbol.Literal) Lists2.mapTail(io.crate.common.collections.Lists2.mapTail) Symbol(io.crate.expression.symbol.Symbol) Singleton(org.elasticsearch.common.inject.Singleton) WindowFunction(io.crate.expression.symbol.WindowFunction) WindowFunction(io.crate.expression.symbol.WindowFunction) AliasSymbol(io.crate.expression.symbol.AliasSymbol) ParameterSymbol(io.crate.expression.symbol.ParameterSymbol) SelectSymbol(io.crate.expression.symbol.SelectSymbol) ScopedSymbol(io.crate.expression.symbol.ScopedSymbol) Symbol(io.crate.expression.symbol.Symbol)

Example 4 with WindowFunction

use of io.crate.expression.symbol.WindowFunction in project crate by crate.

the class SplitPointsBuilder method create.

public static SplitPoints create(QueriedSelectRelation relation) {
    Context context = new Context();
    INSTANCE.process(relation.outputs(), context);
    OrderBy orderBy = relation.orderBy();
    if (orderBy != null) {
        INSTANCE.process(orderBy.orderBySymbols(), context);
    }
    Symbol having = relation.having();
    if (having != null) {
        having.accept(INSTANCE, context);
    }
    LinkedHashSet<Symbol> toCollect = new LinkedHashSet<>();
    for (Function tableFunction : context.tableFunctions) {
        toCollect.addAll(extractColumns(tableFunction.arguments()));
    }
    for (Function aggregate : context.aggregates) {
        toCollect.addAll(aggregate.arguments());
        if (aggregate.filter() != null) {
            toCollect.add(aggregate.filter());
        }
    }
    for (WindowFunction windowFunction : context.windowFunctions) {
        toCollect.addAll(extractColumns(windowFunction.arguments()));
        if (windowFunction.filter() != null) {
            toCollect.add(windowFunction.filter());
        }
        INSTANCE.process(windowFunction.windowDefinition().partitions(), context);
        OrderBy windowOrderBy = windowFunction.windowDefinition().orderBy();
        if (windowOrderBy != null) {
            INSTANCE.process(windowOrderBy.orderBySymbols(), context);
        }
    }
    // group-by symbols must be processed on a dedicated context to be able extract table functions which must
    // be processed *below* a grouping operator
    var groupByContext = new Context();
    if (relation.groupBy().isEmpty() == false) {
        INSTANCE.process(relation.groupBy(), groupByContext);
        for (Function tableFunction : groupByContext.tableFunctions) {
            toCollect.addAll(extractColumns(tableFunction.arguments()));
        }
        toCollect.addAll(groupByContext.standalone);
        context.tableFunctions.removeAll(groupByContext.tableFunctions);
    } else if (context.aggregates.isEmpty() && relation.groupBy().isEmpty()) {
        toCollect.addAll(context.standalone);
    }
    return new SplitPoints(new ArrayList<>(toCollect), context.aggregates, context.tableFunctions, groupByContext.tableFunctions, context.windowFunctions);
}
Also used : OrderBy(io.crate.analyze.OrderBy) LinkedHashSet(java.util.LinkedHashSet) WindowFunction(io.crate.expression.symbol.WindowFunction) Function(io.crate.expression.symbol.Function) WindowFunction(io.crate.expression.symbol.WindowFunction) Symbol(io.crate.expression.symbol.Symbol)

Example 5 with WindowFunction

use of io.crate.expression.symbol.WindowFunction in project crate by crate.

the class WindowAggProjectionSerialisationTest method test_window_agg_projection_serialization_with_filter_before_4_1_0.

@Test
public void test_window_agg_projection_serialization_with_filter_before_4_1_0() throws IOException {
    FunctionImplementation sumFunctionImpl = getSumFunction();
    WindowDefinition partitionByOneWindowDef = new WindowDefinition(singletonList(Literal.of(1L)), null, null);
    WindowFunction windowFunction = new WindowFunction(sumFunctionImpl.signature(), singletonList(Literal.of(2L)), sumFunctionImpl.boundSignature().getReturnType().createType(), null, partitionByOneWindowDef, null);
    Symbol standaloneInput = Literal.of(42L);
    var windowAggProjection = new WindowAggProjection(partitionByOneWindowDef, List.of(windowFunction), List.of(standaloneInput));
    var output = new BytesStreamOutput();
    output.setVersion(Version.V_4_0_0);
    windowAggProjection.writeTo(output);
    var input = output.bytes().streamInput();
    input.setVersion(Version.V_4_0_0);
    var actualWindowAggProjection = new WindowAggProjection(input);
    assertThat(actualWindowAggProjection.outputs(), contains(standaloneInput, windowFunction));
    assertThat(actualWindowAggProjection.windowFunctions().get(0).filter(), Matchers.nullValue());
}
Also used : WindowFunction(io.crate.expression.symbol.WindowFunction) Symbol(io.crate.expression.symbol.Symbol) FunctionImplementation(io.crate.metadata.FunctionImplementation) WindowDefinition(io.crate.analyze.WindowDefinition) BytesStreamOutput(org.elasticsearch.common.io.stream.BytesStreamOutput) Test(org.junit.Test)

Aggregations

WindowFunction (io.crate.expression.symbol.WindowFunction)24 Symbol (io.crate.expression.symbol.Symbol)21 Test (org.junit.Test)16 CrateDummyClusterServiceUnitTest (io.crate.test.integration.CrateDummyClusterServiceUnitTest)14 Filter (io.crate.planner.operators.Filter)6 LogicalPlan (io.crate.planner.operators.LogicalPlan)6 WindowAgg (io.crate.planner.operators.WindowAgg)6 TableStats (io.crate.statistics.TableStats)6 ArrayList (java.util.ArrayList)5 WindowDefinition (io.crate.analyze.WindowDefinition)4 AnalyzedRelation (io.crate.analyze.relations.AnalyzedRelation)3 Function (io.crate.expression.symbol.Function)3 FunctionImplementation (io.crate.metadata.FunctionImplementation)3 List (java.util.List)3 RandomizedTest (com.carrotsearch.randomizedtesting.RandomizedTest)2 OrderBy (io.crate.analyze.OrderBy)2 Projection (io.crate.execution.dsl.projection.Projection)2 WindowAggProjection (io.crate.execution.dsl.projection.WindowAggProjection)2 ScopedSymbol (io.crate.expression.symbol.ScopedSymbol)2 SelectSymbol (io.crate.expression.symbol.SelectSymbol)2