Search in sources :

Example 1 with PartitioningScheme

use of io.prestosql.sql.planner.PartitioningScheme in project hetu-core by openlookeng.

the class TestUtil method createExchangePlanFragment.

private static PlanFragment createExchangePlanFragment(RowExpression expr) {
    Symbol testSymbol = new Symbol("a");
    Map<Symbol, ColumnHandle> scanAssignments = ImmutableMap.<Symbol, ColumnHandle>builder().put(testSymbol, new TestingMetadata.TestingColumnHandle("a")).build();
    Map<Symbol, ColumnHandle> assignments = Maps.filterKeys(scanAssignments, Predicates.in(ImmutableList.of(testSymbol)));
    TableScanNode tableScanNode = new TableScanNode(new PlanNodeId(UUID.randomUUID().toString()), makeTableHandle(TupleDomain.none()), ImmutableList.copyOf(assignments.keySet()), assignments, TupleDomain.none(), Optional.empty(), ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), 0, false);
    PlanBuilder planBuilder = new PlanBuilder(new PlanNodeIdAllocator(), dummyMetadata());
    FilterNode filterNode = planBuilder.filter(expr, tableScanNode);
    PlanNode planNode = new LimitNode(new PlanNodeId("limit"), filterNode, 1, false);
    ImmutableMap.Builder<Symbol, Type> types = ImmutableMap.builder();
    for (Symbol symbol : planNode.getOutputSymbols()) {
        types.put(symbol, VARCHAR);
    }
    return new PlanFragment(new PlanFragmentId("limit_fragment_id"), planNode, types.build(), SOURCE_DISTRIBUTION, ImmutableList.of(planNode.getId()), new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), planNode.getOutputSymbols()), ungroupedExecution(), StatsAndCosts.empty(), Optional.empty(), Optional.empty(), Optional.empty());
}
Also used : ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Symbol(io.prestosql.spi.plan.Symbol) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) FilterNode(io.prestosql.spi.plan.FilterNode) PlanBuilder(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder) ImmutableMap(com.google.common.collect.ImmutableMap) PlanFragment(io.prestosql.sql.planner.PlanFragment) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Type(io.prestosql.spi.type.Type) PlanNode(io.prestosql.spi.plan.PlanNode) TableScanNode(io.prestosql.spi.plan.TableScanNode) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) LimitNode(io.prestosql.spi.plan.LimitNode) PlanFragmentId(io.prestosql.sql.planner.plan.PlanFragmentId) UUID(java.util.UUID)

Example 2 with PartitioningScheme

use of io.prestosql.sql.planner.PartitioningScheme in project hetu-core by openlookeng.

the class PlanPrinter method formatFragment.

private static String formatFragment(Function<TableScanNode, TableInfo> tableInfoSupplier, ValuePrinter valuePrinter, PlanFragment fragment, Optional<StageInfo> stageInfo, Optional<Map<PlanNodeId, PlanNodeStats>> planNodeStats, boolean verbose, List<PlanFragment> allFragments, Metadata metadata) {
    StringBuilder builder = new StringBuilder();
    builder.append(format("Fragment %s [%s]\n", fragment.getId(), fragment.getPartitioning()));
    if (stageInfo.isPresent()) {
        StageStats stageStats = stageInfo.get().getStageStats();
        double avgPositionsPerTask = stageInfo.get().getTasks().stream().mapToLong(task -> task.getStats().getProcessedInputPositions()).average().orElse(Double.NaN);
        double squaredDifferences = stageInfo.get().getTasks().stream().mapToDouble(task -> Math.pow(task.getStats().getProcessedInputPositions() - avgPositionsPerTask, 2)).sum();
        double sdAmongTasks = Math.sqrt(squaredDifferences / stageInfo.get().getTasks().size());
        builder.append(indentString(1)).append(format("CPU: %s, Scheduled: %s, Input: %s (%s); per task: avg.: %s std.dev.: %s, Output: %s (%s)\n", stageStats.getTotalCpuTime().convertToMostSuccinctTimeUnit(), stageStats.getTotalScheduledTime().convertToMostSuccinctTimeUnit(), formatPositions(stageStats.getProcessedInputPositions()), stageStats.getProcessedInputDataSize(), formatDouble(avgPositionsPerTask), formatDouble(sdAmongTasks), formatPositions(stageStats.getOutputPositions()), stageStats.getOutputDataSize()));
    }
    PartitioningScheme partitioningScheme = fragment.getPartitioningScheme();
    builder.append(indentString(1)).append(format("Output layout: [%s]\n", Joiner.on(", ").join(partitioningScheme.getOutputLayout())));
    boolean replicateNullsAndAny = partitioningScheme.isReplicateNullsAndAny();
    List<String> arguments = partitioningScheme.getPartitioning().getArguments().stream().map(argument -> {
        if (argument.isConstant()) {
            NullableValue constant = argument.getConstant();
            String printableValue = valuePrinter.castToVarchar(constant.getType(), constant.getValue());
            return constant.getType().getDisplayName() + "(" + printableValue + ")";
        }
        return argument.getColumn().toString();
    }).collect(toImmutableList());
    builder.append(indentString(1));
    if (replicateNullsAndAny) {
        builder.append(format("Output partitioning: %s (replicate nulls and any) [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
    } else {
        builder.append(format("Output partitioning: %s [%s]%s\n", partitioningScheme.getPartitioning().getHandle(), Joiner.on(", ").join(arguments), formatHash(partitioningScheme.getHashColumn())));
    }
    builder.append(indentString(1)).append(format("Stage Execution Strategy: %s\n", fragment.getStageExecutionDescriptor().getStageExecutionStrategy()));
    TypeProvider typeProvider = TypeProvider.copyOf(allFragments.stream().flatMap(f -> f.getSymbols().entrySet().stream()).distinct().collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)));
    builder.append(new PlanPrinter(fragment.getRoot(), typeProvider, Optional.of(fragment.getStageExecutionDescriptor()), tableInfoSupplier, valuePrinter, fragment.getStatsAndCosts(), planNodeStats, metadata).toText(verbose, 1)).append("\n");
    return builder.toString();
}
Also used : TableDeleteNode(io.prestosql.sql.planner.plan.TableDeleteNode) SortNode(io.prestosql.sql.planner.plan.SortNode) SubPlan(io.prestosql.sql.planner.SubPlan) LogicalRowExpressions(io.prestosql.expressions.LogicalRowExpressions) REUSE_STRATEGY_PRODUCER(io.prestosql.spi.operator.ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_PRODUCER) TypeProvider(io.prestosql.sql.planner.TypeProvider) NullableValue(io.prestosql.spi.predicate.NullableValue) PlanFragmentId(io.prestosql.sql.planner.plan.PlanFragmentId) CTEScanNode(io.prestosql.spi.plan.CTEScanNode) AggregationNode(io.prestosql.spi.plan.AggregationNode) TableUpdateNode(io.prestosql.sql.planner.plan.TableUpdateNode) Map(java.util.Map) OutputNode(io.prestosql.sql.planner.plan.OutputNode) Partitioning(io.prestosql.sql.planner.Partitioning) TopNRankingNumberNode(io.prestosql.sql.planner.plan.TopNRankingNumberNode) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) TypedSymbol(io.prestosql.sql.planner.planprinter.NodeRepresentation.TypedSymbol) RowExpressionDeterminismEvaluator(io.prestosql.sql.relational.RowExpressionDeterminismEvaluator) CreateIndexNode(io.prestosql.sql.planner.plan.CreateIndexNode) SortExpressionContext(io.prestosql.sql.planner.SortExpressionContext) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) TableScanNode(io.prestosql.spi.plan.TableScanNode) Set(java.util.Set) DynamicFilters(io.prestosql.sql.DynamicFilters) IndexSourceNode(io.prestosql.sql.planner.plan.IndexSourceNode) PlanNode(io.prestosql.spi.plan.PlanNode) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) ProjectNode(io.prestosql.spi.plan.ProjectNode) Metadata(io.prestosql.metadata.Metadata) Collectors.joining(java.util.stream.Collectors.joining) SymbolUtils.toSymbolReference(io.prestosql.sql.planner.SymbolUtils.toSymbolReference) SpatialJoinNode(io.prestosql.sql.planner.plan.SpatialJoinNode) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) Stream(java.util.stream.Stream) Domain(io.prestosql.spi.predicate.Domain) StatisticAggregations(io.prestosql.sql.planner.plan.StatisticAggregations) ColumnStatisticMetadata(io.prestosql.spi.statistics.ColumnStatisticMetadata) StatisticsWriterNode(io.prestosql.sql.planner.plan.StatisticsWriterNode) VacuumTableNode(io.prestosql.sql.planner.plan.VacuumTableNode) DistinctLimitNode(io.prestosql.sql.planner.plan.DistinctLimitNode) Joiner(com.google.common.base.Joiner) GroupIdNode(io.prestosql.spi.plan.GroupIdNode) Iterables(com.google.common.collect.Iterables) IntersectNode(io.prestosql.spi.plan.IntersectNode) Marker(io.prestosql.spi.predicate.Marker) DynamicFilters.extractDynamicFilters(io.prestosql.sql.DynamicFilters.extractDynamicFilters) StageExecutionDescriptor(io.prestosql.operator.StageExecutionDescriptor) AssignUniqueId(io.prestosql.sql.planner.plan.AssignUniqueId) UnnestNode(io.prestosql.sql.planner.plan.UnnestNode) ArrayList(java.util.ArrayList) Lists(com.google.common.collect.Lists) Session(io.prestosql.Session) DeleteNode(io.prestosql.sql.planner.plan.DeleteNode) Functions(com.google.common.base.Functions) PlanNodeStatsEstimate(io.prestosql.cost.PlanNodeStatsEstimate) Assignments(io.prestosql.spi.plan.Assignments) ComparisonExpression(io.prestosql.sql.tree.ComparisonExpression) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) TextRenderer.formatPositions(io.prestosql.sql.planner.planprinter.TextRenderer.formatPositions) ValuesNode(io.prestosql.spi.plan.ValuesNode) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) SampleNode(io.prestosql.sql.planner.plan.SampleNode) WindowNode(io.prestosql.spi.plan.WindowNode) LimitNode(io.prestosql.spi.plan.LimitNode) Expression(io.prestosql.sql.tree.Expression) OffsetNode(io.prestosql.sql.planner.plan.OffsetNode) StatisticAggregationsDescriptor(io.prestosql.sql.planner.plan.StatisticAggregationsDescriptor) Scope(io.prestosql.sql.planner.plan.ExchangeNode.Scope) UpdateIndexNode(io.prestosql.sql.planner.plan.UpdateIndexNode) Duration(io.airlift.units.Duration) PlanNodeStatsSummarizer.aggregateStageStats(io.prestosql.sql.planner.planprinter.PlanNodeStatsSummarizer.aggregateStageStats) TableStatisticType(io.prestosql.spi.statistics.TableStatisticType) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) TableFinishNode(io.prestosql.sql.planner.plan.TableFinishNode) PlanCostEstimate(io.prestosql.cost.PlanCostEstimate) ExchangeNode(io.prestosql.sql.planner.plan.ExchangeNode) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) FilterNode(io.prestosql.spi.plan.FilterNode) TextRenderer.indentString(io.prestosql.sql.planner.planprinter.TextRenderer.indentString) TextRenderer.formatDouble(io.prestosql.sql.planner.planprinter.TextRenderer.formatDouble) Type(io.prestosql.spi.type.Type) ApplyNode(io.prestosql.sql.planner.plan.ApplyNode) ImmutableSet(com.google.common.collect.ImmutableSet) JoinNodeUtils(io.prestosql.sql.planner.optimizations.JoinNodeUtils) Collection(java.util.Collection) Streams(com.google.common.collect.Streams) IndexJoinNode(io.prestosql.sql.planner.plan.IndexJoinNode) CubeFinishNode(io.prestosql.sql.planner.plan.CubeFinishNode) Collectors(java.util.stream.Collectors) RowNumberNode(io.prestosql.sql.planner.plan.RowNumberNode) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) FunctionResolution(io.prestosql.sql.relational.FunctionResolution) EnforceSingleRowNode(io.prestosql.sql.planner.plan.EnforceSingleRowNode) TopNNode(io.prestosql.spi.plan.TopNNode) StageInfo.getAllStages(io.prestosql.execution.StageInfo.getAllStages) StageInfo(io.prestosql.execution.StageInfo) Entry(java.util.Map.Entry) UnionNode(io.prestosql.spi.plan.UnionNode) Optional(java.util.Optional) ExceptNode(io.prestosql.spi.plan.ExceptNode) Arrays.stream(java.util.Arrays.stream) StageStats(io.prestosql.execution.StageStats) InternalPlanVisitor(io.prestosql.sql.planner.plan.InternalPlanVisitor) SINGLE_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION) LateralJoinNode(io.prestosql.sql.planner.plan.LateralJoinNode) StatsAndCosts(io.prestosql.cost.StatsAndCosts) RemoteSourceNode(io.prestosql.sql.planner.plan.RemoteSourceNode) TableHandle(io.prestosql.spi.metadata.TableHandle) Function(java.util.function.Function) TRUE_LITERAL(io.prestosql.sql.tree.BooleanLiteral.TRUE_LITERAL) SemiJoinNode(io.prestosql.sql.planner.plan.SemiJoinNode) ImmutableList(com.google.common.collect.ImmutableList) OrderingScheme(io.prestosql.spi.plan.OrderingScheme) GraphvizPrinter(io.prestosql.util.GraphvizPrinter) Verify.verify(com.google.common.base.Verify.verify) Range(io.prestosql.spi.predicate.Range) Objects.requireNonNull(java.util.Objects.requireNonNull) LinkedList(java.util.LinkedList) MarkDistinctNode(io.prestosql.spi.plan.MarkDistinctNode) JoinNode(io.prestosql.spi.plan.JoinNode) Symbol(io.prestosql.spi.plan.Symbol) PlanFragment(io.prestosql.sql.planner.PlanFragment) TableWriterNode(io.prestosql.sql.planner.plan.TableWriterNode) StageExecutionDescriptor.ungroupedExecution(io.prestosql.operator.StageExecutionDescriptor.ungroupedExecution) REUSE_STRATEGY_CONSUMER(io.prestosql.spi.operator.ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_CONSUMER) TableInfo(io.prestosql.execution.TableInfo) CaseFormat(com.google.common.base.CaseFormat) GroupReference(io.prestosql.spi.plan.GroupReference) TupleDomain(io.prestosql.spi.predicate.TupleDomain) UpdateNode(io.prestosql.sql.planner.plan.UpdateNode) UPPER_UNDERSCORE(com.google.common.base.CaseFormat.UPPER_UNDERSCORE) Collectors.toList(java.util.stream.Collectors.toList) Aggregation(io.prestosql.spi.plan.AggregationNode.Aggregation) RowExpression(io.prestosql.spi.relation.RowExpression) SortExpressionExtractor(io.prestosql.sql.planner.SortExpressionExtractor) ExplainAnalyzeNode(io.prestosql.sql.planner.plan.ExplainAnalyzeNode) Entry(java.util.Map.Entry) PlanNodeStatsSummarizer.aggregateStageStats(io.prestosql.sql.planner.planprinter.PlanNodeStatsSummarizer.aggregateStageStats) StageStats(io.prestosql.execution.StageStats) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) NullableValue(io.prestosql.spi.predicate.NullableValue) TypeProvider(io.prestosql.sql.planner.TypeProvider) TextRenderer.indentString(io.prestosql.sql.planner.planprinter.TextRenderer.indentString)

Example 3 with PartitioningScheme

use of io.prestosql.sql.planner.PartitioningScheme in project hetu-core by openlookeng.

the class TestSqlStageExecution method createExchangePlanFragment.

private static PlanFragment createExchangePlanFragment() {
    PlanNode planNode = new RemoteSourceNode(new PlanNodeId("exchange"), ImmutableList.of(new PlanFragmentId("source")), ImmutableList.of(new Symbol("column")), Optional.empty(), REPARTITION);
    ImmutableMap.Builder<Symbol, Type> types = ImmutableMap.builder();
    for (Symbol symbol : planNode.getOutputSymbols()) {
        types.put(symbol, VARCHAR);
    }
    return new PlanFragment(new PlanFragmentId("exchange_fragment_id"), planNode, types.build(), SOURCE_DISTRIBUTION, ImmutableList.of(planNode.getId()), new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), planNode.getOutputSymbols()), ungroupedExecution(), StatsAndCosts.empty(), Optional.empty(), Optional.empty(), Optional.empty());
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) RemoteSourceNode(io.prestosql.sql.planner.plan.RemoteSourceNode) Type(io.prestosql.spi.type.Type) PlanNode(io.prestosql.spi.plan.PlanNode) Symbol(io.prestosql.spi.plan.Symbol) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) PlanFragmentId(io.prestosql.sql.planner.plan.PlanFragmentId) ImmutableMap(com.google.common.collect.ImmutableMap) PlanFragment(io.prestosql.sql.planner.PlanFragment)

Example 4 with PartitioningScheme

use of io.prestosql.sql.planner.PartitioningScheme in project hetu-core by openlookeng.

the class PushProjectionThroughExchange method apply.

@Override
public Result apply(ProjectNode project, Captures captures, Context context) {
    ExchangeNode exchange = captures.get(CHILD);
    Set<Symbol> partitioningColumns = exchange.getPartitioningScheme().getPartitioning().getColumns();
    ImmutableList.Builder<PlanNode> newSourceBuilder = ImmutableList.builder();
    ImmutableList.Builder<List<Symbol>> inputsBuilder = ImmutableList.builder();
    for (int i = 0; i < exchange.getSources().size(); i++) {
        Map<Symbol, VariableReferenceExpression> outputToInputMap = extractExchangeOutputToInput(exchange, i, context.getSymbolAllocator().getTypes());
        Assignments.Builder projections = Assignments.builder();
        ImmutableList.Builder<Symbol> inputs = ImmutableList.builder();
        // Need to retain the partition keys for the exchange
        partitioningColumns.stream().map(outputToInputMap::get).forEach(nameReference -> {
            Symbol symbol = new Symbol(nameReference.getName());
            projections.put(symbol, nameReference);
            inputs.add(symbol);
        });
        if (exchange.getPartitioningScheme().getHashColumn().isPresent()) {
            // Need to retain the hash symbol for the exchange
            projections.put(exchange.getPartitioningScheme().getHashColumn().get(), toVariableReference(exchange.getPartitioningScheme().getHashColumn().get(), context.getSymbolAllocator().getTypes()));
            inputs.add(exchange.getPartitioningScheme().getHashColumn().get());
        }
        if (exchange.getOrderingScheme().isPresent()) {
            // need to retain ordering columns for the exchange
            exchange.getOrderingScheme().get().getOrderBy().stream().filter(symbol -> !partitioningColumns.contains(symbol)).map(outputToInputMap::get).forEach(nameReference -> {
                Symbol symbol = new Symbol(nameReference.getName());
                projections.put(symbol, nameReference);
                inputs.add(symbol);
            });
        }
        for (Map.Entry<Symbol, RowExpression> projection : project.getAssignments().entrySet()) {
            checkArgument(!isExpression(projection.getValue()), "Cannot contain OriginalExpression after AddExchange");
            Map<VariableReferenceExpression, VariableReferenceExpression> variableOutputToInputMap = new LinkedHashMap<>();
            outputToInputMap.forEach(((symbol, variable) -> variableOutputToInputMap.put(new VariableReferenceExpression(symbol.getName(), context.getSymbolAllocator().getTypes().get(symbol)), variable)));
            RowExpression translatedExpression = RowExpressionVariableInliner.inlineVariables(variableOutputToInputMap, projection.getValue());
            Symbol symbol = context.getSymbolAllocator().newSymbol(translatedExpression);
            projections.put(symbol, translatedExpression);
            inputs.add(symbol);
        }
        newSourceBuilder.add(new ProjectNode(context.getIdAllocator().getNextId(), exchange.getSources().get(i), projections.build()));
        inputsBuilder.add(inputs.build());
    }
    // Construct the output symbols in the same order as the sources
    ImmutableList.Builder<Symbol> outputBuilder = ImmutableList.builder();
    partitioningColumns.forEach(outputBuilder::add);
    exchange.getPartitioningScheme().getHashColumn().ifPresent(outputBuilder::add);
    if (exchange.getOrderingScheme().isPresent()) {
        exchange.getOrderingScheme().get().getOrderBy().stream().filter(symbol -> !partitioningColumns.contains(symbol)).forEach(outputBuilder::add);
    }
    for (Map.Entry<Symbol, RowExpression> projection : project.getAssignments().entrySet()) {
        outputBuilder.add(projection.getKey());
    }
    // outputBuilder contains all partition and hash symbols so simply swap the output layout
    PartitioningScheme partitioningScheme = new PartitioningScheme(exchange.getPartitioningScheme().getPartitioning(), outputBuilder.build(), exchange.getPartitioningScheme().getHashColumn(), exchange.getPartitioningScheme().isReplicateNullsAndAny(), exchange.getPartitioningScheme().getBucketToPartition());
    PlanNode result = new ExchangeNode(exchange.getId(), exchange.getType(), exchange.getScope(), partitioningScheme, newSourceBuilder.build(), inputsBuilder.build(), exchange.getOrderingScheme(), AggregationNode.AggregationType.HASH);
    // we need to strip unnecessary symbols (hash, partitioning columns).
    return Result.ofPlanNode(restrictOutputs(context.getIdAllocator(), result, ImmutableSet.copyOf(project.getOutputSymbols()), true, context.getSymbolAllocator().getTypes()).orElse(result));
}
Also used : Patterns.source(io.prestosql.sql.planner.plan.Patterns.source) TypeProvider(io.prestosql.sql.planner.TypeProvider) HashMap(java.util.HashMap) Pattern(io.prestosql.matching.Pattern) RowExpressionVariableInliner(io.prestosql.sql.planner.RowExpressionVariableInliner) AggregationNode(io.prestosql.spi.plan.AggregationNode) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) LinkedHashMap(java.util.LinkedHashMap) OriginalExpressionUtils.isExpression(io.prestosql.sql.relational.OriginalExpressionUtils.isExpression) ExchangeNode(io.prestosql.sql.planner.plan.ExchangeNode) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Capture.newCapture(io.prestosql.matching.Capture.newCapture) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) OriginalExpressionUtils.castToExpression(io.prestosql.sql.relational.OriginalExpressionUtils.castToExpression) Util.restrictOutputs(io.prestosql.sql.planner.iterative.rule.Util.restrictOutputs) Patterns.project(io.prestosql.sql.planner.plan.Patterns.project) Symbol(io.prestosql.spi.plan.Symbol) ImmutableSet(com.google.common.collect.ImmutableSet) Assignments(io.prestosql.spi.plan.Assignments) Rule(io.prestosql.sql.planner.iterative.Rule) Set(java.util.Set) PlanNode(io.prestosql.spi.plan.PlanNode) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) ProjectNode(io.prestosql.spi.plan.ProjectNode) VariableReferenceSymbolConverter.toVariableReference(io.prestosql.sql.planner.VariableReferenceSymbolConverter.toVariableReference) Patterns.exchange(io.prestosql.sql.planner.plan.Patterns.exchange) Captures(io.prestosql.matching.Captures) List(java.util.List) SymbolReference(io.prestosql.sql.tree.SymbolReference) Capture(io.prestosql.matching.Capture) InputReferenceExpression(io.prestosql.spi.relation.InputReferenceExpression) RowExpression(io.prestosql.spi.relation.RowExpression) ExchangeNode(io.prestosql.sql.planner.plan.ExchangeNode) Symbol(io.prestosql.spi.plan.Symbol) ImmutableList(com.google.common.collect.ImmutableList) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) Assignments(io.prestosql.spi.plan.Assignments) RowExpression(io.prestosql.spi.relation.RowExpression) LinkedHashMap(java.util.LinkedHashMap) PlanNode(io.prestosql.spi.plan.PlanNode) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) ImmutableList(com.google.common.collect.ImmutableList) List(java.util.List) ProjectNode(io.prestosql.spi.plan.ProjectNode) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map)

Example 5 with PartitioningScheme

use of io.prestosql.sql.planner.PartitioningScheme in project hetu-core by openlookeng.

the class PushPartialAggregationThroughExchange method pushPartial.

private PlanNode pushPartial(AggregationNode aggregation, ExchangeNode exchange, Context context) {
    List<PlanNode> partials = new ArrayList<>();
    for (int i = 0; i < exchange.getSources().size(); i++) {
        PlanNode source = exchange.getSources().get(i);
        SymbolMapper.Builder mappingsBuilder = SymbolMapper.builder();
        for (int outputIndex = 0; outputIndex < exchange.getOutputSymbols().size(); outputIndex++) {
            Symbol output = exchange.getOutputSymbols().get(outputIndex);
            Symbol input = exchange.getInputs().get(i).get(outputIndex);
            if (!output.equals(input)) {
                mappingsBuilder.put(output, input);
            }
        }
        SymbolMapper symbolMapper = mappingsBuilder.build();
        if (symbolMapper.getTypes() == null) {
            symbolMapper.setTypes(context.getSymbolAllocator().getTypes());
        }
        AggregationNode mappedPartial = symbolMapper.map(aggregation, source, context.getIdAllocator());
        Assignments.Builder assignments = Assignments.builder();
        for (Symbol output : aggregation.getOutputSymbols()) {
            Symbol input = symbolMapper.map(output);
            assignments.put(output, VariableReferenceSymbolConverter.toVariableReference(input, context.getSymbolAllocator().getTypes()));
        }
        partials.add(new ProjectNode(context.getIdAllocator().getNextId(), mappedPartial, assignments.build()));
    }
    for (PlanNode node : partials) {
        verify(aggregation.getOutputSymbols().equals(node.getOutputSymbols()));
    }
    // Since this exchange source is now guaranteed to have the same symbols as the inputs to the the partial
    // aggregation, we don't need to rewrite symbols in the partitioning function
    PartitioningScheme partitioning = new PartitioningScheme(exchange.getPartitioningScheme().getPartitioning(), aggregation.getOutputSymbols(), exchange.getPartitioningScheme().getHashColumn(), exchange.getPartitioningScheme().isReplicateNullsAndAny(), exchange.getPartitioningScheme().getBucketToPartition());
    return new ExchangeNode(context.getIdAllocator().getNextId(), exchange.getType(), exchange.getScope(), partitioning, partials, ImmutableList.copyOf(Collections.nCopies(partials.size(), aggregation.getOutputSymbols())), Optional.empty(), aggregation.getAggregationType());
}
Also used : SymbolMapper(io.prestosql.sql.planner.optimizations.SymbolMapper) ExchangeNode(io.prestosql.sql.planner.plan.ExchangeNode) Symbol(io.prestosql.spi.plan.Symbol) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) ArrayList(java.util.ArrayList) Assignments(io.prestosql.spi.plan.Assignments) AggregationNode(io.prestosql.spi.plan.AggregationNode) PlanNode(io.prestosql.spi.plan.PlanNode) ProjectNode(io.prestosql.spi.plan.ProjectNode)

Aggregations

PartitioningScheme (io.prestosql.sql.planner.PartitioningScheme)11 Symbol (io.prestosql.spi.plan.Symbol)8 PlanNode (io.prestosql.spi.plan.PlanNode)7 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)6 PlanFragment (io.prestosql.sql.planner.PlanFragment)6 PlanFragmentId (io.prestosql.sql.planner.plan.PlanFragmentId)6 AggregationNode (io.prestosql.spi.plan.AggregationNode)4 ExchangeNode (io.prestosql.sql.planner.plan.ExchangeNode)4 Assignments (io.prestosql.spi.plan.Assignments)3 ProjectNode (io.prestosql.spi.plan.ProjectNode)3 TableScanNode (io.prestosql.spi.plan.TableScanNode)3 Type (io.prestosql.spi.type.Type)3 RemoteSourceNode (io.prestosql.sql.planner.plan.RemoteSourceNode)3 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)2 ImmutableList (com.google.common.collect.ImmutableList)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 ImmutableSet (com.google.common.collect.ImmutableSet)2 Session (io.prestosql.Session)2 ColumnHandle (io.prestosql.spi.connector.ColumnHandle)2 FilterNode (io.prestosql.spi.plan.FilterNode)2