Search in sources :

Example 36 with PlanNode

use of com.facebook.presto.spi.plan.PlanNode in project presto by prestodb.

the class TestPinotPlanOptimizer method testDistinctCountInSubQueryPushdown.

private void testDistinctCountInSubQueryPushdown(String distinctCountFunctionName, PinotConfig pinotConfig) {
    PlanBuilder planBuilder = createPlanBuilder(new SessionHolder(pinotConfig));
    Map<VariableReferenceExpression, PinotColumnHandle> leftColumnHandleMap = ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "regionid", regionId.getDataType()), regionId);
    PlanNode leftJustScan = tableScan(planBuilder, pinotTable, leftColumnHandleMap);
    PlanNode leftMarkDistinct = markDistinct(planBuilder, variable("regionid$distinct"), ImmutableList.of(variable("regionid")), leftJustScan);
    PlanNode leftAggregation = planBuilder.aggregation(aggBuilder -> aggBuilder.source(leftMarkDistinct).addAggregation(planBuilder.variable("count(regionid)"), getRowExpression("count(regionid)", defaultSessionHolder), Optional.empty(), Optional.empty(), false, Optional.of(variable("regionid$distinct"))).globalGrouping());
    PlanNode optimized = getOptimizedPlan(pinotConfig, planBuilder, leftAggregation);
    assertPlanMatch(optimized, PinotTableScanMatcher.match(pinotTable, Optional.of(String.format("SELECT %s\\(regionId\\) FROM hybrid", distinctCountFunctionName)), Optional.of(false), leftAggregation.getOutputVariables(), useSqlSyntax()), typeProvider);
    Map<VariableReferenceExpression, PinotColumnHandle> rightColumnHandleMap = ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "regionid_33", regionId.getDataType()), regionId);
    PlanNode rightJustScan = tableScan(planBuilder, pinotTable, rightColumnHandleMap);
    PlanNode rightMarkDistinct = markDistinct(planBuilder, variable("regionid$distinct_62"), ImmutableList.of(variable("regionid")), rightJustScan);
    PlanNode rightAggregation = planBuilder.aggregation(aggBuilder -> aggBuilder.source(rightMarkDistinct).addAggregation(planBuilder.variable("count(regionid_33)"), getRowExpression("count(regionid_33)", defaultSessionHolder), Optional.empty(), Optional.empty(), false, Optional.of(variable("regionid$distinct_62"))).globalGrouping());
    optimized = getOptimizedPlan(pinotConfig, planBuilder, rightAggregation);
    assertPlanMatch(optimized, PinotTableScanMatcher.match(pinotTable, Optional.of(String.format("SELECT %s\\(regionId\\) FROM hybrid", distinctCountFunctionName)), Optional.of(false), rightAggregation.getOutputVariables(), useSqlSyntax()), typeProvider);
}
Also used : PinotColumnHandle(com.facebook.presto.pinot.PinotColumnHandle) PlanNode(com.facebook.presto.spi.plan.PlanNode) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) PlanBuilder(com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder)

Example 37 with PlanNode

use of com.facebook.presto.spi.plan.PlanNode in project presto by prestodb.

the class RuntimeReorderJoinSides method derivePropertiesRecursively.

private StreamProperties derivePropertiesRecursively(PlanNode node, Metadata metadata, SqlParser parser, Context context) {
    PlanNode actual = context.getLookup().resolve(node);
    List<StreamProperties> inputProperties = actual.getSources().stream().map(source -> derivePropertiesRecursively(source, metadata, parser, context)).collect(toImmutableList());
    return StreamPropertyDerivations.deriveProperties(actual, inputProperties, metadata, context.getSession(), context.getVariableAllocator().getTypes(), parser);
}
Also used : ExchangeNode.systemPartitionedExchange(com.facebook.presto.sql.planner.plan.ExchangeNode.systemPartitionedExchange) Logger(com.facebook.airlift.log.Logger) StreamPropertyDerivations(com.facebook.presto.sql.planner.optimizations.StreamPropertyDerivations) Captures(com.facebook.presto.matching.Captures) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) StreamPreferredProperties.exactlyPartitionedOn(com.facebook.presto.sql.planner.optimizations.StreamPreferredProperties.exactlyPartitionedOn) Patterns.join(com.facebook.presto.sql.planner.plan.Patterns.join) Pattern(com.facebook.presto.matching.Pattern) StreamPreferredProperties.fixedParallelism(com.facebook.presto.sql.planner.optimizations.StreamPreferredProperties.fixedParallelism) LOCAL(com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.LOCAL) Objects.requireNonNull(java.util.Objects.requireNonNull) SystemSessionProperties.isJoinSpillingEnabled(com.facebook.presto.SystemSessionProperties.isJoinSpillingEnabled) StreamPreferredProperties.singleStream(com.facebook.presto.sql.planner.optimizations.StreamPreferredProperties.singleStream) SystemSessionProperties.getTaskConcurrency(com.facebook.presto.SystemSessionProperties.getTaskConcurrency) PlanNodeSearcher.searchFrom(com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom) ExchangeNode.gatheringExchange(com.facebook.presto.sql.planner.plan.ExchangeNode.gatheringExchange) JoinNode(com.facebook.presto.sql.planner.plan.JoinNode) Rule(com.facebook.presto.sql.planner.iterative.Rule) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) StatsProvider(com.facebook.presto.cost.StatsProvider) String.format(java.lang.String.format) SqlParser(com.facebook.presto.sql.parser.SqlParser) StreamPreferredProperties.defaultParallelism(com.facebook.presto.sql.planner.optimizations.StreamPreferredProperties.defaultParallelism) RIGHT(com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT) SystemSessionProperties.isSpillEnabled(com.facebook.presto.SystemSessionProperties.isSpillEnabled) PlanNode(com.facebook.presto.spi.plan.PlanNode) List(java.util.List) REPLICATED(com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.REPLICATED) LEFT(com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT) TableScanNode(com.facebook.presto.spi.plan.TableScanNode) StreamProperties(com.facebook.presto.sql.planner.optimizations.StreamPropertyDerivations.StreamProperties) StreamPreferredProperties(com.facebook.presto.sql.planner.optimizations.StreamPreferredProperties) Optional(java.util.Optional) PARTITIONED(com.facebook.presto.sql.planner.plan.JoinNode.DistributionType.PARTITIONED) Metadata(com.facebook.presto.metadata.Metadata) ExchangeNode(com.facebook.presto.sql.planner.plan.ExchangeNode) PlanNode(com.facebook.presto.spi.plan.PlanNode) StreamProperties(com.facebook.presto.sql.planner.optimizations.StreamPropertyDerivations.StreamProperties)

Example 38 with PlanNode

use of com.facebook.presto.spi.plan.PlanNode in project presto by prestodb.

the class PushPartialAggregationThroughExchange method pushPartial.

private PlanNode pushPartial(AggregationNode aggregation, ExchangeNode exchange, Context context) {
    List<PlanNode> partials = new ArrayList<>();
    for (int i = 0; i < exchange.getSources().size(); i++) {
        PlanNode source = exchange.getSources().get(i);
        SymbolMapper.Builder mappingsBuilder = SymbolMapper.builder();
        for (int outputIndex = 0; outputIndex < exchange.getOutputVariables().size(); outputIndex++) {
            VariableReferenceExpression output = exchange.getOutputVariables().get(outputIndex);
            VariableReferenceExpression input = exchange.getInputs().get(i).get(outputIndex);
            if (!output.equals(input)) {
                mappingsBuilder.put(output, input);
            }
        }
        SymbolMapper symbolMapper = mappingsBuilder.build();
        AggregationNode mappedPartial = symbolMapper.map(aggregation, source, context.getIdAllocator());
        Assignments.Builder assignments = Assignments.builder();
        for (VariableReferenceExpression output : aggregation.getOutputVariables()) {
            VariableReferenceExpression input = symbolMapper.map(output);
            assignments.put(output, input);
        }
        partials.add(new ProjectNode(exchange.getSourceLocation(), context.getIdAllocator().getNextId(), mappedPartial, assignments.build(), LOCAL));
    }
    for (PlanNode node : partials) {
        verify(aggregation.getOutputVariables().equals(node.getOutputVariables()));
    }
    // Since this exchange source is now guaranteed to have the same symbols as the inputs to the the partial
    // aggregation, we don't need to rewrite symbols in the partitioning function
    List<VariableReferenceExpression> aggregationOutputs = aggregation.getOutputVariables();
    PartitioningScheme partitioning = new PartitioningScheme(exchange.getPartitioningScheme().getPartitioning(), aggregationOutputs, exchange.getPartitioningScheme().getHashColumn(), exchange.getPartitioningScheme().isReplicateNullsAndAny(), exchange.getPartitioningScheme().getBucketToPartition());
    return new ExchangeNode(aggregation.getSourceLocation(), context.getIdAllocator().getNextId(), exchange.getType(), exchange.getScope(), partitioning, partials, ImmutableList.copyOf(Collections.nCopies(partials.size(), aggregationOutputs)), exchange.isEnsureSourceOrdering(), Optional.empty());
}
Also used : SymbolMapper(com.facebook.presto.sql.planner.optimizations.SymbolMapper) ExchangeNode(com.facebook.presto.sql.planner.plan.ExchangeNode) PartitioningScheme(com.facebook.presto.sql.planner.PartitioningScheme) ArrayList(java.util.ArrayList) Assignments(com.facebook.presto.spi.plan.Assignments) AggregationNode(com.facebook.presto.spi.plan.AggregationNode) PlanNode(com.facebook.presto.spi.plan.PlanNode) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) ProjectNode(com.facebook.presto.spi.plan.ProjectNode)

Example 39 with PlanNode

use of com.facebook.presto.spi.plan.PlanNode in project presto by prestodb.

the class PushProjectionThroughExchange method apply.

@Override
public Result apply(ProjectNode project, Captures captures, Context context) {
    ExchangeNode exchange = captures.get(CHILD);
    Set<VariableReferenceExpression> partitioningColumns = exchange.getPartitioningScheme().getPartitioning().getVariableReferences();
    ImmutableList.Builder<PlanNode> newSourceBuilder = ImmutableList.builder();
    ImmutableList.Builder<List<VariableReferenceExpression>> inputsBuilder = ImmutableList.builder();
    for (int i = 0; i < exchange.getSources().size(); i++) {
        Map<VariableReferenceExpression, VariableReferenceExpression> outputToInputMap = extractExchangeOutputToInput(exchange, i);
        Assignments.Builder projections = Assignments.builder();
        ImmutableList.Builder<VariableReferenceExpression> inputs = ImmutableList.builder();
        // Need to retain the partition keys for the exchange
        partitioningColumns.stream().map(outputToInputMap::get).forEach(variable -> {
            projections.put(variable, variable);
            inputs.add(variable);
        });
        if (exchange.getPartitioningScheme().getHashColumn().isPresent()) {
            // Need to retain the hash symbol for the exchange
            VariableReferenceExpression hashVariable = exchange.getPartitioningScheme().getHashColumn().get();
            projections.put(hashVariable, hashVariable);
            inputs.add(hashVariable);
        }
        if (exchange.getOrderingScheme().isPresent()) {
            // need to retain ordering columns for the exchange
            exchange.getOrderingScheme().get().getOrderByVariables().stream().filter(variable -> !partitioningColumns.contains(variable)).map(outputToInputMap::get).forEach(variable -> {
                projections.put(variable, variable);
                inputs.add(variable);
            });
        }
        for (Map.Entry<VariableReferenceExpression, RowExpression> projection : project.getAssignments().entrySet()) {
            RowExpression translatedExpression = RowExpressionVariableInliner.inlineVariables(outputToInputMap, projection.getValue());
            VariableReferenceExpression variable = context.getVariableAllocator().newVariable(translatedExpression);
            projections.put(variable, translatedExpression);
            inputs.add(variable);
        }
        newSourceBuilder.add(new ProjectNode(project.getSourceLocation(), context.getIdAllocator().getNextId(), exchange.getSources().get(i), projections.build(), project.getLocality()));
        inputsBuilder.add(inputs.build());
    }
    // Construct the output symbols in the same order as the sources
    ImmutableList.Builder<VariableReferenceExpression> outputBuilder = ImmutableList.builder();
    partitioningColumns.forEach(outputBuilder::add);
    exchange.getPartitioningScheme().getHashColumn().ifPresent(outputBuilder::add);
    if (exchange.getOrderingScheme().isPresent()) {
        exchange.getOrderingScheme().get().getOrderByVariables().stream().filter(variable -> !partitioningColumns.contains(variable)).forEach(outputBuilder::add);
    }
    for (Map.Entry<VariableReferenceExpression, RowExpression> projection : project.getAssignments().entrySet()) {
        outputBuilder.add(projection.getKey());
    }
    // outputBuilder contains all partition and hash symbols so simply swap the output layout
    PartitioningScheme partitioningScheme = new PartitioningScheme(exchange.getPartitioningScheme().getPartitioning(), outputBuilder.build(), exchange.getPartitioningScheme().getHashColumn(), exchange.getPartitioningScheme().isReplicateNullsAndAny(), exchange.getPartitioningScheme().getBucketToPartition());
    PlanNode result = new ExchangeNode(exchange.getSourceLocation(), exchange.getId(), exchange.getType(), exchange.getScope(), partitioningScheme, newSourceBuilder.build(), inputsBuilder.build(), exchange.isEnsureSourceOrdering(), exchange.getOrderingScheme());
    // we need to strip unnecessary symbols (hash, partitioning columns).
    return Result.ofPlanNode(restrictOutputs(context.getIdAllocator(), result, ImmutableSet.copyOf(project.getOutputVariables()), true).orElse(result));
}
Also used : RowExpression(com.facebook.presto.spi.relation.RowExpression) Patterns.exchange(com.facebook.presto.sql.planner.plan.Patterns.exchange) ImmutableSet(com.google.common.collect.ImmutableSet) RowExpressionVariableInliner(com.facebook.presto.sql.planner.RowExpressionVariableInliner) Rule(com.facebook.presto.sql.planner.iterative.Rule) Captures(com.facebook.presto.matching.Captures) Patterns.project(com.facebook.presto.sql.planner.plan.Patterns.project) Assignments(com.facebook.presto.spi.plan.Assignments) Set(java.util.Set) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) HashMap(java.util.HashMap) Util.restrictOutputs(com.facebook.presto.sql.planner.iterative.rule.Util.restrictOutputs) Pattern(com.facebook.presto.matching.Pattern) Patterns.source(com.facebook.presto.sql.planner.plan.Patterns.source) PlanNode(com.facebook.presto.spi.plan.PlanNode) List(java.util.List) Capture(com.facebook.presto.matching.Capture) ImmutableList(com.google.common.collect.ImmutableList) ProjectNode(com.facebook.presto.spi.plan.ProjectNode) Capture.newCapture(com.facebook.presto.matching.Capture.newCapture) Map(java.util.Map) PartitioningScheme(com.facebook.presto.sql.planner.PartitioningScheme) ExchangeNode(com.facebook.presto.sql.planner.plan.ExchangeNode) ExchangeNode(com.facebook.presto.sql.planner.plan.ExchangeNode) ImmutableList(com.google.common.collect.ImmutableList) PartitioningScheme(com.facebook.presto.sql.planner.PartitioningScheme) Assignments(com.facebook.presto.spi.plan.Assignments) RowExpression(com.facebook.presto.spi.relation.RowExpression) PlanNode(com.facebook.presto.spi.plan.PlanNode) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) ProjectNode(com.facebook.presto.spi.plan.ProjectNode) HashMap(java.util.HashMap) Map(java.util.Map)

Example 40 with PlanNode

use of com.facebook.presto.spi.plan.PlanNode in project presto by prestodb.

the class TransformCorrelatedScalarSubquery method apply.

@Override
public Result apply(LateralJoinNode lateralJoinNode, Captures captures, Context context) {
    PlanNode subquery = context.getLookup().resolve(lateralJoinNode.getSubquery());
    if (!searchFrom(subquery, context.getLookup()).where(EnforceSingleRowNode.class::isInstance).recurseOnlyWhen(ProjectNode.class::isInstance).matches()) {
        return Result.empty();
    }
    PlanNode rewrittenSubquery = searchFrom(subquery, context.getLookup()).where(EnforceSingleRowNode.class::isInstance).recurseOnlyWhen(ProjectNode.class::isInstance).removeFirst();
    if (isAtMostScalar(rewrittenSubquery, context.getLookup())) {
        return Result.ofPlanNode(new LateralJoinNode(lateralJoinNode.getSourceLocation(), context.getIdAllocator().getNextId(), lateralJoinNode.getInput(), rewrittenSubquery, lateralJoinNode.getCorrelation(), lateralJoinNode.getType(), lateralJoinNode.getOriginSubqueryError()));
    }
    VariableReferenceExpression unique = context.getVariableAllocator().newVariable("unique", BIGINT);
    LateralJoinNode rewrittenLateralJoinNode = new LateralJoinNode(lateralJoinNode.getSourceLocation(), context.getIdAllocator().getNextId(), new AssignUniqueId(lateralJoinNode.getSourceLocation(), context.getIdAllocator().getNextId(), lateralJoinNode.getInput(), unique), rewrittenSubquery, lateralJoinNode.getCorrelation(), lateralJoinNode.getType(), lateralJoinNode.getOriginSubqueryError());
    VariableReferenceExpression isDistinct = context.getVariableAllocator().newVariable("is_distinct", BooleanType.BOOLEAN);
    MarkDistinctNode markDistinctNode = new MarkDistinctNode(rewrittenLateralJoinNode.getSourceLocation(), context.getIdAllocator().getNextId(), rewrittenLateralJoinNode, isDistinct, rewrittenLateralJoinNode.getInput().getOutputVariables(), Optional.empty());
    FilterNode filterNode = new FilterNode(markDistinctNode.getSourceLocation(), context.getIdAllocator().getNextId(), markDistinctNode, castToRowExpression(new SimpleCaseExpression(createSymbolReference(isDistinct), ImmutableList.of(new WhenClause(TRUE_LITERAL, TRUE_LITERAL)), Optional.of(new Cast(new FunctionCall(QualifiedName.of("fail"), ImmutableList.of(new LongLiteral(Integer.toString(SUBQUERY_MULTIPLE_ROWS.toErrorCode().getCode())), new StringLiteral("Scalar sub-query has returned multiple rows"))), BOOLEAN)))));
    return Result.ofPlanNode(new ProjectNode(context.getIdAllocator().getNextId(), filterNode, identityAssignmentsAsSymbolReferences(lateralJoinNode.getOutputVariables())));
}
Also used : Cast(com.facebook.presto.sql.tree.Cast) MarkDistinctNode(com.facebook.presto.spi.plan.MarkDistinctNode) LongLiteral(com.facebook.presto.sql.tree.LongLiteral) FilterNode(com.facebook.presto.spi.plan.FilterNode) SimpleCaseExpression(com.facebook.presto.sql.tree.SimpleCaseExpression) WhenClause(com.facebook.presto.sql.tree.WhenClause) PlanNode(com.facebook.presto.spi.plan.PlanNode) AssignUniqueId(com.facebook.presto.sql.planner.plan.AssignUniqueId) LateralJoinNode(com.facebook.presto.sql.planner.plan.LateralJoinNode) StringLiteral(com.facebook.presto.sql.tree.StringLiteral) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) EnforceSingleRowNode(com.facebook.presto.sql.planner.plan.EnforceSingleRowNode) ProjectNode(com.facebook.presto.spi.plan.ProjectNode) FunctionCall(com.facebook.presto.sql.tree.FunctionCall)

Aggregations

PlanNode (com.facebook.presto.spi.plan.PlanNode)228 Test (org.testng.annotations.Test)114 VariableReferenceExpression (com.facebook.presto.spi.relation.VariableReferenceExpression)94 ImmutableList (com.google.common.collect.ImmutableList)56 RowExpression (com.facebook.presto.spi.relation.RowExpression)45 ImmutableMap (com.google.common.collect.ImmutableMap)44 PlanBuilder (com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder)42 TableScanNode (com.facebook.presto.spi.plan.TableScanNode)41 AggregationNode (com.facebook.presto.spi.plan.AggregationNode)40 ProjectNode (com.facebook.presto.spi.plan.ProjectNode)40 Map (java.util.Map)39 Optional (java.util.Optional)38 List (java.util.List)36 JoinNode (com.facebook.presto.sql.planner.plan.JoinNode)35 Set (java.util.Set)30 Assert.assertEquals (org.testng.Assert.assertEquals)23 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)22 OrderingScheme (com.facebook.presto.spi.plan.OrderingScheme)20 Function (java.util.function.Function)20 Metadata (com.facebook.presto.metadata.Metadata)19