Search in sources :

Example 1 with LimitNode

use of com.facebook.presto.spi.plan.LimitNode in project presto by prestodb.

the class PushLimitThroughOffset method apply.

@Override
public Result apply(LimitNode parent, Captures captures, Context context) {
    OffsetNode child = captures.get(CHILD);
    long count;
    try {
        count = addExact(parent.getCount(), child.getCount());
    } catch (ArithmeticException e) {
        return Result.empty();
    }
    return Result.ofPlanNode(child.replaceChildren(ImmutableList.of(new LimitNode(parent.getSourceLocation(), parent.getId(), child.getSource(), count, parent.getStep()))));
}
Also used : LimitNode(com.facebook.presto.spi.plan.LimitNode) OffsetNode(com.facebook.presto.sql.planner.plan.OffsetNode)

Example 2 with LimitNode

use of com.facebook.presto.spi.plan.LimitNode in project presto by prestodb.

the class LogicalPlanner method createTableWriterPlan.

private RelationPlan createTableWriterPlan(Analysis analysis, RelationPlan plan, WriterTarget target, List<String> columnNames, List<ColumnMetadata> columnMetadataList, Optional<NewTableLayout> writeTableLayout, Optional<NewTableLayout> preferredShuffleLayout, TableStatisticsMetadata statisticsMetadata) {
    verify(!(writeTableLayout.isPresent() && preferredShuffleLayout.isPresent()), "writeTableLayout and preferredShuffleLayout cannot both exist");
    PlanNode source = plan.getRoot();
    if (!analysis.isCreateTableAsSelectWithData()) {
        source = new LimitNode(source.getSourceLocation(), idAllocator.getNextId(), source, 0L, FINAL);
    }
    List<VariableReferenceExpression> variables = plan.getFieldMappings();
    Optional<PartitioningScheme> tablePartitioningScheme = getPartitioningSchemeForTableWrite(writeTableLayout, columnNames, variables);
    Optional<PartitioningScheme> preferredShufflePartitioningScheme = getPartitioningSchemeForTableWrite(preferredShuffleLayout, columnNames, variables);
    verify(columnNames.size() == variables.size(), "columnNames.size() != variables.size(): %s and %s", columnNames, variables);
    Map<String, VariableReferenceExpression> columnToVariableMap = zip(columnNames.stream(), plan.getFieldMappings().stream(), SimpleImmutableEntry::new).collect(toImmutableMap(Entry::getKey, Entry::getValue));
    Set<VariableReferenceExpression> notNullColumnVariables = columnMetadataList.stream().filter(column -> !column.isNullable()).map(ColumnMetadata::getName).map(columnToVariableMap::get).collect(toImmutableSet());
    if (!statisticsMetadata.isEmpty()) {
        TableStatisticAggregation result = statisticsAggregationPlanner.createStatisticsAggregation(statisticsMetadata, columnToVariableMap, true);
        StatisticAggregations.Parts aggregations = result.getAggregations().splitIntoPartialAndFinal(variableAllocator, metadata.getFunctionAndTypeManager());
        TableFinishNode commitNode = new TableFinishNode(source.getSourceLocation(), idAllocator.getNextId(), new TableWriterNode(source.getSourceLocation(), idAllocator.getNextId(), source, Optional.of(target), variableAllocator.newVariable("rows", BIGINT), variableAllocator.newVariable("fragments", VARBINARY), variableAllocator.newVariable("commitcontext", VARBINARY), plan.getFieldMappings(), columnNames, notNullColumnVariables, tablePartitioningScheme, preferredShufflePartitioningScheme, // the data consumed by the TableWriteOperator
        Optional.of(aggregations.getPartialAggregation())), Optional.of(target), variableAllocator.newVariable("rows", BIGINT), // by the partial aggregation from all of the writer nodes
        Optional.of(aggregations.getFinalAggregation()), Optional.of(result.getDescriptor()));
        return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputVariables());
    }
    TableFinishNode commitNode = new TableFinishNode(source.getSourceLocation(), idAllocator.getNextId(), new TableWriterNode(source.getSourceLocation(), idAllocator.getNextId(), source, Optional.of(target), variableAllocator.newVariable("rows", BIGINT), variableAllocator.newVariable("fragments", VARBINARY), variableAllocator.newVariable("commitcontext", VARBINARY), plan.getFieldMappings(), columnNames, notNullColumnVariables, tablePartitioningScheme, preferredShufflePartitioningScheme, Optional.empty()), Optional.of(target), variableAllocator.newVariable("rows", BIGINT), Optional.empty(), Optional.empty());
    return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputVariables());
}
Also used : WarningCollector(com.facebook.presto.spi.WarningCollector) FINAL(com.facebook.presto.spi.plan.LimitNode.Step.FINAL) NOT_FOUND(com.facebook.presto.spi.StandardErrorCode.NOT_FOUND) Analyze(com.facebook.presto.sql.tree.Analyze) Field(com.facebook.presto.sql.analyzer.Field) CachingStatsProvider(com.facebook.presto.cost.CachingStatsProvider) ValuesNode(com.facebook.presto.spi.plan.ValuesNode) TableStatisticAggregation(com.facebook.presto.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation) Delete(com.facebook.presto.sql.tree.Delete) Map(java.util.Map) QualifiedObjectName(com.facebook.presto.common.QualifiedObjectName) AggregationNode.singleGroupingSet(com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet) Query(com.facebook.presto.sql.tree.Query) Explain(com.facebook.presto.sql.tree.Explain) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) LimitNode(com.facebook.presto.spi.plan.LimitNode) NullLiteral(com.facebook.presto.sql.tree.NullLiteral) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) ProjectNode(com.facebook.presto.spi.plan.ProjectNode) DeleteHandle(com.facebook.presto.sql.planner.plan.TableWriterNode.DeleteHandle) ExpressionTreeUtils.getSourceLocation(com.facebook.presto.sql.analyzer.ExpressionTreeUtils.getSourceLocation) TableStatisticsMetadata(com.facebook.presto.spi.statistics.TableStatisticsMetadata) SystemSessionProperties(com.facebook.presto.SystemSessionProperties) PlanOptimizer(com.facebook.presto.sql.planner.optimizations.PlanOptimizer) LambdaArgumentDeclaration(com.facebook.presto.sql.tree.LambdaArgumentDeclaration) SystemSessionProperties.isPrintStatsForNonJoinQuery(com.facebook.presto.SystemSessionProperties.isPrintStatsForNonJoinQuery) NewTableLayout(com.facebook.presto.metadata.NewTableLayout) Assignments(com.facebook.presto.spi.plan.Assignments) ArrayList(java.util.ArrayList) Identifier(com.facebook.presto.sql.tree.Identifier) LinkedHashMap(java.util.LinkedHashMap) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) TableHandle(com.facebook.presto.spi.TableHandle) Cast(com.facebook.presto.sql.tree.Cast) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) MetadataUtil.toSchemaTableName(com.facebook.presto.metadata.MetadataUtil.toSchemaTableName) PlanNodeIdAllocator(com.facebook.presto.spi.plan.PlanNodeIdAllocator) Session(com.facebook.presto.Session) InsertReference(com.facebook.presto.sql.planner.plan.TableWriterNode.InsertReference) StatsProvider(com.facebook.presto.cost.StatsProvider) RefreshMaterializedViewReference(com.facebook.presto.sql.planner.plan.TableWriterNode.RefreshMaterializedViewReference) NodeRef(com.facebook.presto.sql.tree.NodeRef) Scope(com.facebook.presto.sql.analyzer.Scope) ColumnHandle(com.facebook.presto.spi.ColumnHandle) TableScanNode(com.facebook.presto.spi.plan.TableScanNode) SemiJoinNode(com.facebook.presto.sql.planner.plan.SemiJoinNode) Metadata(com.facebook.presto.metadata.Metadata) ExpressionTreeUtils.createSymbolReference(com.facebook.presto.sql.analyzer.ExpressionTreeUtils.createSymbolReference) OriginalExpressionUtils.castToRowExpression(com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression) TableMetadata(com.facebook.presto.metadata.TableMetadata) AggregationNode(com.facebook.presto.spi.plan.AggregationNode) ExplainAnalyzeNode(com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) TableWriterNode(com.facebook.presto.sql.planner.plan.TableWriterNode) Expressions.constant(com.facebook.presto.sql.relational.Expressions.constant) StatsCalculator(com.facebook.presto.cost.StatsCalculator) StatisticAggregations(com.facebook.presto.sql.planner.plan.StatisticAggregations) RefreshMaterializedView(com.facebook.presto.sql.tree.RefreshMaterializedView) CostProvider(com.facebook.presto.cost.CostProvider) OutputNode(com.facebook.presto.sql.planner.plan.OutputNode) RelationId(com.facebook.presto.sql.analyzer.RelationId) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) CostCalculator(com.facebook.presto.cost.CostCalculator) String.format(java.lang.String.format) SqlParser(com.facebook.presto.sql.parser.SqlParser) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) Entry(java.util.Map.Entry) Analysis(com.facebook.presto.sql.analyzer.Analysis) Optional(java.util.Optional) ConnectorId(com.facebook.presto.spi.ConnectorId) CachingCostProvider(com.facebook.presto.cost.CachingCostProvider) WriterTarget(com.facebook.presto.sql.planner.plan.TableWriterNode.WriterTarget) PrestoException(com.facebook.presto.spi.PrestoException) SimpleImmutableEntry(java.util.AbstractMap.SimpleImmutableEntry) ImmutableList(com.google.common.collect.ImmutableList) PlanChecker(com.facebook.presto.sql.planner.sanity.PlanChecker) Verify.verify(com.google.common.base.Verify.verify) Objects.requireNonNull(java.util.Objects.requireNonNull) Type(com.facebook.presto.common.type.Type) JoinNode(com.facebook.presto.sql.planner.plan.JoinNode) CreateTableAsSelect(com.facebook.presto.sql.tree.CreateTableAsSelect) ROW_COUNT(com.facebook.presto.spi.statistics.TableStatisticType.ROW_COUNT) PlanNodeSearcher(com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher) RelationType(com.facebook.presto.sql.analyzer.RelationType) VARBINARY(com.facebook.presto.common.type.VarbinaryType.VARBINARY) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) DeleteNode(com.facebook.presto.sql.planner.plan.DeleteNode) Insert(com.facebook.presto.sql.tree.Insert) PlanNode(com.facebook.presto.spi.plan.PlanNode) Expression(com.facebook.presto.sql.tree.Expression) CreateName(com.facebook.presto.sql.planner.plan.TableWriterNode.CreateName) StatsAndCosts(com.facebook.presto.cost.StatsAndCosts) TableFinishNode(com.facebook.presto.sql.planner.plan.TableFinishNode) Streams.zip(com.google.common.collect.Streams.zip) StatisticsWriterNode(com.facebook.presto.sql.planner.plan.StatisticsWriterNode) Statement(com.facebook.presto.sql.tree.Statement) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) TableStatisticAggregation(com.facebook.presto.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation) TableFinishNode(com.facebook.presto.sql.planner.plan.TableFinishNode) StatisticAggregations(com.facebook.presto.sql.planner.plan.StatisticAggregations) PlanNode(com.facebook.presto.spi.plan.PlanNode) LimitNode(com.facebook.presto.spi.plan.LimitNode) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) TableWriterNode(com.facebook.presto.sql.planner.plan.TableWriterNode)

Example 3 with LimitNode

use of com.facebook.presto.spi.plan.LimitNode in project presto by prestodb.

the class PushLimitThroughUnion method apply.

@Override
public Result apply(LimitNode parent, Captures captures, Context context) {
    UnionNode unionNode = captures.get(CHILD);
    if (unionNode.getSources().stream().allMatch(source -> isAtMost(source, context.getLookup(), parent.getCount()))) {
        return Result.empty();
    }
    ImmutableList.Builder<PlanNode> builder = ImmutableList.builder();
    for (PlanNode source : unionNode.getSources()) {
        // This check is to ensure that we don't fire the optimizer if it was previously applied.
        if (isAtMost(source, context.getLookup(), parent.getCount())) {
            builder.add(source);
        } else {
            builder.add(new LimitNode(parent.getSourceLocation(), context.getIdAllocator().getNextId(), source, parent.getCount(), LimitNode.Step.PARTIAL));
        }
    }
    return Result.ofPlanNode(parent.replaceChildren(ImmutableList.of(unionNode.replaceChildren(builder.build()))));
}
Also used : PlanNode(com.facebook.presto.spi.plan.PlanNode) UnionNode(com.facebook.presto.spi.plan.UnionNode) LimitNode(com.facebook.presto.spi.plan.LimitNode) ImmutableList(com.google.common.collect.ImmutableList)

Example 4 with LimitNode

use of com.facebook.presto.spi.plan.LimitNode in project presto by prestodb.

the class PushLimitThroughOuterJoin method apply.

@Override
public Result apply(LimitNode parent, Captures captures, Context context) {
    if (!isPushLimitThroughOuterJoin(context.getSession())) {
        return Result.empty();
    }
    JoinNode joinNode = captures.get(CHILD);
    PlanNode left = joinNode.getLeft();
    PlanNode right = joinNode.getRight();
    if (joinNode.getType() == LEFT && !isLimited(left, context.getLookup(), parent.getCount())) {
        left = new LimitNode(parent.getSourceLocation(), context.getIdAllocator().getNextId(), left, parent.getCount(), PARTIAL);
    }
    if (joinNode.getType() == RIGHT && !isLimited(right, context.getLookup(), parent.getCount())) {
        right = new LimitNode(parent.getSourceLocation(), context.getIdAllocator().getNextId(), right, parent.getCount(), PARTIAL);
    }
    if (joinNode.getLeft() != left || joinNode.getRight() != right) {
        return Result.ofPlanNode(parent.replaceChildren(ImmutableList.of(joinNode.replaceChildren(ImmutableList.of(left, right)))));
    }
    return Result.empty();
}
Also used : PlanNode(com.facebook.presto.spi.plan.PlanNode) LimitNode(com.facebook.presto.spi.plan.LimitNode) JoinNode(com.facebook.presto.sql.planner.plan.JoinNode)

Example 5 with LimitNode

use of com.facebook.presto.spi.plan.LimitNode in project presto by prestodb.

the class TransformExistsApplyToLateralNode method rewriteToNonDefaultAggregation.

private Optional<PlanNode> rewriteToNonDefaultAggregation(ApplyNode applyNode, Context context) {
    checkState(applyNode.getSubquery().getOutputVariables().isEmpty(), "Expected subquery output variables to be pruned");
    VariableReferenceExpression exists = getOnlyElement(applyNode.getSubqueryAssignments().getVariables());
    VariableReferenceExpression subqueryTrue = context.getVariableAllocator().newVariable(exists.getSourceLocation(), "subqueryTrue", BOOLEAN);
    Assignments.Builder assignments = Assignments.builder();
    assignments.putAll(identitiesAsSymbolReferences(applyNode.getInput().getOutputVariables()));
    assignments.put(exists, castToRowExpression(new CoalesceExpression(ImmutableList.of(createSymbolReference(subqueryTrue), BooleanLiteral.FALSE_LITERAL))));
    PlanNode subquery = new ProjectNode(context.getIdAllocator().getNextId(), new LimitNode(applyNode.getSourceLocation(), context.getIdAllocator().getNextId(), applyNode.getSubquery(), 1L, FINAL), Assignments.of(subqueryTrue, castToRowExpression(TRUE_LITERAL)));
    PlanNodeDecorrelator decorrelator = new PlanNodeDecorrelator(context.getIdAllocator(), context.getVariableAllocator(), context.getLookup());
    if (!decorrelator.decorrelateFilters(subquery, applyNode.getCorrelation()).isPresent()) {
        return Optional.empty();
    }
    return Optional.of(new ProjectNode(context.getIdAllocator().getNextId(), new LateralJoinNode(applyNode.getSourceLocation(), applyNode.getId(), applyNode.getInput(), subquery, applyNode.getCorrelation(), LEFT, applyNode.getOriginSubqueryError()), assignments.build()));
}
Also used : PlanNodeDecorrelator(com.facebook.presto.sql.planner.optimizations.PlanNodeDecorrelator) PlanNode(com.facebook.presto.spi.plan.PlanNode) LateralJoinNode(com.facebook.presto.sql.planner.plan.LateralJoinNode) LimitNode(com.facebook.presto.spi.plan.LimitNode) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) Assignments(com.facebook.presto.spi.plan.Assignments) ProjectNode(com.facebook.presto.spi.plan.ProjectNode) CoalesceExpression(com.facebook.presto.sql.tree.CoalesceExpression)

Aggregations

LimitNode (com.facebook.presto.spi.plan.LimitNode)6 PlanNode (com.facebook.presto.spi.plan.PlanNode)5 Assignments (com.facebook.presto.spi.plan.Assignments)2 ProjectNode (com.facebook.presto.spi.plan.ProjectNode)2 VariableReferenceExpression (com.facebook.presto.spi.relation.VariableReferenceExpression)2 Session (com.facebook.presto.Session)1 SystemSessionProperties (com.facebook.presto.SystemSessionProperties)1 SystemSessionProperties.isPrintStatsForNonJoinQuery (com.facebook.presto.SystemSessionProperties.isPrintStatsForNonJoinQuery)1 QualifiedObjectName (com.facebook.presto.common.QualifiedObjectName)1 TupleDomain (com.facebook.presto.common.predicate.TupleDomain)1 BIGINT (com.facebook.presto.common.type.BigintType.BIGINT)1 Type (com.facebook.presto.common.type.Type)1 VARBINARY (com.facebook.presto.common.type.VarbinaryType.VARBINARY)1 CachingCostProvider (com.facebook.presto.cost.CachingCostProvider)1 CachingStatsProvider (com.facebook.presto.cost.CachingStatsProvider)1 CostCalculator (com.facebook.presto.cost.CostCalculator)1 CostProvider (com.facebook.presto.cost.CostProvider)1 StatsAndCosts (com.facebook.presto.cost.StatsAndCosts)1 StatsCalculator (com.facebook.presto.cost.StatsCalculator)1 StatsProvider (com.facebook.presto.cost.StatsProvider)1