Search in sources :

Example 1 with TableStatisticAggregation

use of com.facebook.presto.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation in project presto by prestodb.

the class LogicalPlanner method createTableWriterPlan.

private RelationPlan createTableWriterPlan(Analysis analysis, RelationPlan plan, WriterTarget target, List<String> columnNames, List<ColumnMetadata> columnMetadataList, Optional<NewTableLayout> writeTableLayout, Optional<NewTableLayout> preferredShuffleLayout, TableStatisticsMetadata statisticsMetadata) {
    verify(!(writeTableLayout.isPresent() && preferredShuffleLayout.isPresent()), "writeTableLayout and preferredShuffleLayout cannot both exist");
    PlanNode source = plan.getRoot();
    if (!analysis.isCreateTableAsSelectWithData()) {
        source = new LimitNode(source.getSourceLocation(), idAllocator.getNextId(), source, 0L, FINAL);
    }
    List<VariableReferenceExpression> variables = plan.getFieldMappings();
    Optional<PartitioningScheme> tablePartitioningScheme = getPartitioningSchemeForTableWrite(writeTableLayout, columnNames, variables);
    Optional<PartitioningScheme> preferredShufflePartitioningScheme = getPartitioningSchemeForTableWrite(preferredShuffleLayout, columnNames, variables);
    verify(columnNames.size() == variables.size(), "columnNames.size() != variables.size(): %s and %s", columnNames, variables);
    Map<String, VariableReferenceExpression> columnToVariableMap = zip(columnNames.stream(), plan.getFieldMappings().stream(), SimpleImmutableEntry::new).collect(toImmutableMap(Entry::getKey, Entry::getValue));
    Set<VariableReferenceExpression> notNullColumnVariables = columnMetadataList.stream().filter(column -> !column.isNullable()).map(ColumnMetadata::getName).map(columnToVariableMap::get).collect(toImmutableSet());
    if (!statisticsMetadata.isEmpty()) {
        TableStatisticAggregation result = statisticsAggregationPlanner.createStatisticsAggregation(statisticsMetadata, columnToVariableMap, true);
        StatisticAggregations.Parts aggregations = result.getAggregations().splitIntoPartialAndFinal(variableAllocator, metadata.getFunctionAndTypeManager());
        TableFinishNode commitNode = new TableFinishNode(source.getSourceLocation(), idAllocator.getNextId(), new TableWriterNode(source.getSourceLocation(), idAllocator.getNextId(), source, Optional.of(target), variableAllocator.newVariable("rows", BIGINT), variableAllocator.newVariable("fragments", VARBINARY), variableAllocator.newVariable("commitcontext", VARBINARY), plan.getFieldMappings(), columnNames, notNullColumnVariables, tablePartitioningScheme, preferredShufflePartitioningScheme, // the data consumed by the TableWriteOperator
        Optional.of(aggregations.getPartialAggregation())), Optional.of(target), variableAllocator.newVariable("rows", BIGINT), // by the partial aggregation from all of the writer nodes
        Optional.of(aggregations.getFinalAggregation()), Optional.of(result.getDescriptor()));
        return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputVariables());
    }
    TableFinishNode commitNode = new TableFinishNode(source.getSourceLocation(), idAllocator.getNextId(), new TableWriterNode(source.getSourceLocation(), idAllocator.getNextId(), source, Optional.of(target), variableAllocator.newVariable("rows", BIGINT), variableAllocator.newVariable("fragments", VARBINARY), variableAllocator.newVariable("commitcontext", VARBINARY), plan.getFieldMappings(), columnNames, notNullColumnVariables, tablePartitioningScheme, preferredShufflePartitioningScheme, Optional.empty()), Optional.of(target), variableAllocator.newVariable("rows", BIGINT), Optional.empty(), Optional.empty());
    return new RelationPlan(commitNode, analysis.getRootScope(), commitNode.getOutputVariables());
}
Also used : WarningCollector(com.facebook.presto.spi.WarningCollector) FINAL(com.facebook.presto.spi.plan.LimitNode.Step.FINAL) NOT_FOUND(com.facebook.presto.spi.StandardErrorCode.NOT_FOUND) Analyze(com.facebook.presto.sql.tree.Analyze) Field(com.facebook.presto.sql.analyzer.Field) CachingStatsProvider(com.facebook.presto.cost.CachingStatsProvider) ValuesNode(com.facebook.presto.spi.plan.ValuesNode) TableStatisticAggregation(com.facebook.presto.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation) Delete(com.facebook.presto.sql.tree.Delete) Map(java.util.Map) QualifiedObjectName(com.facebook.presto.common.QualifiedObjectName) AggregationNode.singleGroupingSet(com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet) Query(com.facebook.presto.sql.tree.Query) Explain(com.facebook.presto.sql.tree.Explain) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) LimitNode(com.facebook.presto.spi.plan.LimitNode) NullLiteral(com.facebook.presto.sql.tree.NullLiteral) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) ProjectNode(com.facebook.presto.spi.plan.ProjectNode) DeleteHandle(com.facebook.presto.sql.planner.plan.TableWriterNode.DeleteHandle) ExpressionTreeUtils.getSourceLocation(com.facebook.presto.sql.analyzer.ExpressionTreeUtils.getSourceLocation) TableStatisticsMetadata(com.facebook.presto.spi.statistics.TableStatisticsMetadata) SystemSessionProperties(com.facebook.presto.SystemSessionProperties) PlanOptimizer(com.facebook.presto.sql.planner.optimizations.PlanOptimizer) LambdaArgumentDeclaration(com.facebook.presto.sql.tree.LambdaArgumentDeclaration) SystemSessionProperties.isPrintStatsForNonJoinQuery(com.facebook.presto.SystemSessionProperties.isPrintStatsForNonJoinQuery) NewTableLayout(com.facebook.presto.metadata.NewTableLayout) Assignments(com.facebook.presto.spi.plan.Assignments) ArrayList(java.util.ArrayList) Identifier(com.facebook.presto.sql.tree.Identifier) LinkedHashMap(java.util.LinkedHashMap) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) TableHandle(com.facebook.presto.spi.TableHandle) Cast(com.facebook.presto.sql.tree.Cast) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) MetadataUtil.toSchemaTableName(com.facebook.presto.metadata.MetadataUtil.toSchemaTableName) PlanNodeIdAllocator(com.facebook.presto.spi.plan.PlanNodeIdAllocator) Session(com.facebook.presto.Session) InsertReference(com.facebook.presto.sql.planner.plan.TableWriterNode.InsertReference) StatsProvider(com.facebook.presto.cost.StatsProvider) RefreshMaterializedViewReference(com.facebook.presto.sql.planner.plan.TableWriterNode.RefreshMaterializedViewReference) NodeRef(com.facebook.presto.sql.tree.NodeRef) Scope(com.facebook.presto.sql.analyzer.Scope) ColumnHandle(com.facebook.presto.spi.ColumnHandle) TableScanNode(com.facebook.presto.spi.plan.TableScanNode) SemiJoinNode(com.facebook.presto.sql.planner.plan.SemiJoinNode) Metadata(com.facebook.presto.metadata.Metadata) ExpressionTreeUtils.createSymbolReference(com.facebook.presto.sql.analyzer.ExpressionTreeUtils.createSymbolReference) OriginalExpressionUtils.castToRowExpression(com.facebook.presto.sql.relational.OriginalExpressionUtils.castToRowExpression) TableMetadata(com.facebook.presto.metadata.TableMetadata) AggregationNode(com.facebook.presto.spi.plan.AggregationNode) ExplainAnalyzeNode(com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) TableWriterNode(com.facebook.presto.sql.planner.plan.TableWriterNode) Expressions.constant(com.facebook.presto.sql.relational.Expressions.constant) StatsCalculator(com.facebook.presto.cost.StatsCalculator) StatisticAggregations(com.facebook.presto.sql.planner.plan.StatisticAggregations) RefreshMaterializedView(com.facebook.presto.sql.tree.RefreshMaterializedView) CostProvider(com.facebook.presto.cost.CostProvider) OutputNode(com.facebook.presto.sql.planner.plan.OutputNode) RelationId(com.facebook.presto.sql.analyzer.RelationId) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) CostCalculator(com.facebook.presto.cost.CostCalculator) String.format(java.lang.String.format) SqlParser(com.facebook.presto.sql.parser.SqlParser) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) Entry(java.util.Map.Entry) Analysis(com.facebook.presto.sql.analyzer.Analysis) Optional(java.util.Optional) ConnectorId(com.facebook.presto.spi.ConnectorId) CachingCostProvider(com.facebook.presto.cost.CachingCostProvider) WriterTarget(com.facebook.presto.sql.planner.plan.TableWriterNode.WriterTarget) PrestoException(com.facebook.presto.spi.PrestoException) SimpleImmutableEntry(java.util.AbstractMap.SimpleImmutableEntry) ImmutableList(com.google.common.collect.ImmutableList) PlanChecker(com.facebook.presto.sql.planner.sanity.PlanChecker) Verify.verify(com.google.common.base.Verify.verify) Objects.requireNonNull(java.util.Objects.requireNonNull) Type(com.facebook.presto.common.type.Type) JoinNode(com.facebook.presto.sql.planner.plan.JoinNode) CreateTableAsSelect(com.facebook.presto.sql.tree.CreateTableAsSelect) ROW_COUNT(com.facebook.presto.spi.statistics.TableStatisticType.ROW_COUNT) PlanNodeSearcher(com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher) RelationType(com.facebook.presto.sql.analyzer.RelationType) VARBINARY(com.facebook.presto.common.type.VarbinaryType.VARBINARY) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) DeleteNode(com.facebook.presto.sql.planner.plan.DeleteNode) Insert(com.facebook.presto.sql.tree.Insert) PlanNode(com.facebook.presto.spi.plan.PlanNode) Expression(com.facebook.presto.sql.tree.Expression) CreateName(com.facebook.presto.sql.planner.plan.TableWriterNode.CreateName) StatsAndCosts(com.facebook.presto.cost.StatsAndCosts) TableFinishNode(com.facebook.presto.sql.planner.plan.TableFinishNode) Streams.zip(com.google.common.collect.Streams.zip) StatisticsWriterNode(com.facebook.presto.sql.planner.plan.StatisticsWriterNode) Statement(com.facebook.presto.sql.tree.Statement) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) TableStatisticAggregation(com.facebook.presto.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation) TableFinishNode(com.facebook.presto.sql.planner.plan.TableFinishNode) StatisticAggregations(com.facebook.presto.sql.planner.plan.StatisticAggregations) PlanNode(com.facebook.presto.spi.plan.PlanNode) LimitNode(com.facebook.presto.spi.plan.LimitNode) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) TableWriterNode(com.facebook.presto.sql.planner.plan.TableWriterNode)

Example 2 with TableStatisticAggregation

use of com.facebook.presto.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation in project presto by prestodb.

the class LogicalPlanner method createAnalyzePlan.

private RelationPlan createAnalyzePlan(Analysis analysis, Analyze analyzeStatement) {
    TableHandle targetTable = analysis.getAnalyzeTarget().get();
    // Plan table scan
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, targetTable);
    ImmutableList.Builder<VariableReferenceExpression> tableScanOutputsBuilder = ImmutableList.builder();
    ImmutableMap.Builder<VariableReferenceExpression, ColumnHandle> variableToColumnHandle = ImmutableMap.builder();
    ImmutableMap.Builder<String, VariableReferenceExpression> columnNameToVariable = ImmutableMap.builder();
    TableMetadata tableMetadata = metadata.getTableMetadata(session, targetTable);
    for (ColumnMetadata column : tableMetadata.getColumns()) {
        VariableReferenceExpression variable = variableAllocator.newVariable(getSourceLocation(analyzeStatement), column.getName(), column.getType());
        tableScanOutputsBuilder.add(variable);
        variableToColumnHandle.put(variable, columnHandles.get(column.getName()));
        columnNameToVariable.put(column.getName(), variable);
    }
    List<VariableReferenceExpression> tableScanOutputs = tableScanOutputsBuilder.build();
    TableStatisticsMetadata tableStatisticsMetadata = metadata.getStatisticsCollectionMetadata(session, targetTable.getConnectorId().getCatalogName(), tableMetadata.getMetadata());
    TableStatisticAggregation tableStatisticAggregation = statisticsAggregationPlanner.createStatisticsAggregation(tableStatisticsMetadata, columnNameToVariable.build(), true);
    StatisticAggregations statisticAggregations = tableStatisticAggregation.getAggregations();
    PlanNode planNode = new StatisticsWriterNode(getSourceLocation(analyzeStatement), idAllocator.getNextId(), new AggregationNode(getSourceLocation(analyzeStatement), idAllocator.getNextId(), new TableScanNode(getSourceLocation(analyzeStatement), idAllocator.getNextId(), targetTable, tableScanOutputs, variableToColumnHandle.build(), TupleDomain.all(), TupleDomain.all()), statisticAggregations.getAggregations(), singleGroupingSet(statisticAggregations.getGroupingVariables()), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), Optional.empty()), targetTable, variableAllocator.newVariable(getSourceLocation(analyzeStatement), "rows", BIGINT), tableStatisticsMetadata.getTableStatistics().contains(ROW_COUNT), tableStatisticAggregation.getDescriptor());
    return new RelationPlan(planNode, analysis.getScope(analyzeStatement), planNode.getOutputVariables());
}
Also used : ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) TableMetadata(com.facebook.presto.metadata.TableMetadata) ColumnHandle(com.facebook.presto.spi.ColumnHandle) TableStatisticsMetadata(com.facebook.presto.spi.statistics.TableStatisticsMetadata) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) TableStatisticAggregation(com.facebook.presto.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ImmutableList(com.google.common.collect.ImmutableList) StatisticsWriterNode(com.facebook.presto.sql.planner.plan.StatisticsWriterNode) AggregationNode(com.facebook.presto.spi.plan.AggregationNode) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) ImmutableMap(com.google.common.collect.ImmutableMap) StatisticAggregations(com.facebook.presto.sql.planner.plan.StatisticAggregations) PlanNode(com.facebook.presto.spi.plan.PlanNode) TableScanNode(com.facebook.presto.spi.plan.TableScanNode) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) TableHandle(com.facebook.presto.spi.TableHandle)

Aggregations

TableMetadata (com.facebook.presto.metadata.TableMetadata)2 ColumnHandle (com.facebook.presto.spi.ColumnHandle)2 ColumnMetadata (com.facebook.presto.spi.ColumnMetadata)2 ConnectorTableMetadata (com.facebook.presto.spi.ConnectorTableMetadata)2 TableHandle (com.facebook.presto.spi.TableHandle)2 AggregationNode (com.facebook.presto.spi.plan.AggregationNode)2 PlanNode (com.facebook.presto.spi.plan.PlanNode)2 TableScanNode (com.facebook.presto.spi.plan.TableScanNode)2 VariableReferenceExpression (com.facebook.presto.spi.relation.VariableReferenceExpression)2 Session (com.facebook.presto.Session)1 SystemSessionProperties (com.facebook.presto.SystemSessionProperties)1 SystemSessionProperties.isPrintStatsForNonJoinQuery (com.facebook.presto.SystemSessionProperties.isPrintStatsForNonJoinQuery)1 QualifiedObjectName (com.facebook.presto.common.QualifiedObjectName)1 TupleDomain (com.facebook.presto.common.predicate.TupleDomain)1 BIGINT (com.facebook.presto.common.type.BigintType.BIGINT)1 Type (com.facebook.presto.common.type.Type)1 VARBINARY (com.facebook.presto.common.type.VarbinaryType.VARBINARY)1 CachingCostProvider (com.facebook.presto.cost.CachingCostProvider)1 CachingStatsProvider (com.facebook.presto.cost.CachingStatsProvider)1 CostCalculator (com.facebook.presto.cost.CostCalculator)1