Search in sources :

Example 1 with TableStatisticsMetadata

use of io.prestosql.spi.statistics.TableStatisticsMetadata in project hetu-core by openlookeng.

the class LogicalPlanner method createInsertPlan.

private RelationPlan createInsertPlan(Analysis analysis, Insert insertStatement) {
    Analysis.Insert insert = analysis.getInsert().get();
    TableMetadata tableMetadata = metadata.getTableMetadata(session, insert.getTarget());
    List<ColumnMetadata> visibleTableColumns = tableMetadata.getColumns().stream().filter(column -> !column.isHidden()).collect(toImmutableList());
    List<String> visibleTableColumnNames = visibleTableColumns.stream().map(ColumnMetadata::getName).collect(toImmutableList());
    RelationPlan plan = createRelationPlan(analysis, insertStatement.getQuery());
    Map<String, ColumnHandle> columns = metadata.getColumnHandles(session, insert.getTarget());
    Assignments.Builder assignments = Assignments.builder();
    for (ColumnMetadata column : tableMetadata.getColumns()) {
        if (column.isHidden()) {
            continue;
        }
        Symbol output = planSymbolAllocator.newSymbol(column.getName(), column.getType());
        int index = insert.getColumns().indexOf(columns.get(column.getName()));
        if (index < 0) {
            Expression cast = new Cast(new NullLiteral(), column.getType().getTypeSignature().toString());
            assignments.put(output, castToRowExpression(cast));
        } else {
            Symbol input = plan.getSymbol(index);
            Type tableType = column.getType();
            Type queryType = planSymbolAllocator.getTypes().get(input);
            if (queryType.equals(tableType) || typeCoercion.isTypeOnlyCoercion(queryType, tableType)) {
                assignments.put(output, castToRowExpression(toSymbolReference(input)));
            } else {
                Expression cast = noTruncationCast(toSymbolReference(input), queryType, tableType);
                assignments.put(output, castToRowExpression(cast));
            }
        }
    }
    ProjectNode projectNode = new ProjectNode(idAllocator.getNextId(), plan.getRoot(), assignments.build());
    List<Field> fields = visibleTableColumns.stream().map(column -> Field.newUnqualified(column.getName(), column.getType())).collect(toImmutableList());
    Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields)).build();
    plan = new RelationPlan(projectNode, scope, projectNode.getOutputSymbols());
    Optional<NewTableLayout> newTableLayout = metadata.getInsertLayout(session, insert.getTarget());
    String catalogName = insert.getTarget().getCatalogName().getCatalogName();
    TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogName, tableMetadata.getMetadata());
    return createTableWriterPlan(analysis, plan, new InsertReference(insert.getTarget(), analysis.isInsertOverwrite()), visibleTableColumnNames, newTableLayout, statisticsMetadata);
}
Also used : GREATER_THAN_OR_EQUAL(io.prestosql.sql.tree.ComparisonExpression.Operator.GREATER_THAN_OR_EQUAL) LongSupplier(java.util.function.LongSupplier) CostCalculator(io.prestosql.cost.CostCalculator) ConstantExpression(io.prestosql.spi.relation.ConstantExpression) CreateReference(io.prestosql.sql.planner.plan.TableWriterNode.CreateReference) AggregationNode(io.prestosql.spi.plan.AggregationNode) Cast(io.prestosql.sql.tree.Cast) Statement(io.prestosql.sql.tree.Statement) WarningCollector(io.prestosql.execution.warnings.WarningCollector) ExpressionFormatter(io.prestosql.sql.ExpressionFormatter) SystemSessionProperties.isSkipAttachingStatsWithPlan(io.prestosql.SystemSessionProperties.isSkipAttachingStatsWithPlan) PlanSanityChecker(io.prestosql.sql.planner.sanity.PlanSanityChecker) Map(java.util.Map) OriginalExpressionUtils.castToRowExpression(io.prestosql.sql.relational.OriginalExpressionUtils.castToRowExpression) OutputNode(io.prestosql.sql.planner.plan.OutputNode) CubeFilter(io.hetu.core.spi.cube.CubeFilter) Identifier(io.prestosql.sql.tree.Identifier) CostProvider(io.prestosql.cost.CostProvider) Delete(io.prestosql.sql.tree.Delete) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) TableScanNode(io.prestosql.spi.plan.TableScanNode) TableStatisticsMetadata(io.prestosql.spi.statistics.TableStatisticsMetadata) Set(java.util.Set) NullLiteral(io.prestosql.sql.tree.NullLiteral) PlanNode(io.prestosql.spi.plan.PlanNode) ProjectNode(io.prestosql.spi.plan.ProjectNode) Metadata(io.prestosql.metadata.Metadata) Insert(io.prestosql.sql.tree.Insert) NodeRef(io.prestosql.sql.tree.NodeRef) CachingCostProvider(io.prestosql.cost.CachingCostProvider) SymbolUtils.toSymbolReference(io.prestosql.sql.planner.SymbolUtils.toSymbolReference) ReuseExchangeOperator(io.prestosql.spi.operator.ReuseExchangeOperator) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) ExpressionUtils(io.prestosql.sql.ExpressionUtils) GenericLiteral(io.prestosql.sql.tree.GenericLiteral) VacuumTargetReference(io.prestosql.sql.planner.plan.TableWriterNode.VacuumTargetReference) StringLiteral(io.prestosql.sql.tree.StringLiteral) StatisticAggregations(io.prestosql.sql.planner.plan.StatisticAggregations) StatisticsWriterNode(io.prestosql.sql.planner.plan.StatisticsWriterNode) VacuumTableNode(io.prestosql.sql.planner.plan.VacuumTableNode) WriterTarget(io.prestosql.sql.planner.plan.TableWriterNode.WriterTarget) Field(io.prestosql.sql.analyzer.Field) OriginalExpressionUtils(io.prestosql.sql.relational.OriginalExpressionUtils) Analyze(io.prestosql.sql.tree.Analyze) FIXED_HASH_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.FIXED_HASH_DISTRIBUTION) TableMetadata(io.prestosql.metadata.TableMetadata) VacuumTable(io.prestosql.sql.tree.VacuumTable) CharType(io.prestosql.spi.type.CharType) Node(io.prestosql.sql.tree.Node) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) VARCHAR(io.prestosql.spi.type.VarcharType.VARCHAR) CreateTableAsSelect(io.prestosql.sql.tree.CreateTableAsSelect) BooleanLiteral(io.prestosql.sql.tree.BooleanLiteral) Session(io.prestosql.Session) ParsingUtil.createParsingOptions(io.prestosql.sql.ParsingUtil.createParsingOptions) Signature(io.prestosql.spi.function.Signature) DeleteNode(io.prestosql.sql.planner.plan.DeleteNode) TypeSignatureProvider.fromTypes(io.prestosql.sql.analyzer.TypeSignatureProvider.fromTypes) StatsProvider(io.prestosql.cost.StatsProvider) Query(io.prestosql.sql.tree.Query) Assignments(io.prestosql.spi.plan.Assignments) QUERY_REJECTED(io.prestosql.spi.StandardErrorCode.QUERY_REJECTED) ComparisonExpression(io.prestosql.sql.tree.ComparisonExpression) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) Explain(io.prestosql.sql.tree.Explain) MetadataUtil.toSchemaTableName(io.prestosql.metadata.MetadataUtil.toSchemaTableName) VARBINARY(io.prestosql.spi.type.VarbinaryType.VARBINARY) ValuesNode(io.prestosql.spi.plan.ValuesNode) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) LimitNode(io.prestosql.spi.plan.LimitNode) CubeMetadata(io.hetu.core.spi.cube.CubeMetadata) PlanOptimizer(io.prestosql.sql.planner.optimizations.PlanOptimizer) VarcharType(io.prestosql.spi.type.VarcharType) Expression(io.prestosql.sql.tree.Expression) StatisticAggregationsDescriptor(io.prestosql.sql.planner.plan.StatisticAggregationsDescriptor) QualifiedName(io.prestosql.sql.tree.QualifiedName) APPLY_ALL_RULES(io.prestosql.spi.plan.PlanNode.SkipOptRuleLevel.APPLY_ALL_RULES) SqlParser(io.prestosql.sql.parser.SqlParser) CachingStatsProvider(io.prestosql.cost.CachingStatsProvider) TableFinishNode(io.prestosql.sql.planner.plan.TableFinishNode) Type(io.prestosql.spi.type.Type) TypeCoercion(io.prestosql.type.TypeCoercion) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) StatsCalculator(io.prestosql.cost.StatsCalculator) NewTableLayout(io.prestosql.metadata.NewTableLayout) AggregationNode.singleGroupingSet(io.prestosql.spi.plan.AggregationNode.singleGroupingSet) PrestoException(io.prestosql.spi.PrestoException) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) CatalogName(io.prestosql.spi.connector.CatalogName) ROW_COUNT(io.prestosql.spi.statistics.TableStatisticType.ROW_COUNT) UUID(java.util.UUID) RelationType(io.prestosql.sql.analyzer.RelationType) CubeFinishNode(io.prestosql.sql.planner.plan.CubeFinishNode) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) OptimizerUtils(io.prestosql.utils.OptimizerUtils) Scope(io.prestosql.sql.analyzer.Scope) List(java.util.List) Entry(java.util.Map.Entry) Optional(java.util.Optional) Analysis(io.prestosql.sql.analyzer.Analysis) NOT_SUPPORTED(io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED) IfExpression(io.prestosql.sql.tree.IfExpression) StatsAndCosts(io.prestosql.cost.StatsAndCosts) HashMap(java.util.HashMap) RelationId(io.prestosql.sql.analyzer.RelationId) SimpleImmutableEntry(java.util.AbstractMap.SimpleImmutableEntry) NOT_FOUND(io.prestosql.spi.StandardErrorCode.NOT_FOUND) TableHandle(io.prestosql.spi.metadata.TableHandle) Function(java.util.function.Function) HashSet(java.util.HashSet) CubeStatus(io.hetu.core.spi.cube.CubeStatus) ImmutableList(com.google.common.collect.ImmutableList) FunctionCall(io.prestosql.sql.tree.FunctionCall) Verify.verify(com.google.common.base.Verify.verify) InsertReference(io.prestosql.sql.planner.plan.TableWriterNode.InsertReference) Objects.requireNonNull(java.util.Objects.requireNonNull) TableStatisticAggregation(io.prestosql.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation) Symbol(io.prestosql.spi.plan.Symbol) TableWriterNode(io.prestosql.sql.planner.plan.TableWriterNode) CubeUpdateMetadata(io.prestosql.spi.cube.CubeUpdateMetadata) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) UpdateNode(io.prestosql.sql.planner.plan.UpdateNode) LambdaArgumentDeclaration(io.prestosql.sql.tree.LambdaArgumentDeclaration) Update(io.prestosql.sql.tree.Update) InsertCube(io.prestosql.sql.tree.InsertCube) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) DISTRIBUTED_PLAN_SANITY_CHECKER(io.prestosql.sql.planner.sanity.PlanSanityChecker.DISTRIBUTED_PLAN_SANITY_CHECKER) ExplainAnalyzeNode(io.prestosql.sql.planner.plan.ExplainAnalyzeNode) Streams.zip(com.google.common.collect.Streams.zip) Cast(io.prestosql.sql.tree.Cast) TableStatisticsMetadata(io.prestosql.spi.statistics.TableStatisticsMetadata) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) Symbol(io.prestosql.spi.plan.Symbol) NewTableLayout(io.prestosql.metadata.NewTableLayout) Assignments(io.prestosql.spi.plan.Assignments) InsertReference(io.prestosql.sql.planner.plan.TableWriterNode.InsertReference) Field(io.prestosql.sql.analyzer.Field) RelationType(io.prestosql.sql.analyzer.RelationType) TableMetadata(io.prestosql.metadata.TableMetadata) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) CharType(io.prestosql.spi.type.CharType) VarcharType(io.prestosql.spi.type.VarcharType) Type(io.prestosql.spi.type.Type) RelationType(io.prestosql.sql.analyzer.RelationType) Scope(io.prestosql.sql.analyzer.Scope) ConstantExpression(io.prestosql.spi.relation.ConstantExpression) OriginalExpressionUtils.castToRowExpression(io.prestosql.sql.relational.OriginalExpressionUtils.castToRowExpression) ComparisonExpression(io.prestosql.sql.tree.ComparisonExpression) Expression(io.prestosql.sql.tree.Expression) IfExpression(io.prestosql.sql.tree.IfExpression) Analysis(io.prestosql.sql.analyzer.Analysis) ProjectNode(io.prestosql.spi.plan.ProjectNode) NullLiteral(io.prestosql.sql.tree.NullLiteral)

Example 2 with TableStatisticsMetadata

use of io.prestosql.spi.statistics.TableStatisticsMetadata in project hetu-core by openlookeng.

the class LogicalPlanner method createInsertCubePlan.

private RelationPlan createInsertCubePlan(Analysis analysis, InsertCube insertCubeStatement) {
    Analysis.CubeInsert insert = analysis.getCubeInsert().get();
    TableMetadata tableMetadata = metadata.getTableMetadata(session, insert.getTarget());
    List<ColumnMetadata> visibleTableColumns = tableMetadata.getColumns().stream().filter(column -> !column.isHidden()).collect(toImmutableList());
    List<String> visibleTableColumnNames = visibleTableColumns.stream().map(ColumnMetadata::getName).collect(toImmutableList());
    RelationPlan plan = createRelationPlan(analysis, insertCubeStatement.getQuery());
    Map<String, ColumnHandle> columns = metadata.getColumnHandles(session, insert.getTarget());
    Assignments.Builder assignments = Assignments.builder();
    for (ColumnMetadata column : tableMetadata.getColumns()) {
        if (column.isHidden()) {
            continue;
        }
        Symbol output = planSymbolAllocator.newSymbol(column.getName(), column.getType());
        int index = insert.getColumns().indexOf(columns.get(column.getName()));
        if (index < 0) {
            Expression cast = new Cast(new NullLiteral(), column.getType().getTypeSignature().toString());
            assignments.put(output, castToRowExpression(cast));
        } else {
            Symbol input = plan.getSymbol(index);
            Type tableType = column.getType();
            Type queryType = planSymbolAllocator.getTypes().get(input);
            if (queryType.equals(tableType) || typeCoercion.isTypeOnlyCoercion(queryType, tableType)) {
                assignments.put(output, castToRowExpression(toSymbolReference(input)));
            } else {
                Expression cast = noTruncationCast(toSymbolReference(input), queryType, tableType);
                assignments.put(output, castToRowExpression(cast));
            }
        }
    }
    ProjectNode projectNode = new ProjectNode(idAllocator.getNextId(), plan.getRoot(), assignments.build());
    List<Field> fields = visibleTableColumns.stream().map(column -> Field.newUnqualified(column.getName(), column.getType())).collect(toImmutableList());
    Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields)).build();
    plan = new RelationPlan(projectNode, scope, projectNode.getOutputSymbols());
    Optional<NewTableLayout> newTableLayout = metadata.getInsertLayout(session, insert.getTarget());
    String catalogName = insert.getTarget().getCatalogName().getCatalogName();
    TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogName, tableMetadata.getMetadata());
    RelationPlan tableWriterPlan = createTableWriterPlan(analysis, plan, new InsertReference(insert.getTarget(), analysis.isCubeOverwrite()), visibleTableColumnNames, newTableLayout, statisticsMetadata);
    Expression rewritten = null;
    Set<Identifier> predicateColumns = new HashSet<>();
    if (insertCubeStatement.getWhere().isPresent()) {
        rewritten = new QueryPlanner(analysis, planSymbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator), metadata, session, namedSubPlan, uniqueIdAllocator).rewriteExpression(tableWriterPlan, insertCubeStatement.getWhere().get(), analysis, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator));
        predicateColumns.addAll(ExpressionUtils.getIdentifiers(rewritten));
    }
    CubeMetadata cubeMetadata = insert.getMetadata();
    if (!insertCubeStatement.isOverwrite() && !insertCubeStatement.getWhere().isPresent() && cubeMetadata.getCubeStatus() != CubeStatus.INACTIVE) {
        // Means data some data was inserted before, but trying to insert entire dataset
        throw new PrestoException(QUERY_REJECTED, "Cannot allow insert. Inserting entire dataset but cube already has partial data");
    } else if (insertCubeStatement.getWhere().isPresent()) {
        if (!canSupportPredicate(rewritten)) {
            throw new PrestoException(QUERY_REJECTED, String.format("Cannot support predicate '%s'", ExpressionFormatter.formatExpression(rewritten, Optional.empty())));
        }
        if (!insertCubeStatement.isOverwrite() && arePredicatesOverlapping(rewritten, cubeMetadata)) {
            throw new PrestoException(QUERY_REJECTED, String.format("Cannot allow insert. Cube already contains data for the given predicate '%s'", ExpressionFormatter.formatExpression(insertCubeStatement.getWhere().get(), Optional.empty())));
        }
    }
    TableHandle sourceTableHandle = insert.getSourceTable();
    // At this point it has been verified that source table has not been updated
    // so insert into cube should be allowed
    LongSupplier tableLastModifiedTimeSupplier = metadata.getTableLastModifiedTimeSupplier(session, sourceTableHandle);
    checkState(tableLastModifiedTimeSupplier != null, "Table last modified time is null");
    Map<Symbol, Type> predicateColumnsType = predicateColumns.stream().map(identifier -> new Symbol(identifier.getValue())).collect(Collectors.toMap(Function.identity(), symbol -> planSymbolAllocator.getTypes().get(symbol), (key1, ignored) -> key1));
    CubeFinishNode cubeFinishNode = new CubeFinishNode(idAllocator.getNextId(), tableWriterPlan.getRoot(), planSymbolAllocator.newSymbol("rows", BIGINT), new CubeUpdateMetadata(tableMetadata.getQualifiedName().toString(), tableLastModifiedTimeSupplier.getAsLong(), rewritten != null ? ExpressionFormatter.formatExpression(rewritten, Optional.empty()) : null, insertCubeStatement.isOverwrite()), predicateColumnsType);
    return new RelationPlan(cubeFinishNode, analysis.getScope(insertCubeStatement), cubeFinishNode.getOutputSymbols());
}
Also used : GREATER_THAN_OR_EQUAL(io.prestosql.sql.tree.ComparisonExpression.Operator.GREATER_THAN_OR_EQUAL) LongSupplier(java.util.function.LongSupplier) CostCalculator(io.prestosql.cost.CostCalculator) ConstantExpression(io.prestosql.spi.relation.ConstantExpression) CreateReference(io.prestosql.sql.planner.plan.TableWriterNode.CreateReference) AggregationNode(io.prestosql.spi.plan.AggregationNode) Cast(io.prestosql.sql.tree.Cast) Statement(io.prestosql.sql.tree.Statement) WarningCollector(io.prestosql.execution.warnings.WarningCollector) ExpressionFormatter(io.prestosql.sql.ExpressionFormatter) SystemSessionProperties.isSkipAttachingStatsWithPlan(io.prestosql.SystemSessionProperties.isSkipAttachingStatsWithPlan) PlanSanityChecker(io.prestosql.sql.planner.sanity.PlanSanityChecker) Map(java.util.Map) OriginalExpressionUtils.castToRowExpression(io.prestosql.sql.relational.OriginalExpressionUtils.castToRowExpression) OutputNode(io.prestosql.sql.planner.plan.OutputNode) CubeFilter(io.hetu.core.spi.cube.CubeFilter) Identifier(io.prestosql.sql.tree.Identifier) CostProvider(io.prestosql.cost.CostProvider) Delete(io.prestosql.sql.tree.Delete) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) TableScanNode(io.prestosql.spi.plan.TableScanNode) TableStatisticsMetadata(io.prestosql.spi.statistics.TableStatisticsMetadata) Set(java.util.Set) NullLiteral(io.prestosql.sql.tree.NullLiteral) PlanNode(io.prestosql.spi.plan.PlanNode) ProjectNode(io.prestosql.spi.plan.ProjectNode) Metadata(io.prestosql.metadata.Metadata) Insert(io.prestosql.sql.tree.Insert) NodeRef(io.prestosql.sql.tree.NodeRef) CachingCostProvider(io.prestosql.cost.CachingCostProvider) SymbolUtils.toSymbolReference(io.prestosql.sql.planner.SymbolUtils.toSymbolReference) ReuseExchangeOperator(io.prestosql.spi.operator.ReuseExchangeOperator) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) ExpressionUtils(io.prestosql.sql.ExpressionUtils) GenericLiteral(io.prestosql.sql.tree.GenericLiteral) VacuumTargetReference(io.prestosql.sql.planner.plan.TableWriterNode.VacuumTargetReference) StringLiteral(io.prestosql.sql.tree.StringLiteral) StatisticAggregations(io.prestosql.sql.planner.plan.StatisticAggregations) StatisticsWriterNode(io.prestosql.sql.planner.plan.StatisticsWriterNode) VacuumTableNode(io.prestosql.sql.planner.plan.VacuumTableNode) WriterTarget(io.prestosql.sql.planner.plan.TableWriterNode.WriterTarget) Field(io.prestosql.sql.analyzer.Field) OriginalExpressionUtils(io.prestosql.sql.relational.OriginalExpressionUtils) Analyze(io.prestosql.sql.tree.Analyze) FIXED_HASH_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.FIXED_HASH_DISTRIBUTION) TableMetadata(io.prestosql.metadata.TableMetadata) VacuumTable(io.prestosql.sql.tree.VacuumTable) CharType(io.prestosql.spi.type.CharType) Node(io.prestosql.sql.tree.Node) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) VARCHAR(io.prestosql.spi.type.VarcharType.VARCHAR) CreateTableAsSelect(io.prestosql.sql.tree.CreateTableAsSelect) BooleanLiteral(io.prestosql.sql.tree.BooleanLiteral) Session(io.prestosql.Session) ParsingUtil.createParsingOptions(io.prestosql.sql.ParsingUtil.createParsingOptions) Signature(io.prestosql.spi.function.Signature) DeleteNode(io.prestosql.sql.planner.plan.DeleteNode) TypeSignatureProvider.fromTypes(io.prestosql.sql.analyzer.TypeSignatureProvider.fromTypes) StatsProvider(io.prestosql.cost.StatsProvider) Query(io.prestosql.sql.tree.Query) Assignments(io.prestosql.spi.plan.Assignments) QUERY_REJECTED(io.prestosql.spi.StandardErrorCode.QUERY_REJECTED) ComparisonExpression(io.prestosql.sql.tree.ComparisonExpression) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) Explain(io.prestosql.sql.tree.Explain) MetadataUtil.toSchemaTableName(io.prestosql.metadata.MetadataUtil.toSchemaTableName) VARBINARY(io.prestosql.spi.type.VarbinaryType.VARBINARY) ValuesNode(io.prestosql.spi.plan.ValuesNode) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) LimitNode(io.prestosql.spi.plan.LimitNode) CubeMetadata(io.hetu.core.spi.cube.CubeMetadata) PlanOptimizer(io.prestosql.sql.planner.optimizations.PlanOptimizer) VarcharType(io.prestosql.spi.type.VarcharType) Expression(io.prestosql.sql.tree.Expression) StatisticAggregationsDescriptor(io.prestosql.sql.planner.plan.StatisticAggregationsDescriptor) QualifiedName(io.prestosql.sql.tree.QualifiedName) APPLY_ALL_RULES(io.prestosql.spi.plan.PlanNode.SkipOptRuleLevel.APPLY_ALL_RULES) SqlParser(io.prestosql.sql.parser.SqlParser) CachingStatsProvider(io.prestosql.cost.CachingStatsProvider) TableFinishNode(io.prestosql.sql.planner.plan.TableFinishNode) Type(io.prestosql.spi.type.Type) TypeCoercion(io.prestosql.type.TypeCoercion) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) StatsCalculator(io.prestosql.cost.StatsCalculator) NewTableLayout(io.prestosql.metadata.NewTableLayout) AggregationNode.singleGroupingSet(io.prestosql.spi.plan.AggregationNode.singleGroupingSet) PrestoException(io.prestosql.spi.PrestoException) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) CatalogName(io.prestosql.spi.connector.CatalogName) ROW_COUNT(io.prestosql.spi.statistics.TableStatisticType.ROW_COUNT) UUID(java.util.UUID) RelationType(io.prestosql.sql.analyzer.RelationType) CubeFinishNode(io.prestosql.sql.planner.plan.CubeFinishNode) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) OptimizerUtils(io.prestosql.utils.OptimizerUtils) Scope(io.prestosql.sql.analyzer.Scope) List(java.util.List) Entry(java.util.Map.Entry) Optional(java.util.Optional) Analysis(io.prestosql.sql.analyzer.Analysis) NOT_SUPPORTED(io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED) IfExpression(io.prestosql.sql.tree.IfExpression) StatsAndCosts(io.prestosql.cost.StatsAndCosts) HashMap(java.util.HashMap) RelationId(io.prestosql.sql.analyzer.RelationId) SimpleImmutableEntry(java.util.AbstractMap.SimpleImmutableEntry) NOT_FOUND(io.prestosql.spi.StandardErrorCode.NOT_FOUND) TableHandle(io.prestosql.spi.metadata.TableHandle) Function(java.util.function.Function) HashSet(java.util.HashSet) CubeStatus(io.hetu.core.spi.cube.CubeStatus) ImmutableList(com.google.common.collect.ImmutableList) FunctionCall(io.prestosql.sql.tree.FunctionCall) Verify.verify(com.google.common.base.Verify.verify) InsertReference(io.prestosql.sql.planner.plan.TableWriterNode.InsertReference) Objects.requireNonNull(java.util.Objects.requireNonNull) TableStatisticAggregation(io.prestosql.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation) Symbol(io.prestosql.spi.plan.Symbol) TableWriterNode(io.prestosql.sql.planner.plan.TableWriterNode) CubeUpdateMetadata(io.prestosql.spi.cube.CubeUpdateMetadata) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) UpdateNode(io.prestosql.sql.planner.plan.UpdateNode) LambdaArgumentDeclaration(io.prestosql.sql.tree.LambdaArgumentDeclaration) Update(io.prestosql.sql.tree.Update) InsertCube(io.prestosql.sql.tree.InsertCube) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) DISTRIBUTED_PLAN_SANITY_CHECKER(io.prestosql.sql.planner.sanity.PlanSanityChecker.DISTRIBUTED_PLAN_SANITY_CHECKER) ExplainAnalyzeNode(io.prestosql.sql.planner.plan.ExplainAnalyzeNode) Streams.zip(com.google.common.collect.Streams.zip) Cast(io.prestosql.sql.tree.Cast) TableStatisticsMetadata(io.prestosql.spi.statistics.TableStatisticsMetadata) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) Symbol(io.prestosql.spi.plan.Symbol) NewTableLayout(io.prestosql.metadata.NewTableLayout) CubeUpdateMetadata(io.prestosql.spi.cube.CubeUpdateMetadata) Assignments(io.prestosql.spi.plan.Assignments) PrestoException(io.prestosql.spi.PrestoException) CubeMetadata(io.hetu.core.spi.cube.CubeMetadata) InsertReference(io.prestosql.sql.planner.plan.TableWriterNode.InsertReference) Field(io.prestosql.sql.analyzer.Field) Identifier(io.prestosql.sql.tree.Identifier) RelationType(io.prestosql.sql.analyzer.RelationType) HashSet(java.util.HashSet) TableMetadata(io.prestosql.metadata.TableMetadata) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) CubeFinishNode(io.prestosql.sql.planner.plan.CubeFinishNode) CharType(io.prestosql.spi.type.CharType) VarcharType(io.prestosql.spi.type.VarcharType) Type(io.prestosql.spi.type.Type) RelationType(io.prestosql.sql.analyzer.RelationType) Scope(io.prestosql.sql.analyzer.Scope) ConstantExpression(io.prestosql.spi.relation.ConstantExpression) OriginalExpressionUtils.castToRowExpression(io.prestosql.sql.relational.OriginalExpressionUtils.castToRowExpression) ComparisonExpression(io.prestosql.sql.tree.ComparisonExpression) Expression(io.prestosql.sql.tree.Expression) IfExpression(io.prestosql.sql.tree.IfExpression) Analysis(io.prestosql.sql.analyzer.Analysis) ProjectNode(io.prestosql.spi.plan.ProjectNode) TableHandle(io.prestosql.spi.metadata.TableHandle) LongSupplier(java.util.function.LongSupplier) NullLiteral(io.prestosql.sql.tree.NullLiteral)

Example 3 with TableStatisticsMetadata

use of io.prestosql.spi.statistics.TableStatisticsMetadata in project hetu-core by openlookeng.

the class LogicalPlanner method createDeletePlan.

private RelationPlan createDeletePlan(Analysis analysis, Delete node) {
    TableHandle handle = analysis.getTableHandle(node.getTable());
    if (handle.getConnectorHandle().isDeleteAsInsertSupported()) {
        QueryPlanner.UpdateDeleteRelationPlan deletePlan = new QueryPlanner(analysis, planSymbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator), metadata, session, namedSubPlan, uniqueIdAllocator).planDeleteRowAsInsert(node);
        RelationPlan plan = deletePlan.getPlan();
        Optional<NewTableLayout> newTableLayout = metadata.getUpdateLayout(session, handle);
        TableMetadata tableMetadata = metadata.getTableMetadata(session, handle);
        String catalogName = handle.getCatalogName().getCatalogName();
        Optional<Expression> constraint = deletePlan.getPredicate().isPresent() ? Optional.of(OriginalExpressionUtils.castToExpression(deletePlan.getPredicate().get())) : Optional.empty();
        // Skip statistics collection for delete,
        // because stats collection for delete seems to corrupt existing statistics
        TableStatisticsMetadata statisticsMetadata = TableStatisticsMetadata.empty();
        return createTableWriterPlan(analysis, plan, new TableWriterNode.DeleteAsInsertReference(handle, constraint, deletePlan.getColumnAssignments()), deletePlan.getColumNames(), newTableLayout, statisticsMetadata);
    } else {
        DeleteNode deleteNode = new QueryPlanner(analysis, planSymbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator), metadata, session, namedSubPlan, uniqueIdAllocator).plan(node);
        TableFinishNode commitNode = new TableFinishNode(idAllocator.getNextId(), deleteNode, deleteNode.getTarget(), planSymbolAllocator.newSymbol("rows", BIGINT), Optional.empty(), Optional.empty());
        return new RelationPlan(commitNode, analysis.getScope(node), commitNode.getOutputSymbols());
    }
}
Also used : TableMetadata(io.prestosql.metadata.TableMetadata) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) TableStatisticsMetadata(io.prestosql.spi.statistics.TableStatisticsMetadata) NewTableLayout(io.prestosql.metadata.NewTableLayout) TableFinishNode(io.prestosql.sql.planner.plan.TableFinishNode) DeleteNode(io.prestosql.sql.planner.plan.DeleteNode) ConstantExpression(io.prestosql.spi.relation.ConstantExpression) OriginalExpressionUtils.castToRowExpression(io.prestosql.sql.relational.OriginalExpressionUtils.castToRowExpression) ComparisonExpression(io.prestosql.sql.tree.ComparisonExpression) Expression(io.prestosql.sql.tree.Expression) IfExpression(io.prestosql.sql.tree.IfExpression) TableHandle(io.prestosql.spi.metadata.TableHandle) TableWriterNode(io.prestosql.sql.planner.plan.TableWriterNode)

Example 4 with TableStatisticsMetadata

use of io.prestosql.spi.statistics.TableStatisticsMetadata in project hetu-core by openlookeng.

the class LogicalPlanner method createAnalyzePlan.

private RelationPlan createAnalyzePlan(Analysis analysis, Analyze analyzeStatement) {
    TableHandle targetTable = analysis.getAnalyzeTarget().get();
    // Plan table scan
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, targetTable);
    ImmutableList.Builder<Symbol> tableScanOutputs = ImmutableList.builder();
    ImmutableMap.Builder<Symbol, ColumnHandle> symbolToColumnHandle = ImmutableMap.builder();
    ImmutableMap.Builder<String, Symbol> columnNameToSymbol = ImmutableMap.builder();
    TableMetadata tableMetadata = metadata.getTableMetadata(session, targetTable);
    for (ColumnMetadata column : tableMetadata.getColumns()) {
        Symbol symbol = planSymbolAllocator.newSymbol(column.getName(), column.getType());
        tableScanOutputs.add(symbol);
        symbolToColumnHandle.put(symbol, columnHandles.get(column.getName()));
        columnNameToSymbol.put(column.getName(), symbol);
    }
    TableStatisticsMetadata tableStatisticsMetadata = metadata.getStatisticsCollectionMetadata(session, targetTable.getCatalogName().getCatalogName(), tableMetadata.getMetadata());
    TableStatisticAggregation tableStatisticAggregation = statisticsAggregationPlanner.createStatisticsAggregation(tableStatisticsMetadata, columnNameToSymbol.build());
    StatisticAggregations statisticAggregations = tableStatisticAggregation.getAggregations();
    List<Symbol> groupingSymbols = statisticAggregations.getGroupingSymbols();
    PlanNode planNode = new StatisticsWriterNode(idAllocator.getNextId(), new AggregationNode(idAllocator.getNextId(), TableScanNode.newInstance(idAllocator.getNextId(), targetTable, tableScanOutputs.build(), symbolToColumnHandle.build(), ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), 0, false), statisticAggregations.getAggregations(), singleGroupingSet(groupingSymbols), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), Optional.empty(), AggregationNode.AggregationType.HASH, Optional.empty()), new StatisticsWriterNode.WriteStatisticsReference(targetTable), planSymbolAllocator.newSymbol("rows", BIGINT), tableStatisticsMetadata.getTableStatistics().contains(ROW_COUNT), tableStatisticAggregation.getDescriptor());
    return new RelationPlan(planNode, analysis.getScope(analyzeStatement), planNode.getOutputSymbols());
}
Also used : TableMetadata(io.prestosql.metadata.TableMetadata) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) TableStatisticsMetadata(io.prestosql.spi.statistics.TableStatisticsMetadata) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) TableStatisticAggregation(io.prestosql.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ImmutableList(com.google.common.collect.ImmutableList) Symbol(io.prestosql.spi.plan.Symbol) StatisticsWriterNode(io.prestosql.sql.planner.plan.StatisticsWriterNode) AggregationNode(io.prestosql.spi.plan.AggregationNode) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) ImmutableMap(com.google.common.collect.ImmutableMap) StatisticAggregations(io.prestosql.sql.planner.plan.StatisticAggregations) PlanNode(io.prestosql.spi.plan.PlanNode) TableHandle(io.prestosql.spi.metadata.TableHandle) UUID(java.util.UUID)

Example 5 with TableStatisticsMetadata

use of io.prestosql.spi.statistics.TableStatisticsMetadata in project hetu-core by openlookeng.

the class LogicalPlanner method createTableCreationPlan.

private RelationPlan createTableCreationPlan(Analysis analysis, Query query) {
    QualifiedObjectName destination = analysis.getCreateTableDestination().get();
    RelationPlan plan = createRelationPlan(analysis, query);
    ConnectorTableMetadata tableMetadata = createTableMetadata(destination, getOutputTableColumns(plan, analysis.getColumnAliases()), analysis.getCreateTableProperties(), analysis.getParameters(), analysis.getCreateTableComment());
    analysis.setCreateTableMetadata(tableMetadata);
    Optional<NewTableLayout> newTableLayout = metadata.getNewTableLayout(session, destination.getCatalogName(), tableMetadata);
    List<String> columnNames = tableMetadata.getColumns().stream().filter(column -> !column.isHidden()).map(ColumnMetadata::getName).collect(toImmutableList());
    TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, destination.getCatalogName(), tableMetadata);
    return createTableWriterPlan(analysis, plan, new CreateReference(destination.getCatalogName(), tableMetadata, newTableLayout), columnNames, newTableLayout, statisticsMetadata);
}
Also used : TableStatisticsMetadata(io.prestosql.spi.statistics.TableStatisticsMetadata) NewTableLayout(io.prestosql.metadata.NewTableLayout) CreateReference(io.prestosql.sql.planner.plan.TableWriterNode.CreateReference) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata)

Aggregations

ConnectorTableMetadata (io.prestosql.spi.connector.ConnectorTableMetadata)7 NewTableLayout (io.prestosql.metadata.NewTableLayout)6 TableMetadata (io.prestosql.metadata.TableMetadata)6 TableHandle (io.prestosql.spi.metadata.TableHandle)6 TableStatisticsMetadata (io.prestosql.spi.statistics.TableStatisticsMetadata)5 ImmutableList (com.google.common.collect.ImmutableList)4 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)4 ImmutableMap (com.google.common.collect.ImmutableMap)4 ImmutableMap.toImmutableMap (com.google.common.collect.ImmutableMap.toImmutableMap)4 ColumnHandle (io.prestosql.spi.connector.ColumnHandle)4 ColumnMetadata (io.prestosql.spi.connector.ColumnMetadata)4 QualifiedObjectName (io.prestosql.spi.connector.QualifiedObjectName)4 AggregationNode (io.prestosql.spi.plan.AggregationNode)4 Preconditions.checkState (com.google.common.base.Preconditions.checkState)3 Verify.verify (com.google.common.base.Verify.verify)3 ImmutableSet (com.google.common.collect.ImmutableSet)3 Streams.zip (com.google.common.collect.Streams.zip)3 CubeFilter (io.hetu.core.spi.cube.CubeFilter)3 CubeMetadata (io.hetu.core.spi.cube.CubeMetadata)3 CubeStatus (io.hetu.core.spi.cube.CubeStatus)3