Search in sources :

Example 21 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class DistributedExecutionPlanner method getTableInfo.

private TableInfo getTableInfo(TableScanNode node, Session session) {
    TableMetadata tableMetadata = metadata.getTableMetadata(session, node.getTable());
    TableProperties tableProperties = metadata.getTableProperties(session, node.getTable());
    return new TableInfo(tableMetadata.getQualifiedName(), tableProperties.getPredicate());
}
Also used : TableMetadata(io.prestosql.metadata.TableMetadata) TableInfo(io.prestosql.execution.TableInfo) TableProperties(io.prestosql.metadata.TableProperties)

Example 22 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class StatementAnalyzer method validateCreateIndex.

private void validateCreateIndex(Table table, Optional<Scope> scope) {
    CreateIndex createIndex = (CreateIndex) analysis.getOriginalStatement();
    QualifiedObjectName tableFullName = createQualifiedObjectName(session, createIndex, createIndex.getTableName());
    accessControl.checkCanCreateIndex(session.getRequiredTransactionId(), session.getIdentity(), tableFullName);
    String tableName = tableFullName.toString();
    // check whether catalog support create index
    if (!metadata.isHeuristicIndexSupported(session, tableFullName)) {
        throw new SemanticException(NOT_SUPPORTED, createIndex, "CREATE INDEX is not supported in catalog '%s'", tableFullName.getCatalogName());
    }
    List<String> partitions = new ArrayList<>();
    String partitionColumn = null;
    if (createIndex.getExpression().isPresent()) {
        partitions = HeuristicIndexUtils.extractPartitions(createIndex.getExpression().get());
        // check partition name validate, create index …… where pt_d = xxx;
        // pt_d must be partition column
        Set<String> partitionColumns = partitions.stream().map(k -> k.substring(0, k.indexOf("="))).collect(Collectors.toSet());
        if (partitionColumns.size() > 1) {
            // currently only support one partition column
            throw new IllegalArgumentException("Heuristic index only supports predicates on one column");
        }
        // The only entry in set should be the only partition column name
        partitionColumn = partitionColumns.iterator().next();
    }
    Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableFullName);
    if (tableHandle.isPresent()) {
        if (!tableHandle.get().getConnectorHandle().isHeuristicIndexSupported()) {
            throw new SemanticException(NOT_SUPPORTED, table, "Catalog supported, but table storage format is not supported by heuristic index");
        }
        TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle.get());
        List<String> availableColumns = tableMetadata.getColumns().stream().map(ColumnMetadata::getName).collect(Collectors.toList());
        for (Identifier column : createIndex.getColumnAliases()) {
            if (!availableColumns.contains(column.getValue().toLowerCase(Locale.ROOT))) {
                throw new SemanticException(MISSING_ATTRIBUTE, table, "Column '%s' cannot be resolved", column.getValue());
            }
        }
        if (partitionColumn != null && !tableHandle.get().getConnectorHandle().isPartitionColumn(partitionColumn)) {
            throw new SemanticException(NOT_SUPPORTED, table, "Heuristic index creation is only supported for predicates on partition columns");
        }
    } else {
        throw new SemanticException(MISSING_ATTRIBUTE, table, "Table '%s' is invalid", tableFullName);
    }
    List<Pair<String, Type>> indexColumns = new LinkedList<>();
    for (Identifier i : createIndex.getColumnAliases()) {
        indexColumns.add(new Pair<>(i.toString(), UNKNOWN));
    }
    // For now, creating index for multiple columns is not supported
    if (indexColumns.size() > 1) {
        throw new SemanticException(NOT_SUPPORTED, table, "Multi-column indexes are currently not supported");
    }
    try {
        // Use this place holder to check the existence of index and lock the place
        Properties properties = new Properties();
        properties.setProperty(INPROGRESS_PROPERTY_KEY, "TRUE");
        CreateIndexMetadata placeHolder = new CreateIndexMetadata(createIndex.getIndexName().toString(), tableName, createIndex.getIndexType(), 0L, indexColumns, partitions, properties, session.getUser(), UNDEFINED);
        synchronized (StatementAnalyzer.class) {
            IndexClient.RecordStatus recordStatus = heuristicIndexerManager.getIndexClient().lookUpIndexRecord(placeHolder);
            switch(recordStatus) {
                case SAME_NAME:
                    throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index '%s' already exists", createIndex.getIndexName().toString());
                case SAME_CONTENT:
                    throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index with same (table,column,indexType) already exists");
                case SAME_INDEX_PART_CONFLICT:
                    throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index with same (table,column,indexType) already exists and partition(s) contain conflicts");
                case IN_PROGRESS_SAME_NAME:
                    throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index '%s' is being created by another user. Check running queries for details. If there is no running query for this index, " + "the index may be in an unexpected error state and should be dropped using 'DROP INDEX %s'", createIndex.getIndexName().toString(), createIndex.getIndexName().toString());
                case IN_PROGRESS_SAME_CONTENT:
                    throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index with same (table,column,indexType) is being created by another user. Check running queries for details. " + "If there is no running query for this index, the index may be in an unexpected error state and should be dropped using 'DROP INDEX'");
                case IN_PROGRESS_SAME_INDEX_PART_CONFLICT:
                    if (partitions.isEmpty()) {
                        throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index with same (table,column,indexType) is being created by another user. Check running queries for details. " + "If there is no running query for this index, the index may be in an unexpected error state and should be dropped using 'DROP INDEX %s'", createIndex.getIndexName().toString());
                    }
                // allow different queries to run with explicitly same partitions
                case SAME_INDEX_PART_CAN_MERGE:
                case IN_PROGRESS_SAME_INDEX_PART_CAN_MERGE:
                    break;
                case NOT_FOUND:
                    heuristicIndexerManager.getIndexClient().addIndexRecord(placeHolder);
            }
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}
Also used : CreateSchema(io.prestosql.sql.tree.CreateSchema) AggregationAnalyzer.verifyOrderByAggregations(io.prestosql.sql.analyzer.AggregationAnalyzer.verifyOrderByAggregations) OperatorNotFoundException(io.prestosql.metadata.OperatorNotFoundException) INVALID_FUNCTION_ARGUMENT(io.prestosql.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT) Prepare(io.prestosql.sql.tree.Prepare) HeuristicIndexUtils(io.prestosql.utils.HeuristicIndexUtils) Statement(io.prestosql.sql.tree.Statement) INDEX_ALREADY_EXISTS(io.prestosql.sql.analyzer.SemanticErrorCode.INDEX_ALREADY_EXISTS) WarningCollector(io.prestosql.execution.warnings.WarningCollector) Execute(io.prestosql.sql.tree.Execute) Map(java.util.Map) RowType(io.prestosql.spi.type.RowType) ArrayConstructor(io.prestosql.sql.tree.ArrayConstructor) FetchFirst(io.prestosql.sql.tree.FetchFirst) TOO_MANY_ARGUMENTS(io.prestosql.sql.analyzer.SemanticErrorCode.TOO_MANY_ARGUMENTS) ENGLISH(java.util.Locale.ENGLISH) Identifier(io.prestosql.sql.tree.Identifier) Cube(io.prestosql.sql.tree.Cube) RenameColumn(io.prestosql.sql.tree.RenameColumn) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) INPROGRESS_PROPERTY_KEY(io.prestosql.spi.heuristicindex.IndexRecord.INPROGRESS_PROPERTY_KEY) AccessControl(io.prestosql.security.AccessControl) Delete(io.prestosql.sql.tree.Delete) SystemSessionProperties.getMaxGroupingSets(io.prestosql.SystemSessionProperties.getMaxGroupingSets) GroupingElement(io.prestosql.sql.tree.GroupingElement) Collectors.joining(java.util.stream.Collectors.joining) Insert(io.prestosql.sql.tree.Insert) DropView(io.prestosql.sql.tree.DropView) ExpressionUtils(io.prestosql.sql.ExpressionUtils) ParsingException(io.prestosql.sql.parser.ParsingException) LongLiteral(io.prestosql.sql.tree.LongLiteral) Call(io.prestosql.sql.tree.Call) ScopeReferenceExtractor.hasReferencesToScope(io.prestosql.sql.analyzer.ScopeReferenceExtractor.hasReferencesToScope) SqlPath(io.prestosql.sql.SqlPath) NodeUtils.mapFromProperties(io.prestosql.sql.NodeUtils.mapFromProperties) Joiner(com.google.common.base.Joiner) ExpressionInterpreter.expressionOptimizer(io.prestosql.sql.planner.ExpressionInterpreter.expressionOptimizer) FunctionKind(io.prestosql.spi.function.FunctionKind) INVALID_WINDOW_FRAME(io.prestosql.sql.analyzer.SemanticErrorCode.INVALID_WINDOW_FRAME) ExpressionInterpreter(io.prestosql.sql.planner.ExpressionInterpreter) COLUMN_NAME_NOT_SPECIFIED(io.prestosql.sql.analyzer.SemanticErrorCode.COLUMN_NAME_NOT_SPECIFIED) VacuumTable(io.prestosql.sql.tree.VacuumTable) DropTable(io.prestosql.sql.tree.DropTable) AllColumns(io.prestosql.sql.tree.AllColumns) Node(io.prestosql.sql.tree.Node) ResetSession(io.prestosql.sql.tree.ResetSession) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) MISSING_SCHEMA(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_SCHEMA) OptionalLong(java.util.OptionalLong) Deallocate(io.prestosql.sql.tree.Deallocate) MISSING_INDEX(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_INDEX) INVALID_PROCEDURE_ARGUMENTS(io.prestosql.sql.analyzer.SemanticErrorCode.INVALID_PROCEDURE_ARGUMENTS) ParsingUtil.createParsingOptions(io.prestosql.sql.ParsingUtil.createParsingOptions) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) Comment(io.prestosql.sql.tree.Comment) SelectItem(io.prestosql.sql.tree.SelectItem) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) MISSING_COLUMN(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_COLUMN) RenameTable(io.prestosql.sql.tree.RenameTable) DUPLICATE_RELATION(io.prestosql.sql.analyzer.SemanticErrorCode.DUPLICATE_RELATION) Query(io.prestosql.sql.tree.Query) ComparisonExpression(io.prestosql.sql.tree.ComparisonExpression) IOException(java.io.IOException) INVALID_COLUMN_MASK(io.prestosql.spi.StandardErrorCode.INVALID_COLUMN_MASK) Lateral(io.prestosql.sql.tree.Lateral) AGGREGATE(io.prestosql.spi.function.FunctionKind.AGGREGATE) COLUMN_TYPE_UNKNOWN(io.prestosql.sql.analyzer.SemanticErrorCode.COLUMN_TYPE_UNKNOWN) Expression(io.prestosql.sql.tree.Expression) TableSubquery(io.prestosql.sql.tree.TableSubquery) SystemSessionProperties(io.prestosql.SystemSessionProperties) Intersect(io.prestosql.sql.tree.Intersect) Analyzer.verifyNoAggregateWindowOrGroupingFunctions(io.prestosql.sql.analyzer.Analyzer.verifyNoAggregateWindowOrGroupingFunctions) SqlParser(io.prestosql.sql.parser.SqlParser) CreateCube(io.prestosql.sql.tree.CreateCube) AMBIGUOUS_ATTRIBUTE(io.prestosql.sql.analyzer.SemanticErrorCode.AMBIGUOUS_ATTRIBUTE) FieldReference(io.prestosql.sql.tree.FieldReference) DropSchema(io.prestosql.sql.tree.DropSchema) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Locale(java.util.Locale) StartTransaction(io.prestosql.sql.tree.StartTransaction) NON_NUMERIC_SAMPLE_PERCENTAGE(io.prestosql.sql.analyzer.SemanticErrorCode.NON_NUMERIC_SAMPLE_PERCENTAGE) VIEW_IS_RECURSIVE(io.prestosql.sql.analyzer.SemanticErrorCode.VIEW_IS_RECURSIVE) BOOLEAN(io.prestosql.spi.type.BooleanType.BOOLEAN) Type(io.prestosql.spi.type.Type) QuerySpecification(io.prestosql.sql.tree.QuerySpecification) Except(io.prestosql.sql.tree.Except) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) CubeMetaStore(io.hetu.core.spi.cube.io.CubeMetaStore) PrestoException(io.prestosql.spi.PrestoException) ImmutableSet(com.google.common.collect.ImmutableSet) RANGE(io.prestosql.spi.sql.expression.Types.WindowFrameType.RANGE) Collection(java.util.Collection) CatalogName(io.prestosql.spi.connector.CatalogName) CubeAggregateFunction(io.hetu.core.spi.cube.CubeAggregateFunction) ExpressionTreeUtils.extractLocation(io.prestosql.sql.analyzer.ExpressionTreeUtils.extractLocation) Iterables.getLast(com.google.common.collect.Iterables.getLast) DropCache(io.prestosql.sql.tree.DropCache) Collectors(java.util.stream.Collectors) Pair(io.prestosql.spi.heuristicindex.Pair) Rollback(io.prestosql.sql.tree.Rollback) AstUtils(io.prestosql.sql.util.AstUtils) ExpressionAnalyzer.createConstantAnalyzer(io.prestosql.sql.analyzer.ExpressionAnalyzer.createConstantAnalyzer) SingleColumn(io.prestosql.sql.tree.SingleColumn) With(io.prestosql.sql.tree.With) ExplainType(io.prestosql.sql.tree.ExplainType) TypeSignature(io.prestosql.spi.type.TypeSignature) DropCube(io.prestosql.sql.tree.DropCube) DataCenterUtility(io.prestosql.connector.DataCenterUtility) UNKNOWN(io.prestosql.spi.type.UnknownType.UNKNOWN) INVALID_OFFSET_ROW_COUNT(io.prestosql.sql.analyzer.SemanticErrorCode.INVALID_OFFSET_ROW_COUNT) MISSING_CATALOG(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_CATALOG) TOO_MANY_GROUPING_SETS(io.prestosql.sql.analyzer.SemanticErrorCode.TOO_MANY_GROUPING_SETS) INVALID_ROW_FILTER(io.prestosql.spi.StandardErrorCode.INVALID_ROW_FILTER) ConnectorViewDefinition(io.prestosql.spi.connector.ConnectorViewDefinition) NOT_FOUND(io.prestosql.spi.StandardErrorCode.NOT_FOUND) TableHandle(io.prestosql.spi.metadata.TableHandle) ViewAccessControl(io.prestosql.security.ViewAccessControl) HashSet(java.util.HashSet) CubeStatus(io.hetu.core.spi.cube.CubeStatus) Values(io.prestosql.sql.tree.Values) ExpressionTreeUtils.extractWindowFunctions(io.prestosql.sql.analyzer.ExpressionTreeUtils.extractWindowFunctions) ImmutableList(com.google.common.collect.ImmutableList) FunctionCall(io.prestosql.sql.tree.FunctionCall) JoinUsing(io.prestosql.sql.tree.JoinUsing) ViewColumn(io.prestosql.spi.connector.ConnectorViewDefinition.ViewColumn) ExpressionTreeUtils.extractExpressions(io.prestosql.sql.analyzer.ExpressionTreeUtils.extractExpressions) Math.toIntExact(java.lang.Math.toIntExact) LinkedList(java.util.LinkedList) Limit(io.prestosql.sql.tree.Limit) DereferenceExpression(io.prestosql.sql.tree.DereferenceExpression) DUPLICATE_PROPERTY(io.prestosql.sql.analyzer.SemanticErrorCode.DUPLICATE_PROPERTY) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) SystemSessionProperties.isEnableStarTreeIndex(io.prestosql.SystemSessionProperties.isEnableStarTreeIndex) WithQuery(io.prestosql.sql.tree.WithQuery) Offset(io.prestosql.sql.tree.Offset) INVALID_ORDINAL(io.prestosql.sql.analyzer.SemanticErrorCode.INVALID_ORDINAL) DropIndex(io.prestosql.sql.tree.DropIndex) INVALID_FUNCTION_NAME(io.prestosql.sql.analyzer.SemanticErrorCode.INVALID_FUNCTION_NAME) Use(io.prestosql.sql.tree.Use) DISTRIBUTED(io.prestosql.sql.tree.ExplainType.Type.DISTRIBUTED) JoinOn(io.prestosql.sql.tree.JoinOn) TABLE_STATE_INCORRECT(io.prestosql.sql.analyzer.SemanticErrorCode.TABLE_STATE_INCORRECT) Table(io.prestosql.sql.tree.Table) SampledRelation(io.prestosql.sql.tree.SampledRelation) LongSupplier(java.util.function.LongSupplier) PrestoWarning(io.prestosql.spi.PrestoWarning) Relation(io.prestosql.sql.tree.Relation) TypeProvider(io.prestosql.sql.planner.TypeProvider) Property(io.prestosql.sql.tree.Property) AliasedRelation(io.prestosql.sql.tree.AliasedRelation) AccessDeniedException(io.prestosql.spi.security.AccessDeniedException) CreateTable(io.prestosql.sql.tree.CreateTable) INVALID_FETCH_FIRST_ROW_COUNT(io.prestosql.sql.analyzer.SemanticErrorCode.INVALID_FETCH_FIRST_ROW_COUNT) Join(io.prestosql.sql.tree.Join) MISMATCHED_COLUMN_ALIASES(io.prestosql.sql.analyzer.SemanticErrorCode.MISMATCHED_COLUMN_ALIASES) Row(io.prestosql.sql.tree.Row) AssignmentItem(io.prestosql.sql.tree.AssignmentItem) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) Identity(io.prestosql.spi.security.Identity) DUPLICATE_COLUMN_NAME(io.prestosql.sql.analyzer.SemanticErrorCode.DUPLICATE_COLUMN_NAME) Metadata(io.prestosql.metadata.Metadata) SetSession(io.prestosql.sql.tree.SetSession) NodeRef(io.prestosql.sql.tree.NodeRef) MISSING_CUBE(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_CUBE) UncheckedIOException(java.io.UncheckedIOException) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) CreateView(io.prestosql.sql.tree.CreateView) ExpressionTreeUtils.extractAggregateFunctions(io.prestosql.sql.analyzer.ExpressionTreeUtils.extractAggregateFunctions) DropColumn(io.prestosql.sql.tree.DropColumn) MoreLists.mappedCopy(io.prestosql.util.MoreLists.mappedCopy) StandardErrorCode(io.prestosql.spi.StandardErrorCode) STAR_TREE(io.prestosql.cube.CubeManager.STAR_TREE) Grant(io.prestosql.sql.tree.Grant) GroupingSets(io.prestosql.sql.tree.GroupingSets) INSERT_INTO_CUBE(io.prestosql.sql.analyzer.SemanticErrorCode.INSERT_INTO_CUBE) Analyze(io.prestosql.sql.tree.Analyze) Iterables(com.google.common.collect.Iterables) TableMetadata(io.prestosql.metadata.TableMetadata) MUST_BE_WINDOW_FUNCTION(io.prestosql.sql.analyzer.SemanticErrorCode.MUST_BE_WINDOW_FUNCTION) VIEW_PARSE_ERROR(io.prestosql.sql.analyzer.SemanticErrorCode.VIEW_PARSE_ERROR) CharType(io.prestosql.spi.type.CharType) TypeNotFoundException(io.prestosql.spi.type.TypeNotFoundException) NodeUtils.getSortItemsFromOrderBy(io.prestosql.sql.NodeUtils.getSortItemsFromOrderBy) ExpressionTreeRewriter(io.prestosql.sql.tree.ExpressionTreeRewriter) Types(io.prestosql.spi.sql.expression.Types) ArrayList(java.util.ArrayList) MapType(io.prestosql.spi.type.MapType) WILDCARD_WITHOUT_FROM(io.prestosql.sql.analyzer.SemanticErrorCode.WILDCARD_WITHOUT_FROM) PRECEDING(io.prestosql.spi.sql.expression.Types.FrameBoundType.PRECEDING) VARCHAR(io.prestosql.spi.type.VarcharType.VARCHAR) CreateTableAsSelect(io.prestosql.sql.tree.CreateTableAsSelect) FOLLOWING(io.prestosql.spi.sql.expression.Types.FrameBoundType.FOLLOWING) Session(io.prestosql.Session) ExpressionDeterminismEvaluator.isDeterministic(io.prestosql.sql.planner.ExpressionDeterminismEvaluator.isDeterministic) MISSING_ATTRIBUTE(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_ATTRIBUTE) CatalogSchemaName(io.prestosql.spi.connector.CatalogSchemaName) IndexRecord(io.prestosql.spi.heuristicindex.IndexRecord) REDUNDANT_ORDER_BY(io.prestosql.spi.connector.StandardWarningCode.REDUNDANT_ORDER_BY) VIEW_IS_STALE(io.prestosql.sql.analyzer.SemanticErrorCode.VIEW_IS_STALE) SetOperation(io.prestosql.sql.tree.SetOperation) Properties(java.util.Properties) WINDOW(io.prestosql.spi.function.FunctionKind.WINDOW) TYPE_MISMATCH(io.prestosql.sql.analyzer.SemanticErrorCode.TYPE_MISMATCH) Throwables.throwIfInstanceOf(com.google.common.base.Throwables.throwIfInstanceOf) CubeManager(io.prestosql.cube.CubeManager) Explain(io.prestosql.sql.tree.Explain) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) CreateIndexMetadata(io.prestosql.spi.connector.CreateIndexMetadata) MISMATCHED_SET_COLUMN_TYPES(io.prestosql.sql.analyzer.SemanticErrorCode.MISMATCHED_SET_COLUMN_TYPES) CubeMetadata(io.hetu.core.spi.cube.CubeMetadata) AddColumn(io.prestosql.sql.tree.AddColumn) VarcharType(io.prestosql.spi.type.VarcharType) JoinCriteria(io.prestosql.sql.tree.JoinCriteria) Unnest(io.prestosql.sql.tree.Unnest) CURRENT_ROW(io.prestosql.spi.sql.expression.Types.FrameBoundType.CURRENT_ROW) AggregationAnalyzer.verifySourceAggregations(io.prestosql.sql.analyzer.AggregationAnalyzer.verifySourceAggregations) AllowAllAccessControl(io.prestosql.security.AllowAllAccessControl) Iterables.transform(com.google.common.collect.Iterables.transform) UNBOUNDED_PRECEDING(io.prestosql.spi.sql.expression.Types.FrameBoundType.UNBOUNDED_PRECEDING) QualifiedName(io.prestosql.sql.tree.QualifiedName) FunctionProperty(io.prestosql.sql.tree.FunctionProperty) DefaultTraversalVisitor(io.prestosql.sql.tree.DefaultTraversalVisitor) NOT_SUPPORTED(io.prestosql.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED) RenameSchema(io.prestosql.sql.tree.RenameSchema) UNDEFINED(io.prestosql.spi.connector.CreateIndexMetadata.Level.UNDEFINED) Select(io.prestosql.sql.tree.Select) Rollup(io.prestosql.sql.tree.Rollup) WindowFrame(io.prestosql.sql.tree.WindowFrame) OperatorType(io.prestosql.spi.function.OperatorType) IndexClient(io.prestosql.spi.heuristicindex.IndexClient) Window(io.prestosql.sql.tree.Window) TABLE_ALREADY_EXISTS(io.prestosql.sql.analyzer.SemanticErrorCode.TABLE_ALREADY_EXISTS) TypeCoercion(io.prestosql.type.TypeCoercion) Commit(io.prestosql.sql.tree.Commit) UNBOUNDED_FOLLOWING(io.prestosql.spi.sql.expression.Types.FrameBoundType.UNBOUNDED_FOLLOWING) SymbolsExtractor(io.prestosql.sql.planner.SymbolsExtractor) ImmutableMap(com.google.common.collect.ImmutableMap) GroupingOperation(io.prestosql.sql.tree.GroupingOperation) Collections.emptyList(java.util.Collections.emptyList) ArrayType(io.prestosql.spi.type.ArrayType) CreateIndex(io.prestosql.sql.tree.CreateIndex) GroupBy(io.prestosql.sql.tree.GroupBy) ViewExpression(io.prestosql.spi.security.ViewExpression) NESTED_WINDOW(io.prestosql.sql.analyzer.SemanticErrorCode.NESTED_WINDOW) NaturalJoin(io.prestosql.sql.tree.NaturalJoin) Sets(com.google.common.collect.Sets) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) INVALID_LIMIT_ROW_COUNT(io.prestosql.sql.analyzer.SemanticErrorCode.INVALID_LIMIT_ROW_COUNT) NONDETERMINISTIC_ORDER_BY_EXPRESSION_WITH_SELECT_DISTINCT(io.prestosql.sql.analyzer.SemanticErrorCode.NONDETERMINISTIC_ORDER_BY_EXPRESSION_WITH_SELECT_DISTINCT) SimpleGroupBy(io.prestosql.sql.tree.SimpleGroupBy) Optional(java.util.Optional) FrameBound(io.prestosql.sql.tree.FrameBound) MetadataUtil.createQualifiedObjectName(io.prestosql.metadata.MetadataUtil.createQualifiedObjectName) UpdateIndex(io.prestosql.sql.tree.UpdateIndex) MISSING_ORDER_BY(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_ORDER_BY) CUBE_NOT_FOUND(io.prestosql.spi.connector.StandardWarningCode.CUBE_NOT_FOUND) ORDER_BY_MUST_BE_IN_SELECT(io.prestosql.sql.analyzer.SemanticErrorCode.ORDER_BY_MUST_BE_IN_SELECT) HashMap(java.util.HashMap) Multimap(com.google.common.collect.Multimap) Objects.requireNonNull(java.util.Objects.requireNonNull) ExpressionRewriter(io.prestosql.sql.tree.ExpressionRewriter) SortItem(io.prestosql.sql.tree.SortItem) VerifyException(com.google.common.base.VerifyException) Iterator(java.util.Iterator) OrderBy(io.prestosql.sql.tree.OrderBy) MISSING_TABLE(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_TABLE) VIEW_ANALYSIS_ERROR(io.prestosql.sql.analyzer.SemanticErrorCode.VIEW_ANALYSIS_ERROR) Update(io.prestosql.sql.tree.Update) InsertCube(io.prestosql.sql.tree.InsertCube) Revoke(io.prestosql.sql.tree.Revoke) TableMetadata(io.prestosql.metadata.TableMetadata) CreateIndexMetadata(io.prestosql.spi.connector.CreateIndexMetadata) IndexClient(io.prestosql.spi.heuristicindex.IndexClient) ArrayList(java.util.ArrayList) UncheckedIOException(java.io.UncheckedIOException) IOException(java.io.IOException) UncheckedIOException(java.io.UncheckedIOException) NodeUtils.mapFromProperties(io.prestosql.sql.NodeUtils.mapFromProperties) SystemSessionProperties(io.prestosql.SystemSessionProperties) Properties(java.util.Properties) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) MetadataUtil.createQualifiedObjectName(io.prestosql.metadata.MetadataUtil.createQualifiedObjectName) LinkedList(java.util.LinkedList) CreateIndex(io.prestosql.sql.tree.CreateIndex) Identifier(io.prestosql.sql.tree.Identifier) TableHandle(io.prestosql.spi.metadata.TableHandle) Pair(io.prestosql.spi.heuristicindex.Pair)

Example 23 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class AggregationRewriteWithCube method rewrite.

public PlanNode rewrite(AggregationNode originalAggregationNode, PlanNode filterNode) {
    QualifiedObjectName starTreeTableName = QualifiedObjectName.valueOf(cubeMetadata.getCubeName());
    TableHandle cubeTableHandle = metadata.getTableHandle(session, starTreeTableName).orElseThrow(() -> new CubeNotFoundException(starTreeTableName.toString()));
    Map<String, ColumnHandle> cubeColumnsMap = metadata.getColumnHandles(session, cubeTableHandle);
    TableMetadata cubeTableMetadata = metadata.getTableMetadata(session, cubeTableHandle);
    List<ColumnMetadata> cubeColumnMetadataList = cubeTableMetadata.getColumns();
    // Add group by
    List<Symbol> groupings = new ArrayList<>(originalAggregationNode.getGroupingKeys().size());
    for (Symbol symbol : originalAggregationNode.getGroupingKeys()) {
        Object column = symbolMappings.get(symbol.getName());
        if (column instanceof ColumnHandle) {
            groupings.add(new Symbol(((ColumnHandle) column).getColumnName()));
        }
    }
    Set<String> cubeGroups = cubeMetadata.getGroup();
    boolean exactGroupsMatch = false;
    if (groupings.size() == cubeGroups.size()) {
        exactGroupsMatch = groupings.stream().map(Symbol::getName).map(String::toLowerCase).allMatch(cubeGroups::contains);
    }
    CubeRewriteResult cubeRewriteResult = createScanNode(originalAggregationNode, filterNode, cubeTableHandle, cubeColumnsMap, cubeColumnMetadataList, exactGroupsMatch);
    PlanNode planNode = cubeRewriteResult.getTableScanNode();
    // Add filter node
    if (filterNode != null) {
        Expression expression = castToExpression(((FilterNode) filterNode).getPredicate());
        expression = rewriteExpression(expression, rewrittenMappings);
        planNode = new FilterNode(idAllocator.getNextId(), planNode, castToRowExpression(expression));
    }
    if (!exactGroupsMatch) {
        Map<Symbol, Symbol> cubeScanToAggOutputMap = new HashMap<>();
        // Rewrite AggregationNode using Cube table
        ImmutableMap.Builder<Symbol, AggregationNode.Aggregation> aggregationsBuilder = ImmutableMap.builder();
        for (CubeRewriteResult.AggregatorSource aggregatorSource : cubeRewriteResult.getAggregationColumns()) {
            ColumnHandle cubeColHandle = cubeRewriteResult.getTableScanNode().getAssignments().get(aggregatorSource.getScanSymbol());
            ColumnMetadata cubeColumnMetadata = cubeRewriteResult.getSymbolMetadataMap().get(aggregatorSource.getScanSymbol());
            Type type = cubeColumnMetadata.getType();
            AggregationSignature aggregationSignature = cubeMetadata.getAggregationSignature(cubeColumnMetadata.getName()).orElseThrow(() -> new ColumnNotFoundException(new SchemaTableName(starTreeTableName.getSchemaName(), starTreeTableName.getObjectName()), cubeColHandle.getColumnName()));
            String aggFunction = COUNT.getName().equals(aggregationSignature.getFunction()) ? "sum" : aggregationSignature.getFunction();
            SymbolReference argument = toSymbolReference(aggregatorSource.getScanSymbol());
            FunctionHandle functionHandle = metadata.getFunctionAndTypeManager().lookupFunction(aggFunction, TypeSignatureProvider.fromTypeSignatures(type.getTypeSignature()));
            cubeScanToAggOutputMap.put(aggregatorSource.getScanSymbol(), aggregatorSource.getOriginalAggSymbol());
            aggregationsBuilder.put(aggregatorSource.getOriginalAggSymbol(), new AggregationNode.Aggregation(new CallExpression(aggFunction, functionHandle, type, ImmutableList.of(OriginalExpressionUtils.castToRowExpression(argument))), ImmutableList.of(OriginalExpressionUtils.castToRowExpression(argument)), false, Optional.empty(), Optional.empty(), Optional.empty()));
        }
        List<Symbol> groupingKeys = originalAggregationNode.getGroupingKeys().stream().map(Symbol::getName).map(rewrittenMappings::get).collect(Collectors.toList());
        planNode = new AggregationNode(idAllocator.getNextId(), planNode, aggregationsBuilder.build(), singleGroupingSet(groupingKeys), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), Optional.empty(), AggregationNode.AggregationType.HASH, Optional.empty());
        AggregationNode aggNode = (AggregationNode) planNode;
        if (!cubeRewriteResult.getAvgAggregationColumns().isEmpty()) {
            if (!cubeRewriteResult.getComputeAvgDividingSumByCount()) {
                Map<Symbol, Expression> aggregateAssignments = new HashMap<>();
                for (CubeRewriteResult.AggregatorSource aggregatorSource : cubeRewriteResult.getAggregationColumns()) {
                    aggregateAssignments.put(aggregatorSource.getOriginalAggSymbol(), toSymbolReference(aggregatorSource.getScanSymbol()));
                }
                planNode = new ProjectNode(idAllocator.getNextId(), aggNode, new Assignments(aggregateAssignments.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> castToRowExpression(entry.getValue())))));
            } else {
                // If there was an AVG aggregation, map it to AVG = SUM/COUNT
                Map<Symbol, Expression> projections = new HashMap<>();
                aggNode.getGroupingKeys().forEach(symbol -> projections.put(symbol, toSymbolReference(symbol)));
                aggNode.getAggregations().keySet().stream().filter(symbol -> symbolMappings.containsValue(symbol.getName())).forEach(aggSymbol -> projections.put(aggSymbol, toSymbolReference(aggSymbol)));
                // Add AVG = SUM / COUNT
                for (CubeRewriteResult.AverageAggregatorSource avgAggSource : cubeRewriteResult.getAvgAggregationColumns()) {
                    Symbol sumSymbol = cubeScanToAggOutputMap.get(avgAggSource.getSum());
                    Symbol countSymbol = cubeScanToAggOutputMap.get(avgAggSource.getCount());
                    Type avgResultType = typeProvider.get(avgAggSource.getOriginalAggSymbol());
                    ArithmeticBinaryExpression division = new ArithmeticBinaryExpression(ArithmeticBinaryExpression.Operator.DIVIDE, new Cast(toSymbolReference(sumSymbol), avgResultType.getTypeSignature().toString()), new Cast(toSymbolReference(countSymbol), avgResultType.getTypeSignature().toString()));
                    projections.put(avgAggSource.getOriginalAggSymbol(), division);
                }
                planNode = new ProjectNode(idAllocator.getNextId(), aggNode, new Assignments(projections.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> castToRowExpression(entry.getValue())))));
            }
        }
    }
    // Safety check to remove redundant symbols and rename original column names to intermediate names
    if (!planNode.getOutputSymbols().equals(originalAggregationNode.getOutputSymbols())) {
        // Map new symbol names to the old symbols
        Map<Symbol, Expression> assignments = new HashMap<>();
        Set<Symbol> planNodeOutput = new HashSet<>(planNode.getOutputSymbols());
        for (Symbol originalAggOutputSymbol : originalAggregationNode.getOutputSymbols()) {
            if (!planNodeOutput.contains(originalAggOutputSymbol)) {
                // Must be grouping key
                assignments.put(originalAggOutputSymbol, toSymbolReference(rewrittenMappings.get(originalAggOutputSymbol.getName())));
            } else {
                // Should be an expression and must have the same name in the new plan node
                assignments.put(originalAggOutputSymbol, toSymbolReference(originalAggOutputSymbol));
            }
        }
        planNode = new ProjectNode(idAllocator.getNextId(), planNode, new Assignments(assignments.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> castToRowExpression(entry.getValue())))));
    }
    return planNode;
}
Also used : TypeProvider(io.prestosql.sql.planner.TypeProvider) ColumnNotFoundException(io.prestosql.spi.connector.ColumnNotFoundException) AggregationNode(io.prestosql.spi.plan.AggregationNode) CallExpression(io.prestosql.spi.relation.CallExpression) Cast(io.prestosql.sql.tree.Cast) FilterNode(io.prestosql.spi.plan.FilterNode) Map(java.util.Map) OriginalExpressionUtils.castToRowExpression(io.prestosql.sql.relational.OriginalExpressionUtils.castToRowExpression) Type(io.prestosql.spi.type.Type) OriginalExpressionUtils.castToExpression(io.prestosql.sql.relational.OriginalExpressionUtils.castToExpression) AggregationNode.singleGroupingSet(io.prestosql.spi.plan.AggregationNode.singleGroupingSet) PrestoException(io.prestosql.spi.PrestoException) SymbolsExtractor(io.prestosql.sql.planner.SymbolsExtractor) SUM(io.hetu.core.spi.cube.CubeAggregateFunction.SUM) ImmutableMap(com.google.common.collect.ImmutableMap) CubeAggregateFunction(io.hetu.core.spi.cube.CubeAggregateFunction) TableScanNode(io.prestosql.spi.plan.TableScanNode) Set(java.util.Set) PlanNode(io.prestosql.spi.plan.PlanNode) UUID(java.util.UUID) ProjectNode(io.prestosql.spi.plan.ProjectNode) Collectors(java.util.stream.Collectors) Metadata(io.prestosql.metadata.Metadata) FunctionHandle(io.prestosql.spi.function.FunctionHandle) PlanSymbolAllocator(io.prestosql.sql.planner.PlanSymbolAllocator) SymbolUtils.toSymbolReference(io.prestosql.sql.planner.SymbolUtils.toSymbolReference) ReuseExchangeOperator(io.prestosql.spi.operator.ReuseExchangeOperator) List(java.util.List) LongLiteral(io.prestosql.sql.tree.LongLiteral) SymbolReference(io.prestosql.sql.tree.SymbolReference) AggregationSignature(io.hetu.core.spi.cube.aggregator.AggregationSignature) Optional(java.util.Optional) StandardErrorCode(io.prestosql.spi.StandardErrorCode) OriginalExpressionUtils(io.prestosql.sql.relational.OriginalExpressionUtils) COUNT(io.hetu.core.spi.cube.CubeAggregateFunction.COUNT) TableMetadata(io.prestosql.metadata.TableMetadata) Logger(io.airlift.log.Logger) TypeSignatureProvider(io.prestosql.sql.analyzer.TypeSignatureProvider) HashMap(java.util.HashMap) TableHandle(io.prestosql.spi.metadata.TableHandle) Function(java.util.function.Function) ExpressionTreeRewriter(io.prestosql.sql.tree.ExpressionTreeRewriter) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) ArithmeticBinaryExpression(io.prestosql.sql.tree.ArithmeticBinaryExpression) ImmutableList(com.google.common.collect.ImmutableList) Session(io.prestosql.Session) ExpressionRewriter(io.prestosql.sql.tree.ExpressionRewriter) Symbol(io.prestosql.spi.plan.Symbol) Assignments(io.prestosql.spi.plan.Assignments) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) SymbolAllocator(io.prestosql.spi.SymbolAllocator) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) CUBE_ERROR(io.prestosql.spi.StandardErrorCode.CUBE_ERROR) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) CubeMetadata(io.hetu.core.spi.cube.CubeMetadata) CubeNotFoundException(io.prestosql.spi.connector.CubeNotFoundException) Comparator(java.util.Comparator) Expression(io.prestosql.sql.tree.Expression) ArithmeticBinaryExpression(io.prestosql.sql.tree.ArithmeticBinaryExpression) Cast(io.prestosql.sql.tree.Cast) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) HashMap(java.util.HashMap) Symbol(io.prestosql.spi.plan.Symbol) SymbolUtils.toSymbolReference(io.prestosql.sql.planner.SymbolUtils.toSymbolReference) SymbolReference(io.prestosql.sql.tree.SymbolReference) FilterNode(io.prestosql.spi.plan.FilterNode) ArrayList(java.util.ArrayList) Assignments(io.prestosql.spi.plan.Assignments) PlanNode(io.prestosql.spi.plan.PlanNode) CubeNotFoundException(io.prestosql.spi.connector.CubeNotFoundException) FunctionHandle(io.prestosql.spi.function.FunctionHandle) CallExpression(io.prestosql.spi.relation.CallExpression) HashSet(java.util.HashSet) TableMetadata(io.prestosql.metadata.TableMetadata) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) AggregationSignature(io.hetu.core.spi.cube.aggregator.AggregationSignature) AggregationNode(io.prestosql.spi.plan.AggregationNode) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) ImmutableMap(com.google.common.collect.ImmutableMap) Type(io.prestosql.spi.type.Type) ColumnNotFoundException(io.prestosql.spi.connector.ColumnNotFoundException) CallExpression(io.prestosql.spi.relation.CallExpression) OriginalExpressionUtils.castToRowExpression(io.prestosql.sql.relational.OriginalExpressionUtils.castToRowExpression) OriginalExpressionUtils.castToExpression(io.prestosql.sql.relational.OriginalExpressionUtils.castToExpression) ArithmeticBinaryExpression(io.prestosql.sql.tree.ArithmeticBinaryExpression) Expression(io.prestosql.sql.tree.Expression) TableHandle(io.prestosql.spi.metadata.TableHandle) ProjectNode(io.prestosql.spi.plan.ProjectNode)

Example 24 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class CreateCubeTask method internalExecute.

@VisibleForTesting
public ListenableFuture<?> internalExecute(CreateCube statement, Metadata metadata, AccessControl accessControl, Session session, QueryStateMachine stateMachine, List<Expression> parameters) {
    Optional<CubeMetaStore> optionalCubeMetaStore = cubeManager.getMetaStore(STAR_TREE);
    if (!optionalCubeMetaStore.isPresent()) {
        throw new RuntimeException("HetuMetaStore is not initialized");
    }
    QualifiedObjectName cubeName = createQualifiedObjectName(session, statement, statement.getCubeName());
    QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getSourceTableName());
    Optional<TableHandle> cubeHandle = metadata.getTableHandle(session, cubeName);
    Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName);
    if (optionalCubeMetaStore.get().getMetadataFromCubeName(cubeName.toString()).isPresent()) {
        if (!statement.isNotExists()) {
            throw new SemanticException(CUBE_ALREADY_EXISTS, statement, "Cube '%s' already exists", cubeName);
        }
        return immediateFuture(null);
    }
    if (cubeHandle.isPresent()) {
        if (!statement.isNotExists()) {
            throw new SemanticException(CUBE_OR_TABLE_ALREADY_EXISTS, statement, "Cube or Table '%s' already exists", cubeName);
        }
        return immediateFuture(null);
    }
    CatalogName catalogName = metadata.getCatalogHandle(session, cubeName.getCatalogName()).orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog not found: " + cubeName.getCatalogName()));
    if (!metadata.isPreAggregationSupported(session, catalogName)) {
        throw new PrestoException(StandardErrorCode.NOT_SUPPORTED, String.format("Cube cannot be created on catalog '%s'", catalogName.toString()));
    }
    if (!tableHandle.isPresent()) {
        throw new SemanticException(MISSING_TABLE, statement, "Table '%s' does not exist", tableName);
    }
    TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle.get());
    List<String> groupingSet = statement.getGroupingSet().stream().map(s -> s.getValue().toLowerCase(ENGLISH)).collect(Collectors.toList());
    Map<String, ColumnMetadata> sourceTableColumns = tableMetadata.getColumns().stream().collect(Collectors.toMap(ColumnMetadata::getName, col -> col));
    List<ColumnMetadata> cubeColumns = new ArrayList<>();
    Map<String, AggregationSignature> aggregations = new HashMap<>();
    Analysis analysis = analyzeStatement(statement, session, metadata, accessControl, parameters, stateMachine.getWarningCollector());
    Map<String, Field> fields = analysis.getOutputDescriptor().getAllFields().stream().collect(Collectors.toMap(col -> col.getName().map(String::toLowerCase).get(), col -> col));
    for (FunctionCall aggFunction : statement.getAggregations()) {
        String aggFunctionName = aggFunction.getName().toString().toLowerCase(ENGLISH);
        String argument = aggFunction.getArguments().isEmpty() || aggFunction.getArguments().get(0) instanceof LongLiteral ? null : ((Identifier) aggFunction.getArguments().get(0)).getValue().toLowerCase(ENGLISH);
        boolean distinct = aggFunction.isDistinct();
        String cubeColumnName = aggFunctionName + "_" + (argument == null ? "all" : argument) + (aggFunction.isDistinct() ? "_distinct" : "");
        CubeAggregateFunction cubeAggregateFunction = CubeAggregateFunction.valueOf(aggFunctionName.toUpperCase(ENGLISH));
        switch(cubeAggregateFunction) {
            case SUM:
                aggregations.put(cubeColumnName, AggregationSignature.sum(argument, distinct));
                break;
            case COUNT:
                AggregationSignature aggregationSignature = argument == null ? AggregationSignature.count() : AggregationSignature.count(argument, distinct);
                aggregations.put(cubeColumnName, aggregationSignature);
                break;
            case AVG:
                aggregations.put(cubeColumnName, AggregationSignature.avg(argument, distinct));
                break;
            case MAX:
                aggregations.put(cubeColumnName, AggregationSignature.max(argument, distinct));
                break;
            case MIN:
                aggregations.put(cubeColumnName, AggregationSignature.min(argument, distinct));
                break;
            default:
                throw new PrestoException(NOT_SUPPORTED, format("Unsupported aggregation function : %s", aggFunctionName));
        }
        Field tableField = fields.get(cubeColumnName);
        ColumnMetadata cubeCol = new ColumnMetadata(cubeColumnName, tableField.getType(), true, null, null, false, Collections.emptyMap());
        cubeColumns.add(cubeCol);
    }
    accessControl.checkCanCreateTable(session.getRequiredTransactionId(), session.getIdentity(), tableName);
    Map<String, Expression> sqlProperties = mapFromProperties(statement.getProperties());
    Map<String, Object> properties = metadata.getTablePropertyManager().getProperties(catalogName, cubeName.getCatalogName(), sqlProperties, session, metadata, parameters);
    if (properties.containsKey("partitioned_by")) {
        List<String> partitionCols = new ArrayList<>(((List<String>) properties.get("partitioned_by")));
        // put all partition columns at the end of the list
        groupingSet.removeAll(partitionCols);
        groupingSet.addAll(partitionCols);
    }
    for (String dimension : groupingSet) {
        if (!sourceTableColumns.containsKey(dimension)) {
            throw new SemanticException(MISSING_COLUMN, statement, "Column %s does not exist", dimension);
        }
        ColumnMetadata tableCol = sourceTableColumns.get(dimension);
        ColumnMetadata cubeCol = new ColumnMetadata(dimension, tableCol.getType(), tableCol.isNullable(), null, null, false, tableCol.getProperties());
        cubeColumns.add(cubeCol);
    }
    ConnectorTableMetadata cubeTableMetadata = new ConnectorTableMetadata(cubeName.asSchemaTableName(), ImmutableList.copyOf(cubeColumns), properties);
    try {
        metadata.createTable(session, cubeName.getCatalogName(), cubeTableMetadata, statement.isNotExists());
    } catch (PrestoException e) {
        // connectors are not required to handle the ignoreExisting flag
        if (!e.getErrorCode().equals(ALREADY_EXISTS.toErrorCode()) || !statement.isNotExists()) {
            throw e;
        }
    }
    CubeMetadataBuilder builder = optionalCubeMetaStore.get().getBuilder(cubeName.toString(), tableName.toString());
    groupingSet.forEach(dimension -> builder.addDimensionColumn(dimension, dimension));
    aggregations.forEach((column, aggregationSignature) -> builder.addAggregationColumn(column, aggregationSignature.getFunction(), aggregationSignature.getDimension(), aggregationSignature.isDistinct()));
    builder.addGroup(new HashSet<>(groupingSet));
    // Status and Table modified time will be updated on the first insert into the cube
    builder.setCubeStatus(CubeStatus.INACTIVE);
    builder.setTableLastUpdatedTime(-1L);
    statement.getSourceFilter().ifPresent(sourceTablePredicate -> {
        sourceTablePredicate = Coercer.addCoercions(sourceTablePredicate, analysis);
        builder.withCubeFilter(new CubeFilter(ExpressionFormatter.formatExpression(sourceTablePredicate, Optional.empty())));
    });
    builder.setCubeLastUpdatedTime(System.currentTimeMillis());
    optionalCubeMetaStore.get().persist(builder.build());
    return immediateFuture(null);
}
Also used : SqlParser(io.prestosql.sql.parser.SqlParser) CreateCube(io.prestosql.sql.tree.CreateCube) Statement(io.prestosql.sql.tree.Statement) CUBE_OR_TABLE_ALREADY_EXISTS(io.prestosql.sql.analyzer.SemanticErrorCode.CUBE_OR_TABLE_ALREADY_EXISTS) WarningCollector(io.prestosql.execution.warnings.WarningCollector) ExpressionFormatter(io.prestosql.sql.ExpressionFormatter) Map(java.util.Map) CubeFilter(io.hetu.core.spi.cube.CubeFilter) ENGLISH(java.util.Locale.ENGLISH) Coercer(io.prestosql.sql.planner.Coercer) Identifier(io.prestosql.sql.tree.Identifier) VisibleForTesting(org.assertj.core.util.VisibleForTesting) CubeMetaStore(io.hetu.core.spi.cube.io.CubeMetaStore) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) PrestoException(io.prestosql.spi.PrestoException) AccessControl(io.prestosql.security.AccessControl) CatalogName(io.prestosql.spi.connector.CatalogName) CubeAggregateFunction(io.hetu.core.spi.cube.CubeAggregateFunction) Collectors(java.util.stream.Collectors) Metadata(io.prestosql.metadata.Metadata) String.format(java.lang.String.format) List(java.util.List) LongLiteral(io.prestosql.sql.tree.LongLiteral) AggregationSignature(io.hetu.core.spi.cube.aggregator.AggregationSignature) Optional(java.util.Optional) Analysis(io.prestosql.sql.analyzer.Analysis) NOT_SUPPORTED(io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED) StandardErrorCode(io.prestosql.spi.StandardErrorCode) STAR_TREE(io.prestosql.cube.CubeManager.STAR_TREE) CubeMetadataBuilder(io.hetu.core.spi.cube.CubeMetadataBuilder) Field(io.prestosql.sql.analyzer.Field) MetadataUtil.createQualifiedObjectName(io.prestosql.metadata.MetadataUtil.createQualifiedObjectName) NodeUtils.mapFromProperties(io.prestosql.sql.NodeUtils.mapFromProperties) ALREADY_EXISTS(io.prestosql.spi.StandardErrorCode.ALREADY_EXISTS) CUBE_ALREADY_EXISTS(io.prestosql.sql.analyzer.SemanticErrorCode.CUBE_ALREADY_EXISTS) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) TableMetadata(io.prestosql.metadata.TableMetadata) TransactionManager(io.prestosql.transaction.TransactionManager) HashMap(java.util.HashMap) NOT_FOUND(io.prestosql.spi.StandardErrorCode.NOT_FOUND) TableHandle(io.prestosql.spi.metadata.TableHandle) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) HashSet(java.util.HashSet) CubeStatus(io.hetu.core.spi.cube.CubeStatus) SemanticException(io.prestosql.sql.analyzer.SemanticException) ImmutableList(com.google.common.collect.ImmutableList) FunctionCall(io.prestosql.sql.tree.FunctionCall) Session(io.prestosql.Session) MISSING_COLUMN(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_COLUMN) Futures.immediateFuture(com.google.common.util.concurrent.Futures.immediateFuture) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) MISSING_TABLE(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_TABLE) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) Analyzer(io.prestosql.sql.analyzer.Analyzer) CubeManager(io.prestosql.cube.CubeManager) Collections(java.util.Collections) Expression(io.prestosql.sql.tree.Expression) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) HashMap(java.util.HashMap) CubeAggregateFunction(io.hetu.core.spi.cube.CubeAggregateFunction) ArrayList(java.util.ArrayList) CubeMetaStore(io.hetu.core.spi.cube.io.CubeMetaStore) PrestoException(io.prestosql.spi.PrestoException) Field(io.prestosql.sql.analyzer.Field) Identifier(io.prestosql.sql.tree.Identifier) CubeMetadataBuilder(io.hetu.core.spi.cube.CubeMetadataBuilder) CubeFilter(io.hetu.core.spi.cube.CubeFilter) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) FunctionCall(io.prestosql.sql.tree.FunctionCall) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) SemanticException(io.prestosql.sql.analyzer.SemanticException) TableMetadata(io.prestosql.metadata.TableMetadata) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) LongLiteral(io.prestosql.sql.tree.LongLiteral) AggregationSignature(io.hetu.core.spi.cube.aggregator.AggregationSignature) MetadataUtil.createQualifiedObjectName(io.prestosql.metadata.MetadataUtil.createQualifiedObjectName) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) Expression(io.prestosql.sql.tree.Expression) Analysis(io.prestosql.sql.analyzer.Analysis) TableHandle(io.prestosql.spi.metadata.TableHandle) CatalogName(io.prestosql.spi.connector.CatalogName) VisibleForTesting(org.assertj.core.util.VisibleForTesting)

Example 25 with TableMetadata

use of io.prestosql.metadata.TableMetadata in project hetu-core by openlookeng.

the class CreateTableTask method internalExecute.

@VisibleForTesting
public ListenableFuture<?> internalExecute(CreateTable statement, Metadata metadata, AccessControl accessControl, Session session, List<Expression> parameters) {
    checkArgument(!statement.getElements().isEmpty(), "no columns for table");
    QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getName());
    Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName);
    if (tableHandle.isPresent()) {
        if (!statement.isNotExists()) {
            throw new SemanticException(TABLE_ALREADY_EXISTS, statement, "Table '%s' already exists", tableName);
        }
        return immediateFuture(null);
    }
    CatalogName catalogName = metadata.getCatalogHandle(session, tableName.getCatalogName()).orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog does not exist: " + tableName.getCatalogName()));
    LinkedHashMap<String, ColumnMetadata> columns = new LinkedHashMap<>();
    Map<String, Object> inheritedProperties = ImmutableMap.of();
    boolean includingProperties = false;
    for (TableElement element : statement.getElements()) {
        if (element instanceof ColumnDefinition) {
            ColumnDefinition column = (ColumnDefinition) element;
            String name = column.getName().getValue().toLowerCase(Locale.ENGLISH);
            Type type;
            try {
                type = metadata.getType(parseTypeSignature(column.getType()));
            } catch (TypeNotFoundException e) {
                throw new SemanticException(TYPE_MISMATCH, element, "Unknown type '%s' for column '%s'", column.getType(), column.getName());
            }
            if (type.equals(UNKNOWN)) {
                throw new SemanticException(TYPE_MISMATCH, element, "Unknown type '%s' for column '%s'", column.getType(), column.getName());
            }
            if (columns.containsKey(name)) {
                throw new SemanticException(DUPLICATE_COLUMN_NAME, column, "Column name '%s' specified more than once", column.getName());
            }
            if (!column.isNullable() && !metadata.getConnectorCapabilities(session, catalogName).contains(NOT_NULL_COLUMN_CONSTRAINT)) {
                throw new SemanticException(NOT_SUPPORTED, column, "Catalog '%s' does not support non-null column for column name '%s'", catalogName.getCatalogName(), column.getName());
            }
            Map<String, Expression> sqlProperties = mapFromProperties(column.getProperties());
            Map<String, Object> columnProperties = metadata.getColumnPropertyManager().getProperties(catalogName, tableName.getCatalogName(), sqlProperties, session, metadata, parameters);
            columns.put(name, new ColumnMetadata(name, type, column.isNullable(), column.getComment().orElse(null), null, false, columnProperties));
        } else if (element instanceof LikeClause) {
            LikeClause likeClause = (LikeClause) element;
            QualifiedObjectName likeTableName = createQualifiedObjectName(session, statement, likeClause.getTableName());
            if (!metadata.getCatalogHandle(session, likeTableName.getCatalogName()).isPresent()) {
                throw new SemanticException(MISSING_CATALOG, statement, "LIKE table catalog '%s' does not exist", likeTableName.getCatalogName());
            }
            if (!tableName.getCatalogName().equals(likeTableName.getCatalogName())) {
                throw new SemanticException(NOT_SUPPORTED, statement, "LIKE table across catalogs is not supported");
            }
            TableHandle likeTable = metadata.getTableHandle(session, likeTableName).orElseThrow(() -> new SemanticException(MISSING_TABLE, statement, "LIKE table '%s' does not exist", likeTableName));
            TableMetadata likeTableMetadata = metadata.getTableMetadata(session, likeTable);
            Optional<LikeClause.PropertiesOption> propertiesOption = likeClause.getPropertiesOption();
            if (propertiesOption.isPresent() && propertiesOption.get().equals(LikeClause.PropertiesOption.INCLUDING)) {
                if (includingProperties) {
                    throw new SemanticException(NOT_SUPPORTED, statement, "Only one LIKE clause can specify INCLUDING PROPERTIES");
                }
                includingProperties = true;
                // Don't inherit location property for sql statement "create table like"
                inheritedProperties = likeTableMetadata.getMetadata().getInheritableProperties();
            }
            likeTableMetadata.getColumns().stream().filter(column -> !column.isHidden()).forEach(column -> {
                if (columns.containsKey(column.getName().toLowerCase(Locale.ENGLISH))) {
                    throw new SemanticException(DUPLICATE_COLUMN_NAME, element, "Column name '%s' specified more than once", column.getName());
                }
                columns.put(column.getName().toLowerCase(Locale.ENGLISH), column);
            });
        } else {
            throw new PrestoException(GENERIC_INTERNAL_ERROR, "Invalid TableElement: " + element.getClass().getName());
        }
    }
    accessControl.checkCanCreateTable(session.getRequiredTransactionId(), session.getIdentity(), tableName);
    Map<String, Expression> sqlProperties = mapFromProperties(statement.getProperties());
    Map<String, Object> properties = metadata.getTablePropertyManager().getProperties(catalogName, tableName.getCatalogName(), sqlProperties, session, metadata, parameters);
    Map<String, Object> finalProperties = combineProperties(sqlProperties.keySet(), properties, inheritedProperties);
    ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(toSchemaTableName(tableName), ImmutableList.copyOf(columns.values()), finalProperties, statement.getComment());
    try {
        metadata.createTable(session, tableName.getCatalogName(), tableMetadata, statement.isNotExists());
    } catch (PrestoException e) {
        // connectors are not required to handle the ignoreExisting flag
        if (!e.getErrorCode().equals(ALREADY_EXISTS.toErrorCode()) || !statement.isNotExists()) {
            throw e;
        }
    }
    return immediateFuture(null);
}
Also used : LikeClause(io.prestosql.sql.tree.LikeClause) TableMetadata(io.prestosql.metadata.TableMetadata) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) ALREADY_EXISTS(io.prestosql.spi.StandardErrorCode.ALREADY_EXISTS) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) TableMetadata(io.prestosql.metadata.TableMetadata) LikeClause(io.prestosql.sql.tree.LikeClause) TransactionManager(io.prestosql.transaction.TransactionManager) HashMap(java.util.HashMap) TypeNotFoundException(io.prestosql.spi.type.TypeNotFoundException) NOT_FOUND(io.prestosql.spi.StandardErrorCode.NOT_FOUND) TableHandle(io.prestosql.spi.metadata.TableHandle) NOT_SUPPORTED(io.prestosql.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) LinkedHashMap(java.util.LinkedHashMap) SemanticException(io.prestosql.sql.analyzer.SemanticException) TypeSignature.parseTypeSignature(io.prestosql.spi.type.TypeSignature.parseTypeSignature) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) CreateTable(io.prestosql.sql.tree.CreateTable) ImmutableList(com.google.common.collect.ImmutableList) Locale(java.util.Locale) Map(java.util.Map) Session(io.prestosql.Session) TABLE_ALREADY_EXISTS(io.prestosql.sql.analyzer.SemanticErrorCode.TABLE_ALREADY_EXISTS) Type(io.prestosql.spi.type.Type) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) PrestoException(io.prestosql.spi.PrestoException) Futures.immediateFuture(com.google.common.util.concurrent.Futures.immediateFuture) AccessControl(io.prestosql.security.AccessControl) ImmutableMap(com.google.common.collect.ImmutableMap) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) TYPE_MISMATCH(io.prestosql.sql.analyzer.SemanticErrorCode.TYPE_MISMATCH) CatalogName(io.prestosql.spi.connector.CatalogName) Set(java.util.Set) MISSING_TABLE(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_TABLE) DUPLICATE_COLUMN_NAME(io.prestosql.sql.analyzer.SemanticErrorCode.DUPLICATE_COLUMN_NAME) Metadata(io.prestosql.metadata.Metadata) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) TableElement(io.prestosql.sql.tree.TableElement) MetadataUtil.toSchemaTableName(io.prestosql.metadata.MetadataUtil.toSchemaTableName) List(java.util.List) ColumnDefinition(io.prestosql.sql.tree.ColumnDefinition) GENERIC_INTERNAL_ERROR(io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR) Optional(java.util.Optional) VisibleForTesting(com.google.common.annotations.VisibleForTesting) MetadataUtil.createQualifiedObjectName(io.prestosql.metadata.MetadataUtil.createQualifiedObjectName) NodeUtils.mapFromProperties(io.prestosql.sql.NodeUtils.mapFromProperties) Expression(io.prestosql.sql.tree.Expression) NOT_NULL_COLUMN_CONSTRAINT(io.prestosql.spi.connector.ConnectorCapabilities.NOT_NULL_COLUMN_CONSTRAINT) UNKNOWN(io.prestosql.spi.type.UnknownType.UNKNOWN) MISSING_CATALOG(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_CATALOG) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) Optional(java.util.Optional) PrestoException(io.prestosql.spi.PrestoException) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) MetadataUtil.createQualifiedObjectName(io.prestosql.metadata.MetadataUtil.createQualifiedObjectName) TableElement(io.prestosql.sql.tree.TableElement) LinkedHashMap(java.util.LinkedHashMap) ColumnDefinition(io.prestosql.sql.tree.ColumnDefinition) Type(io.prestosql.spi.type.Type) Expression(io.prestosql.sql.tree.Expression) TypeNotFoundException(io.prestosql.spi.type.TypeNotFoundException) TableHandle(io.prestosql.spi.metadata.TableHandle) CatalogName(io.prestosql.spi.connector.CatalogName) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) SemanticException(io.prestosql.sql.analyzer.SemanticException) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Aggregations

TableMetadata (io.prestosql.metadata.TableMetadata)93 Language (org.intellij.lang.annotations.Language)34 Test (org.testng.annotations.Test)33 AbstractTestIntegrationSmokeTest (io.prestosql.tests.AbstractTestIntegrationSmokeTest)32 ColumnMetadata (io.prestosql.spi.connector.ColumnMetadata)23 TableHandle (io.prestosql.spi.metadata.TableHandle)20 Session (io.prestosql.Session)18 Constraint (io.prestosql.spi.connector.Constraint)18 ColumnConstraint (io.prestosql.sql.planner.planprinter.IoPlanPrinter.ColumnConstraint)18 TableScanNode (io.prestosql.spi.plan.TableScanNode)15 Expression (io.prestosql.sql.tree.Expression)15 HashMap (java.util.HashMap)15 ColumnHandle (io.prestosql.spi.connector.ColumnHandle)14 PlanNode (io.prestosql.spi.plan.PlanNode)14 Symbol (io.prestosql.spi.plan.Symbol)14 Map (java.util.Map)14 ImmutableList (com.google.common.collect.ImmutableList)13 ProjectNode (io.prestosql.spi.plan.ProjectNode)13 ImmutableMap (com.google.common.collect.ImmutableMap)12 Metadata (io.prestosql.metadata.Metadata)12