Search in sources :

Example 21 with TableHandle

use of io.prestosql.spi.metadata.TableHandle in project hetu-core by openlookeng.

the class TestCostCalculator method tableScan.

private TableScanNode tableScan(String id, String... symbols) {
    List<Symbol> symbolsList = Arrays.stream(symbols).map(Symbol::new).collect(toImmutableList());
    ImmutableMap.Builder<Symbol, ColumnHandle> assignments = ImmutableMap.builder();
    for (Symbol symbol : symbolsList) {
        assignments.put(symbol, new TpchColumnHandle("orderkey", BIGINT));
    }
    TpchTableHandle tableHandle = new TpchTableHandle("orders", 1.0);
    return new TableScanNode(new PlanNodeId(id), new TableHandle(new CatalogName("tpch"), tableHandle, INSTANCE, Optional.of(new TpchTableLayoutHandle(tableHandle, TupleDomain.all()))), symbolsList, assignments.build(), TupleDomain.all(), Optional.empty(), ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), 0, false);
}
Also used : TpchColumnHandle(io.prestosql.plugin.tpch.TpchColumnHandle) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) TpchColumnHandle(io.prestosql.plugin.tpch.TpchColumnHandle) Symbol(io.prestosql.spi.plan.Symbol) TpchTableLayoutHandle(io.prestosql.plugin.tpch.TpchTableLayoutHandle) ImmutableMap(com.google.common.collect.ImmutableMap) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) TableScanNode(io.prestosql.spi.plan.TableScanNode) TpchTableHandle(io.prestosql.plugin.tpch.TpchTableHandle) TableHandle(io.prestosql.spi.metadata.TableHandle) CatalogName(io.prestosql.spi.connector.CatalogName) UUID(java.util.UUID) TpchTableHandle(io.prestosql.plugin.tpch.TpchTableHandle)

Example 22 with TableHandle

use of io.prestosql.spi.metadata.TableHandle in project hetu-core by openlookeng.

the class MetadataManager method applyFilter.

@Override
public Optional<ConstraintApplicationResult<TableHandle>> applyFilter(Session session, TableHandle table, Constraint constraint) {
    CatalogName catalogName = table.getCatalogName();
    ConnectorMetadata metadata = getMetadata(session, catalogName);
    if (metadata.usesLegacyTableLayouts()) {
        return Optional.empty();
    }
    ConnectorSession connectorSession = session.toConnectorSession(catalogName);
    return metadata.applyFilter(connectorSession, table.getConnectorHandle(), constraint).map(result -> new ConstraintApplicationResult<>(new TableHandle(catalogName, result.getHandle(), table.getTransaction(), Optional.empty()), result.getRemainingFilter()));
}
Also used : CatalogName(io.prestosql.spi.connector.CatalogName) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) ConnectorVacuumTableHandle(io.prestosql.spi.connector.ConnectorVacuumTableHandle) ConnectorDeleteAsInsertTableHandle(io.prestosql.spi.connector.ConnectorDeleteAsInsertTableHandle) ConnectorUpdateTableHandle(io.prestosql.spi.connector.ConnectorUpdateTableHandle) ConnectorOutputTableHandle(io.prestosql.spi.connector.ConnectorOutputTableHandle) TableHandle(io.prestosql.spi.metadata.TableHandle) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) ConnectorInsertTableHandle(io.prestosql.spi.connector.ConnectorInsertTableHandle) ConnectorMetadata(io.prestosql.spi.connector.ConnectorMetadata)

Example 23 with TableHandle

use of io.prestosql.spi.metadata.TableHandle in project hetu-core by openlookeng.

the class MetadataManager method getLayout.

@Override
public Optional<TableLayoutResult> getLayout(Session session, TableHandle table, Constraint constraint, Optional<Set<ColumnHandle>> desiredColumns) {
    if (constraint.getSummary().isNone()) {
        return Optional.empty();
    }
    CatalogName catalogName = table.getCatalogName();
    ConnectorTableHandle connectorTable = table.getConnectorHandle();
    CatalogMetadata catalogMetadata = getCatalogMetadata(session, catalogName);
    ConnectorMetadata metadata = catalogMetadata.getMetadataFor(catalogName);
    checkState(metadata.usesLegacyTableLayouts(), "getLayout() was called even though connector doesn't support legacy Table Layout");
    ConnectorTransactionHandle transaction = catalogMetadata.getTransactionHandleFor(catalogName);
    ConnectorSession connectorSession = session.toConnectorSession(catalogName);
    List<ConnectorTableLayoutResult> layouts = metadata.getTableLayouts(connectorSession, connectorTable, constraint, desiredColumns);
    if (layouts.isEmpty()) {
        return Optional.empty();
    }
    if (layouts.size() > 1) {
        throw new PrestoException(NOT_SUPPORTED, format("Connector returned multiple layouts for table %s", table));
    }
    ConnectorTableLayout tableLayout = layouts.get(0).getTableLayout();
    return Optional.of(new TableLayoutResult(new TableHandle(catalogName, connectorTable, transaction, Optional.of(tableLayout.getHandle())), new TableProperties(catalogName, transaction, new ConnectorTableProperties(tableLayout)), layouts.get(0).getUnenforcedConstraint()));
}
Also used : ConnectorTransactionHandle(io.prestosql.spi.connector.ConnectorTransactionHandle) PrestoException(io.prestosql.spi.PrestoException) ConnectorTableLayoutResult(io.prestosql.spi.connector.ConnectorTableLayoutResult) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) ConnectorTableLayoutResult(io.prestosql.spi.connector.ConnectorTableLayoutResult) ConnectorTableLayout(io.prestosql.spi.connector.ConnectorTableLayout) CatalogName(io.prestosql.spi.connector.CatalogName) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) ConnectorVacuumTableHandle(io.prestosql.spi.connector.ConnectorVacuumTableHandle) ConnectorDeleteAsInsertTableHandle(io.prestosql.spi.connector.ConnectorDeleteAsInsertTableHandle) ConnectorUpdateTableHandle(io.prestosql.spi.connector.ConnectorUpdateTableHandle) ConnectorOutputTableHandle(io.prestosql.spi.connector.ConnectorOutputTableHandle) TableHandle(io.prestosql.spi.metadata.TableHandle) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) ConnectorInsertTableHandle(io.prestosql.spi.connector.ConnectorInsertTableHandle) ConnectorMetadata(io.prestosql.spi.connector.ConnectorMetadata) ConnectorTableProperties(io.prestosql.spi.connector.ConnectorTableProperties) ConnectorTableProperties(io.prestosql.spi.connector.ConnectorTableProperties)

Example 24 with TableHandle

use of io.prestosql.spi.metadata.TableHandle in project hetu-core by openlookeng.

the class StatementAnalyzer method validateCreateIndex.

private void validateCreateIndex(Table table, Optional<Scope> scope) {
    CreateIndex createIndex = (CreateIndex) analysis.getOriginalStatement();
    QualifiedObjectName tableFullName = createQualifiedObjectName(session, createIndex, createIndex.getTableName());
    accessControl.checkCanCreateIndex(session.getRequiredTransactionId(), session.getIdentity(), tableFullName);
    String tableName = tableFullName.toString();
    // check whether catalog support create index
    if (!metadata.isHeuristicIndexSupported(session, tableFullName)) {
        throw new SemanticException(NOT_SUPPORTED, createIndex, "CREATE INDEX is not supported in catalog '%s'", tableFullName.getCatalogName());
    }
    List<String> partitions = new ArrayList<>();
    String partitionColumn = null;
    if (createIndex.getExpression().isPresent()) {
        partitions = HeuristicIndexUtils.extractPartitions(createIndex.getExpression().get());
        // check partition name validate, create index …… where pt_d = xxx;
        // pt_d must be partition column
        Set<String> partitionColumns = partitions.stream().map(k -> k.substring(0, k.indexOf("="))).collect(Collectors.toSet());
        if (partitionColumns.size() > 1) {
            // currently only support one partition column
            throw new IllegalArgumentException("Heuristic index only supports predicates on one column");
        }
        // The only entry in set should be the only partition column name
        partitionColumn = partitionColumns.iterator().next();
    }
    Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableFullName);
    if (tableHandle.isPresent()) {
        if (!tableHandle.get().getConnectorHandle().isHeuristicIndexSupported()) {
            throw new SemanticException(NOT_SUPPORTED, table, "Catalog supported, but table storage format is not supported by heuristic index");
        }
        TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle.get());
        List<String> availableColumns = tableMetadata.getColumns().stream().map(ColumnMetadata::getName).collect(Collectors.toList());
        for (Identifier column : createIndex.getColumnAliases()) {
            if (!availableColumns.contains(column.getValue().toLowerCase(Locale.ROOT))) {
                throw new SemanticException(MISSING_ATTRIBUTE, table, "Column '%s' cannot be resolved", column.getValue());
            }
        }
        if (partitionColumn != null && !tableHandle.get().getConnectorHandle().isPartitionColumn(partitionColumn)) {
            throw new SemanticException(NOT_SUPPORTED, table, "Heuristic index creation is only supported for predicates on partition columns");
        }
    } else {
        throw new SemanticException(MISSING_ATTRIBUTE, table, "Table '%s' is invalid", tableFullName);
    }
    List<Pair<String, Type>> indexColumns = new LinkedList<>();
    for (Identifier i : createIndex.getColumnAliases()) {
        indexColumns.add(new Pair<>(i.toString(), UNKNOWN));
    }
    // For now, creating index for multiple columns is not supported
    if (indexColumns.size() > 1) {
        throw new SemanticException(NOT_SUPPORTED, table, "Multi-column indexes are currently not supported");
    }
    try {
        // Use this place holder to check the existence of index and lock the place
        Properties properties = new Properties();
        properties.setProperty(INPROGRESS_PROPERTY_KEY, "TRUE");
        CreateIndexMetadata placeHolder = new CreateIndexMetadata(createIndex.getIndexName().toString(), tableName, createIndex.getIndexType(), 0L, indexColumns, partitions, properties, session.getUser(), UNDEFINED);
        synchronized (StatementAnalyzer.class) {
            IndexClient.RecordStatus recordStatus = heuristicIndexerManager.getIndexClient().lookUpIndexRecord(placeHolder);
            switch(recordStatus) {
                case SAME_NAME:
                    throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index '%s' already exists", createIndex.getIndexName().toString());
                case SAME_CONTENT:
                    throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index with same (table,column,indexType) already exists");
                case SAME_INDEX_PART_CONFLICT:
                    throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index with same (table,column,indexType) already exists and partition(s) contain conflicts");
                case IN_PROGRESS_SAME_NAME:
                    throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index '%s' is being created by another user. Check running queries for details. If there is no running query for this index, " + "the index may be in an unexpected error state and should be dropped using 'DROP INDEX %s'", createIndex.getIndexName().toString(), createIndex.getIndexName().toString());
                case IN_PROGRESS_SAME_CONTENT:
                    throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index with same (table,column,indexType) is being created by another user. Check running queries for details. " + "If there is no running query for this index, the index may be in an unexpected error state and should be dropped using 'DROP INDEX'");
                case IN_PROGRESS_SAME_INDEX_PART_CONFLICT:
                    if (partitions.isEmpty()) {
                        throw new SemanticException(INDEX_ALREADY_EXISTS, createIndex, "Index with same (table,column,indexType) is being created by another user. Check running queries for details. " + "If there is no running query for this index, the index may be in an unexpected error state and should be dropped using 'DROP INDEX %s'", createIndex.getIndexName().toString());
                    }
                // allow different queries to run with explicitly same partitions
                case SAME_INDEX_PART_CAN_MERGE:
                case IN_PROGRESS_SAME_INDEX_PART_CAN_MERGE:
                    break;
                case NOT_FOUND:
                    heuristicIndexerManager.getIndexClient().addIndexRecord(placeHolder);
            }
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}
Also used : CreateSchema(io.prestosql.sql.tree.CreateSchema) AggregationAnalyzer.verifyOrderByAggregations(io.prestosql.sql.analyzer.AggregationAnalyzer.verifyOrderByAggregations) OperatorNotFoundException(io.prestosql.metadata.OperatorNotFoundException) INVALID_FUNCTION_ARGUMENT(io.prestosql.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT) Prepare(io.prestosql.sql.tree.Prepare) HeuristicIndexUtils(io.prestosql.utils.HeuristicIndexUtils) Statement(io.prestosql.sql.tree.Statement) INDEX_ALREADY_EXISTS(io.prestosql.sql.analyzer.SemanticErrorCode.INDEX_ALREADY_EXISTS) WarningCollector(io.prestosql.execution.warnings.WarningCollector) Execute(io.prestosql.sql.tree.Execute) Map(java.util.Map) RowType(io.prestosql.spi.type.RowType) ArrayConstructor(io.prestosql.sql.tree.ArrayConstructor) FetchFirst(io.prestosql.sql.tree.FetchFirst) TOO_MANY_ARGUMENTS(io.prestosql.sql.analyzer.SemanticErrorCode.TOO_MANY_ARGUMENTS) ENGLISH(java.util.Locale.ENGLISH) Identifier(io.prestosql.sql.tree.Identifier) Cube(io.prestosql.sql.tree.Cube) RenameColumn(io.prestosql.sql.tree.RenameColumn) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) INPROGRESS_PROPERTY_KEY(io.prestosql.spi.heuristicindex.IndexRecord.INPROGRESS_PROPERTY_KEY) AccessControl(io.prestosql.security.AccessControl) Delete(io.prestosql.sql.tree.Delete) SystemSessionProperties.getMaxGroupingSets(io.prestosql.SystemSessionProperties.getMaxGroupingSets) GroupingElement(io.prestosql.sql.tree.GroupingElement) Collectors.joining(java.util.stream.Collectors.joining) Insert(io.prestosql.sql.tree.Insert) DropView(io.prestosql.sql.tree.DropView) ExpressionUtils(io.prestosql.sql.ExpressionUtils) ParsingException(io.prestosql.sql.parser.ParsingException) LongLiteral(io.prestosql.sql.tree.LongLiteral) Call(io.prestosql.sql.tree.Call) ScopeReferenceExtractor.hasReferencesToScope(io.prestosql.sql.analyzer.ScopeReferenceExtractor.hasReferencesToScope) SqlPath(io.prestosql.sql.SqlPath) NodeUtils.mapFromProperties(io.prestosql.sql.NodeUtils.mapFromProperties) Joiner(com.google.common.base.Joiner) ExpressionInterpreter.expressionOptimizer(io.prestosql.sql.planner.ExpressionInterpreter.expressionOptimizer) FunctionKind(io.prestosql.spi.function.FunctionKind) INVALID_WINDOW_FRAME(io.prestosql.sql.analyzer.SemanticErrorCode.INVALID_WINDOW_FRAME) ExpressionInterpreter(io.prestosql.sql.planner.ExpressionInterpreter) COLUMN_NAME_NOT_SPECIFIED(io.prestosql.sql.analyzer.SemanticErrorCode.COLUMN_NAME_NOT_SPECIFIED) VacuumTable(io.prestosql.sql.tree.VacuumTable) DropTable(io.prestosql.sql.tree.DropTable) AllColumns(io.prestosql.sql.tree.AllColumns) Node(io.prestosql.sql.tree.Node) ResetSession(io.prestosql.sql.tree.ResetSession) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) MISSING_SCHEMA(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_SCHEMA) OptionalLong(java.util.OptionalLong) Deallocate(io.prestosql.sql.tree.Deallocate) MISSING_INDEX(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_INDEX) INVALID_PROCEDURE_ARGUMENTS(io.prestosql.sql.analyzer.SemanticErrorCode.INVALID_PROCEDURE_ARGUMENTS) ParsingUtil.createParsingOptions(io.prestosql.sql.ParsingUtil.createParsingOptions) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) Comment(io.prestosql.sql.tree.Comment) SelectItem(io.prestosql.sql.tree.SelectItem) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) MISSING_COLUMN(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_COLUMN) RenameTable(io.prestosql.sql.tree.RenameTable) DUPLICATE_RELATION(io.prestosql.sql.analyzer.SemanticErrorCode.DUPLICATE_RELATION) Query(io.prestosql.sql.tree.Query) ComparisonExpression(io.prestosql.sql.tree.ComparisonExpression) IOException(java.io.IOException) INVALID_COLUMN_MASK(io.prestosql.spi.StandardErrorCode.INVALID_COLUMN_MASK) Lateral(io.prestosql.sql.tree.Lateral) AGGREGATE(io.prestosql.spi.function.FunctionKind.AGGREGATE) COLUMN_TYPE_UNKNOWN(io.prestosql.sql.analyzer.SemanticErrorCode.COLUMN_TYPE_UNKNOWN) Expression(io.prestosql.sql.tree.Expression) TableSubquery(io.prestosql.sql.tree.TableSubquery) SystemSessionProperties(io.prestosql.SystemSessionProperties) Intersect(io.prestosql.sql.tree.Intersect) Analyzer.verifyNoAggregateWindowOrGroupingFunctions(io.prestosql.sql.analyzer.Analyzer.verifyNoAggregateWindowOrGroupingFunctions) SqlParser(io.prestosql.sql.parser.SqlParser) CreateCube(io.prestosql.sql.tree.CreateCube) AMBIGUOUS_ATTRIBUTE(io.prestosql.sql.analyzer.SemanticErrorCode.AMBIGUOUS_ATTRIBUTE) FieldReference(io.prestosql.sql.tree.FieldReference) DropSchema(io.prestosql.sql.tree.DropSchema) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Locale(java.util.Locale) StartTransaction(io.prestosql.sql.tree.StartTransaction) NON_NUMERIC_SAMPLE_PERCENTAGE(io.prestosql.sql.analyzer.SemanticErrorCode.NON_NUMERIC_SAMPLE_PERCENTAGE) VIEW_IS_RECURSIVE(io.prestosql.sql.analyzer.SemanticErrorCode.VIEW_IS_RECURSIVE) BOOLEAN(io.prestosql.spi.type.BooleanType.BOOLEAN) Type(io.prestosql.spi.type.Type) QuerySpecification(io.prestosql.sql.tree.QuerySpecification) Except(io.prestosql.sql.tree.Except) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) CubeMetaStore(io.hetu.core.spi.cube.io.CubeMetaStore) PrestoException(io.prestosql.spi.PrestoException) ImmutableSet(com.google.common.collect.ImmutableSet) RANGE(io.prestosql.spi.sql.expression.Types.WindowFrameType.RANGE) Collection(java.util.Collection) CatalogName(io.prestosql.spi.connector.CatalogName) CubeAggregateFunction(io.hetu.core.spi.cube.CubeAggregateFunction) ExpressionTreeUtils.extractLocation(io.prestosql.sql.analyzer.ExpressionTreeUtils.extractLocation) Iterables.getLast(com.google.common.collect.Iterables.getLast) DropCache(io.prestosql.sql.tree.DropCache) Collectors(java.util.stream.Collectors) Pair(io.prestosql.spi.heuristicindex.Pair) Rollback(io.prestosql.sql.tree.Rollback) AstUtils(io.prestosql.sql.util.AstUtils) ExpressionAnalyzer.createConstantAnalyzer(io.prestosql.sql.analyzer.ExpressionAnalyzer.createConstantAnalyzer) SingleColumn(io.prestosql.sql.tree.SingleColumn) With(io.prestosql.sql.tree.With) ExplainType(io.prestosql.sql.tree.ExplainType) TypeSignature(io.prestosql.spi.type.TypeSignature) DropCube(io.prestosql.sql.tree.DropCube) DataCenterUtility(io.prestosql.connector.DataCenterUtility) UNKNOWN(io.prestosql.spi.type.UnknownType.UNKNOWN) INVALID_OFFSET_ROW_COUNT(io.prestosql.sql.analyzer.SemanticErrorCode.INVALID_OFFSET_ROW_COUNT) MISSING_CATALOG(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_CATALOG) TOO_MANY_GROUPING_SETS(io.prestosql.sql.analyzer.SemanticErrorCode.TOO_MANY_GROUPING_SETS) INVALID_ROW_FILTER(io.prestosql.spi.StandardErrorCode.INVALID_ROW_FILTER) ConnectorViewDefinition(io.prestosql.spi.connector.ConnectorViewDefinition) NOT_FOUND(io.prestosql.spi.StandardErrorCode.NOT_FOUND) TableHandle(io.prestosql.spi.metadata.TableHandle) ViewAccessControl(io.prestosql.security.ViewAccessControl) HashSet(java.util.HashSet) CubeStatus(io.hetu.core.spi.cube.CubeStatus) Values(io.prestosql.sql.tree.Values) ExpressionTreeUtils.extractWindowFunctions(io.prestosql.sql.analyzer.ExpressionTreeUtils.extractWindowFunctions) ImmutableList(com.google.common.collect.ImmutableList) FunctionCall(io.prestosql.sql.tree.FunctionCall) JoinUsing(io.prestosql.sql.tree.JoinUsing) ViewColumn(io.prestosql.spi.connector.ConnectorViewDefinition.ViewColumn) ExpressionTreeUtils.extractExpressions(io.prestosql.sql.analyzer.ExpressionTreeUtils.extractExpressions) Math.toIntExact(java.lang.Math.toIntExact) LinkedList(java.util.LinkedList) Limit(io.prestosql.sql.tree.Limit) DereferenceExpression(io.prestosql.sql.tree.DereferenceExpression) DUPLICATE_PROPERTY(io.prestosql.sql.analyzer.SemanticErrorCode.DUPLICATE_PROPERTY) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) SystemSessionProperties.isEnableStarTreeIndex(io.prestosql.SystemSessionProperties.isEnableStarTreeIndex) WithQuery(io.prestosql.sql.tree.WithQuery) Offset(io.prestosql.sql.tree.Offset) INVALID_ORDINAL(io.prestosql.sql.analyzer.SemanticErrorCode.INVALID_ORDINAL) DropIndex(io.prestosql.sql.tree.DropIndex) INVALID_FUNCTION_NAME(io.prestosql.sql.analyzer.SemanticErrorCode.INVALID_FUNCTION_NAME) Use(io.prestosql.sql.tree.Use) DISTRIBUTED(io.prestosql.sql.tree.ExplainType.Type.DISTRIBUTED) JoinOn(io.prestosql.sql.tree.JoinOn) TABLE_STATE_INCORRECT(io.prestosql.sql.analyzer.SemanticErrorCode.TABLE_STATE_INCORRECT) Table(io.prestosql.sql.tree.Table) SampledRelation(io.prestosql.sql.tree.SampledRelation) LongSupplier(java.util.function.LongSupplier) PrestoWarning(io.prestosql.spi.PrestoWarning) Relation(io.prestosql.sql.tree.Relation) TypeProvider(io.prestosql.sql.planner.TypeProvider) Property(io.prestosql.sql.tree.Property) AliasedRelation(io.prestosql.sql.tree.AliasedRelation) AccessDeniedException(io.prestosql.spi.security.AccessDeniedException) CreateTable(io.prestosql.sql.tree.CreateTable) INVALID_FETCH_FIRST_ROW_COUNT(io.prestosql.sql.analyzer.SemanticErrorCode.INVALID_FETCH_FIRST_ROW_COUNT) Join(io.prestosql.sql.tree.Join) MISMATCHED_COLUMN_ALIASES(io.prestosql.sql.analyzer.SemanticErrorCode.MISMATCHED_COLUMN_ALIASES) Row(io.prestosql.sql.tree.Row) AssignmentItem(io.prestosql.sql.tree.AssignmentItem) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) Identity(io.prestosql.spi.security.Identity) DUPLICATE_COLUMN_NAME(io.prestosql.sql.analyzer.SemanticErrorCode.DUPLICATE_COLUMN_NAME) Metadata(io.prestosql.metadata.Metadata) SetSession(io.prestosql.sql.tree.SetSession) NodeRef(io.prestosql.sql.tree.NodeRef) MISSING_CUBE(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_CUBE) UncheckedIOException(java.io.UncheckedIOException) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) CreateView(io.prestosql.sql.tree.CreateView) ExpressionTreeUtils.extractAggregateFunctions(io.prestosql.sql.analyzer.ExpressionTreeUtils.extractAggregateFunctions) DropColumn(io.prestosql.sql.tree.DropColumn) MoreLists.mappedCopy(io.prestosql.util.MoreLists.mappedCopy) StandardErrorCode(io.prestosql.spi.StandardErrorCode) STAR_TREE(io.prestosql.cube.CubeManager.STAR_TREE) Grant(io.prestosql.sql.tree.Grant) GroupingSets(io.prestosql.sql.tree.GroupingSets) INSERT_INTO_CUBE(io.prestosql.sql.analyzer.SemanticErrorCode.INSERT_INTO_CUBE) Analyze(io.prestosql.sql.tree.Analyze) Iterables(com.google.common.collect.Iterables) TableMetadata(io.prestosql.metadata.TableMetadata) MUST_BE_WINDOW_FUNCTION(io.prestosql.sql.analyzer.SemanticErrorCode.MUST_BE_WINDOW_FUNCTION) VIEW_PARSE_ERROR(io.prestosql.sql.analyzer.SemanticErrorCode.VIEW_PARSE_ERROR) CharType(io.prestosql.spi.type.CharType) TypeNotFoundException(io.prestosql.spi.type.TypeNotFoundException) NodeUtils.getSortItemsFromOrderBy(io.prestosql.sql.NodeUtils.getSortItemsFromOrderBy) ExpressionTreeRewriter(io.prestosql.sql.tree.ExpressionTreeRewriter) Types(io.prestosql.spi.sql.expression.Types) ArrayList(java.util.ArrayList) MapType(io.prestosql.spi.type.MapType) WILDCARD_WITHOUT_FROM(io.prestosql.sql.analyzer.SemanticErrorCode.WILDCARD_WITHOUT_FROM) PRECEDING(io.prestosql.spi.sql.expression.Types.FrameBoundType.PRECEDING) VARCHAR(io.prestosql.spi.type.VarcharType.VARCHAR) CreateTableAsSelect(io.prestosql.sql.tree.CreateTableAsSelect) FOLLOWING(io.prestosql.spi.sql.expression.Types.FrameBoundType.FOLLOWING) Session(io.prestosql.Session) ExpressionDeterminismEvaluator.isDeterministic(io.prestosql.sql.planner.ExpressionDeterminismEvaluator.isDeterministic) MISSING_ATTRIBUTE(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_ATTRIBUTE) CatalogSchemaName(io.prestosql.spi.connector.CatalogSchemaName) IndexRecord(io.prestosql.spi.heuristicindex.IndexRecord) REDUNDANT_ORDER_BY(io.prestosql.spi.connector.StandardWarningCode.REDUNDANT_ORDER_BY) VIEW_IS_STALE(io.prestosql.sql.analyzer.SemanticErrorCode.VIEW_IS_STALE) SetOperation(io.prestosql.sql.tree.SetOperation) Properties(java.util.Properties) WINDOW(io.prestosql.spi.function.FunctionKind.WINDOW) TYPE_MISMATCH(io.prestosql.sql.analyzer.SemanticErrorCode.TYPE_MISMATCH) Throwables.throwIfInstanceOf(com.google.common.base.Throwables.throwIfInstanceOf) CubeManager(io.prestosql.cube.CubeManager) Explain(io.prestosql.sql.tree.Explain) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) CreateIndexMetadata(io.prestosql.spi.connector.CreateIndexMetadata) MISMATCHED_SET_COLUMN_TYPES(io.prestosql.sql.analyzer.SemanticErrorCode.MISMATCHED_SET_COLUMN_TYPES) CubeMetadata(io.hetu.core.spi.cube.CubeMetadata) AddColumn(io.prestosql.sql.tree.AddColumn) VarcharType(io.prestosql.spi.type.VarcharType) JoinCriteria(io.prestosql.sql.tree.JoinCriteria) Unnest(io.prestosql.sql.tree.Unnest) CURRENT_ROW(io.prestosql.spi.sql.expression.Types.FrameBoundType.CURRENT_ROW) AggregationAnalyzer.verifySourceAggregations(io.prestosql.sql.analyzer.AggregationAnalyzer.verifySourceAggregations) AllowAllAccessControl(io.prestosql.security.AllowAllAccessControl) Iterables.transform(com.google.common.collect.Iterables.transform) UNBOUNDED_PRECEDING(io.prestosql.spi.sql.expression.Types.FrameBoundType.UNBOUNDED_PRECEDING) QualifiedName(io.prestosql.sql.tree.QualifiedName) FunctionProperty(io.prestosql.sql.tree.FunctionProperty) DefaultTraversalVisitor(io.prestosql.sql.tree.DefaultTraversalVisitor) NOT_SUPPORTED(io.prestosql.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED) RenameSchema(io.prestosql.sql.tree.RenameSchema) UNDEFINED(io.prestosql.spi.connector.CreateIndexMetadata.Level.UNDEFINED) Select(io.prestosql.sql.tree.Select) Rollup(io.prestosql.sql.tree.Rollup) WindowFrame(io.prestosql.sql.tree.WindowFrame) OperatorType(io.prestosql.spi.function.OperatorType) IndexClient(io.prestosql.spi.heuristicindex.IndexClient) Window(io.prestosql.sql.tree.Window) TABLE_ALREADY_EXISTS(io.prestosql.sql.analyzer.SemanticErrorCode.TABLE_ALREADY_EXISTS) TypeCoercion(io.prestosql.type.TypeCoercion) Commit(io.prestosql.sql.tree.Commit) UNBOUNDED_FOLLOWING(io.prestosql.spi.sql.expression.Types.FrameBoundType.UNBOUNDED_FOLLOWING) SymbolsExtractor(io.prestosql.sql.planner.SymbolsExtractor) ImmutableMap(com.google.common.collect.ImmutableMap) GroupingOperation(io.prestosql.sql.tree.GroupingOperation) Collections.emptyList(java.util.Collections.emptyList) ArrayType(io.prestosql.spi.type.ArrayType) CreateIndex(io.prestosql.sql.tree.CreateIndex) GroupBy(io.prestosql.sql.tree.GroupBy) ViewExpression(io.prestosql.spi.security.ViewExpression) NESTED_WINDOW(io.prestosql.sql.analyzer.SemanticErrorCode.NESTED_WINDOW) NaturalJoin(io.prestosql.sql.tree.NaturalJoin) Sets(com.google.common.collect.Sets) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) INVALID_LIMIT_ROW_COUNT(io.prestosql.sql.analyzer.SemanticErrorCode.INVALID_LIMIT_ROW_COUNT) NONDETERMINISTIC_ORDER_BY_EXPRESSION_WITH_SELECT_DISTINCT(io.prestosql.sql.analyzer.SemanticErrorCode.NONDETERMINISTIC_ORDER_BY_EXPRESSION_WITH_SELECT_DISTINCT) SimpleGroupBy(io.prestosql.sql.tree.SimpleGroupBy) Optional(java.util.Optional) FrameBound(io.prestosql.sql.tree.FrameBound) MetadataUtil.createQualifiedObjectName(io.prestosql.metadata.MetadataUtil.createQualifiedObjectName) UpdateIndex(io.prestosql.sql.tree.UpdateIndex) MISSING_ORDER_BY(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_ORDER_BY) CUBE_NOT_FOUND(io.prestosql.spi.connector.StandardWarningCode.CUBE_NOT_FOUND) ORDER_BY_MUST_BE_IN_SELECT(io.prestosql.sql.analyzer.SemanticErrorCode.ORDER_BY_MUST_BE_IN_SELECT) HashMap(java.util.HashMap) Multimap(com.google.common.collect.Multimap) Objects.requireNonNull(java.util.Objects.requireNonNull) ExpressionRewriter(io.prestosql.sql.tree.ExpressionRewriter) SortItem(io.prestosql.sql.tree.SortItem) VerifyException(com.google.common.base.VerifyException) Iterator(java.util.Iterator) OrderBy(io.prestosql.sql.tree.OrderBy) MISSING_TABLE(io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_TABLE) VIEW_ANALYSIS_ERROR(io.prestosql.sql.analyzer.SemanticErrorCode.VIEW_ANALYSIS_ERROR) Update(io.prestosql.sql.tree.Update) InsertCube(io.prestosql.sql.tree.InsertCube) Revoke(io.prestosql.sql.tree.Revoke) TableMetadata(io.prestosql.metadata.TableMetadata) CreateIndexMetadata(io.prestosql.spi.connector.CreateIndexMetadata) IndexClient(io.prestosql.spi.heuristicindex.IndexClient) ArrayList(java.util.ArrayList) UncheckedIOException(java.io.UncheckedIOException) IOException(java.io.IOException) UncheckedIOException(java.io.UncheckedIOException) NodeUtils.mapFromProperties(io.prestosql.sql.NodeUtils.mapFromProperties) SystemSessionProperties(io.prestosql.SystemSessionProperties) Properties(java.util.Properties) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) MetadataUtil.createQualifiedObjectName(io.prestosql.metadata.MetadataUtil.createQualifiedObjectName) LinkedList(java.util.LinkedList) CreateIndex(io.prestosql.sql.tree.CreateIndex) Identifier(io.prestosql.sql.tree.Identifier) TableHandle(io.prestosql.spi.metadata.TableHandle) Pair(io.prestosql.spi.heuristicindex.Pair)

Example 25 with TableHandle

use of io.prestosql.spi.metadata.TableHandle in project hetu-core by openlookeng.

the class PushPredicateIntoTableScan method pushPredicateIntoTableScan.

/**
 * For RowExpression {@param predicate}
 */
public static Optional<PlanNode> pushPredicateIntoTableScan(TableScanNode node, RowExpression predicate, boolean pruneWithPredicateExpression, Session session, PlanNodeIdAllocator idAllocator, PlanSymbolAllocator planSymbolAllocator, Metadata metadata, RowExpressionDomainTranslator domainTranslator, boolean pushPartitionsOnly) {
    // don't include non-deterministic predicates
    LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(metadata), new FunctionResolution(metadata.getFunctionAndTypeManager()), metadata.getFunctionAndTypeManager());
    RowExpression deterministicPredicate = logicalRowExpressions.filterDeterministicConjuncts(predicate);
    RowExpressionDomainTranslator.ExtractionResult<VariableReferenceExpression> decomposedPredicate = domainTranslator.fromPredicate(session.toConnectorSession(), deterministicPredicate);
    TupleDomain<ColumnHandle> newDomain = decomposedPredicate.getTupleDomain().transform(variableName -> node.getAssignments().get(new Symbol(variableName.getName()))).intersect(node.getEnforcedConstraint());
    Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse();
    Set<ColumnHandle> allColumnHandles = new HashSet<>();
    assignments.keySet().stream().forEach(allColumnHandles::add);
    Constraint constraint;
    List<Constraint> disjunctConstraints = ImmutableList.of();
    if (!pushPartitionsOnly) {
        List<RowExpression> orSet = LogicalRowExpressions.extractDisjuncts(decomposedPredicate.getRemainingExpression());
        List<RowExpressionDomainTranslator.ExtractionResult<VariableReferenceExpression>> disjunctPredicates = orSet.stream().map(e -> domainTranslator.fromPredicate(session.toConnectorSession(), e)).collect(Collectors.toList());
        /* Check if any Branch yeild all records; then no need to process OR branches */
        if (!disjunctPredicates.stream().anyMatch(e -> e.getTupleDomain().isAll())) {
            List<TupleDomain<ColumnHandle>> orDomains = disjunctPredicates.stream().map(er -> er.getTupleDomain().transform(variableName -> node.getAssignments().get(new Symbol(variableName.getName())))).collect(Collectors.toList());
            disjunctConstraints = orDomains.stream().filter(d -> !d.isAll() && !d.isNone()).map(d -> new Constraint(d)).collect(Collectors.toList());
        }
    }
    if (pruneWithPredicateExpression) {
        LayoutConstraintEvaluatorForRowExpression evaluator = new LayoutConstraintEvaluatorForRowExpression(metadata, session, node.getAssignments(), logicalRowExpressions.combineConjuncts(deterministicPredicate, // which would be expensive to evaluate in the call to isCandidate below.
        domainTranslator.toPredicate(newDomain.simplify().transform(column -> {
            if (assignments.size() == 0 || assignments.getOrDefault(column, null) == null) {
                return null;
            } else {
                return new VariableReferenceExpression(assignments.getOrDefault(column, null).getName(), planSymbolAllocator.getSymbols().get(assignments.getOrDefault(column, null)));
            }
        }))));
        constraint = new Constraint(newDomain, evaluator::isCandidate);
    } else {
        // Currently, invoking the expression interpreter is very expensive.
        // TODO invoke the interpreter unconditionally when the interpreter becomes cheap enough.
        constraint = new Constraint(newDomain);
    }
    TableHandle newTable;
    TupleDomain<ColumnHandle> remainingFilter;
    if (!metadata.usesLegacyTableLayouts(session, node.getTable())) {
        if (newDomain.isNone()) {
            // to turn the subtree into a Values node
            return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()));
        }
        Optional<ConstraintApplicationResult<TableHandle>> result = metadata.applyFilter(session, node.getTable(), constraint, disjunctConstraints, allColumnHandles, pushPartitionsOnly);
        if (!result.isPresent()) {
            return Optional.empty();
        }
        newTable = result.get().getHandle();
        if (metadata.getTableProperties(session, newTable).getPredicate().isNone()) {
            return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()));
        }
        remainingFilter = result.get().getRemainingFilter();
    } else {
        Optional<TableLayoutResult> layout = metadata.getLayout(session, node.getTable(), constraint, Optional.of(node.getOutputSymbols().stream().map(node.getAssignments()::get).collect(toImmutableSet())));
        if (!layout.isPresent() || layout.get().getTableProperties().getPredicate().isNone()) {
            return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()));
        }
        newTable = layout.get().getNewTableHandle();
        remainingFilter = layout.get().getUnenforcedConstraint();
    }
    TableScanNode tableScan = new TableScanNode(node.getId(), newTable, node.getOutputSymbols(), node.getAssignments(), computeEnforced(newDomain, remainingFilter), Optional.of(deterministicPredicate), node.getStrategy(), node.getReuseTableScanMappingId(), 0, node.isForDelete());
    // The order of the arguments to combineConjuncts matters:
    // * Unenforced constraints go first because they can only be simple column references,
    // which are not prone to logic errors such as out-of-bound access, div-by-zero, etc.
    // * Conjuncts in non-deterministic expressions and non-TupleDomain-expressible expressions should
    // retain their original (maybe intermixed) order from the input predicate. However, this is not implemented yet.
    // * Short of implementing the previous bullet point, the current order of non-deterministic expressions
    // and non-TupleDomain-expressible expressions should be retained. Changing the order can lead
    // to failures of previously successful queries.
    RowExpression resultingPredicate;
    if (remainingFilter.isAll() && newTable.getConnectorHandle().hasDisjunctFiltersPushdown()) {
        resultingPredicate = logicalRowExpressions.combineConjuncts(domainTranslator.toPredicate(remainingFilter.transform(assignments::get), planSymbolAllocator.getSymbols()), logicalRowExpressions.filterNonDeterministicConjuncts(predicate));
    } else {
        resultingPredicate = logicalRowExpressions.combineConjuncts(domainTranslator.toPredicate(remainingFilter.transform(assignments::get), planSymbolAllocator.getSymbols()), logicalRowExpressions.filterNonDeterministicConjuncts(predicate), decomposedPredicate.getRemainingExpression());
    }
    if (!TRUE_CONSTANT.equals(resultingPredicate)) {
        return Optional.of(new FilterNode(idAllocator.getNextId(), tableScan, resultingPredicate));
    }
    return Optional.of(tableScan);
}
Also used : ConstantExpression(io.prestosql.spi.relation.ConstantExpression) LogicalRowExpressions(io.prestosql.expressions.LogicalRowExpressions) TryFunction(io.prestosql.operator.scalar.TryFunction) NullableValue(io.prestosql.spi.predicate.NullableValue) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Capture.newCapture(io.prestosql.matching.Capture.newCapture) FilterNode(io.prestosql.spi.plan.FilterNode) Map(java.util.Map) Constraint(io.prestosql.spi.connector.Constraint) ConstraintApplicationResult(io.prestosql.spi.connector.ConstraintApplicationResult) RowExpressionDeterminismEvaluator(io.prestosql.sql.relational.RowExpressionDeterminismEvaluator) SymbolsExtractor(io.prestosql.sql.planner.SymbolsExtractor) ImmutableMap(com.google.common.collect.ImmutableMap) TableScanNode(io.prestosql.spi.plan.TableScanNode) Set(java.util.Set) PlanNode(io.prestosql.spi.plan.PlanNode) RowExpressionDomainTranslator(io.prestosql.sql.relational.RowExpressionDomainTranslator) Collectors(java.util.stream.Collectors) Metadata(io.prestosql.metadata.Metadata) PlanSymbolAllocator(io.prestosql.sql.planner.PlanSymbolAllocator) Objects(java.util.Objects) Captures(io.prestosql.matching.Captures) RowExpressionInterpreter(io.prestosql.sql.planner.RowExpressionInterpreter) List(java.util.List) FunctionResolution(io.prestosql.sql.relational.FunctionResolution) TableLayoutResult(io.prestosql.metadata.TableLayoutResult) Capture(io.prestosql.matching.Capture) Optional(java.util.Optional) TRUE_CONSTANT(io.prestosql.expressions.LogicalRowExpressions.TRUE_CONSTANT) Patterns.source(io.prestosql.sql.planner.plan.Patterns.source) Logger(io.airlift.log.Logger) Pattern(io.prestosql.matching.Pattern) TableHandle(io.prestosql.spi.metadata.TableHandle) Function(java.util.function.Function) TableLayoutResult.computeEnforced(io.prestosql.metadata.TableLayoutResult.computeEnforced) OPTIMIZED(io.prestosql.sql.planner.RowExpressionInterpreter.Level.OPTIMIZED) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) HashSet(java.util.HashSet) ImmutableList(com.google.common.collect.ImmutableList) Objects.requireNonNull(java.util.Objects.requireNonNull) Session(io.prestosql.Session) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) Symbol(io.prestosql.spi.plan.Symbol) Rule(io.prestosql.sql.planner.iterative.Rule) Patterns.filter(io.prestosql.sql.planner.plan.Patterns.filter) TupleDomain(io.prestosql.spi.predicate.TupleDomain) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) ValuesNode(io.prestosql.spi.plan.ValuesNode) VariableResolver(io.prestosql.sql.planner.VariableResolver) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Sets.intersection(com.google.common.collect.Sets.intersection) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) RowExpression(io.prestosql.spi.relation.RowExpression) Patterns.tableScan(io.prestosql.sql.planner.plan.Patterns.tableScan) ValuesNode(io.prestosql.spi.plan.ValuesNode) Constraint(io.prestosql.spi.connector.Constraint) Symbol(io.prestosql.spi.plan.Symbol) FilterNode(io.prestosql.spi.plan.FilterNode) TableLayoutResult(io.prestosql.metadata.TableLayoutResult) FunctionResolution(io.prestosql.sql.relational.FunctionResolution) ConstraintApplicationResult(io.prestosql.spi.connector.ConstraintApplicationResult) HashSet(java.util.HashSet) RowExpressionDeterminismEvaluator(io.prestosql.sql.relational.RowExpressionDeterminismEvaluator) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) LogicalRowExpressions(io.prestosql.expressions.LogicalRowExpressions) RowExpressionDomainTranslator(io.prestosql.sql.relational.RowExpressionDomainTranslator) RowExpression(io.prestosql.spi.relation.RowExpression) TupleDomain(io.prestosql.spi.predicate.TupleDomain) TableScanNode(io.prestosql.spi.plan.TableScanNode) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) TableHandle(io.prestosql.spi.metadata.TableHandle)

Aggregations

TableHandle (io.prestosql.spi.metadata.TableHandle)79 ColumnHandle (io.prestosql.spi.connector.ColumnHandle)34 Session (io.prestosql.Session)33 QualifiedObjectName (io.prestosql.spi.connector.QualifiedObjectName)32 Symbol (io.prestosql.spi.plan.Symbol)30 CatalogName (io.prestosql.spi.connector.CatalogName)28 Metadata (io.prestosql.metadata.Metadata)27 ImmutableList (com.google.common.collect.ImmutableList)25 TableMetadata (io.prestosql.metadata.TableMetadata)25 ColumnMetadata (io.prestosql.spi.connector.ColumnMetadata)25 TableScanNode (io.prestosql.spi.plan.TableScanNode)25 Map (java.util.Map)24 Optional (java.util.Optional)24 ImmutableMap (com.google.common.collect.ImmutableMap)22 PlanNode (io.prestosql.spi.plan.PlanNode)22 List (java.util.List)22 ArrayList (java.util.ArrayList)21 HashMap (java.util.HashMap)21 ConnectorTableHandle (io.prestosql.spi.connector.ConnectorTableHandle)20 Type (io.prestosql.spi.type.Type)19