Examples with TableHandle com.facebook.presto.spi.TableHandle used on opensource projects

Example 41 with TableHandle

use of com.facebook.presto.spi.TableHandle in project presto by prestodb.

the class HiveFilterPushdown method pushdownFilter.

@VisibleForTesting
public static ConnectorPushdownFilterResult pushdownFilter(ConnectorSession session, ConnectorMetadata metadata, SemiTransactionalHiveMetastore metastore, RowExpressionService rowExpressionService, StandardFunctionResolution functionResolution, HivePartitionManager partitionManager, FunctionMetadataManager functionMetadataManager, ConnectorTableHandle tableHandle, RowExpression filter, Optional<ConnectorTableLayoutHandle> currentLayoutHandle) {
    checkArgument(!FALSE_CONSTANT.equals(filter), "Cannot pushdown filter that is always false");
    if (TRUE_CONSTANT.equals(filter) && currentLayoutHandle.isPresent()) {
        return new ConnectorPushdownFilterResult(metadata.getTableLayout(session, currentLayoutHandle.get()), TRUE_CONSTANT);
    }
    // Split the filter into 3 groups of conjuncts:
    // - range filters that apply to entire columns,
    // - range filters that apply to subfields,
    // - the rest. Intersect these with possibly pre-existing filters.
    DomainTranslator.ExtractionResult<Subfield> decomposedFilter = rowExpressionService.getDomainTranslator().fromPredicate(session, filter, new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session).toColumnExtractor());
    if (currentLayoutHandle.isPresent()) {
        HiveTableLayoutHandle currentHiveLayout = (HiveTableLayoutHandle) currentLayoutHandle.get();
        decomposedFilter = intersectExtractionResult(new DomainTranslator.ExtractionResult(currentHiveLayout.getDomainPredicate(), currentHiveLayout.getRemainingPredicate()), decomposedFilter);
    }
    if (decomposedFilter.getTupleDomain().isNone()) {
        return new ConnectorPushdownFilterResult(EMPTY_TABLE_LAYOUT, FALSE_CONSTANT);
    }
    RowExpression optimizedRemainingExpression = rowExpressionService.getExpressionOptimizer().optimize(decomposedFilter.getRemainingExpression(), OPTIMIZED, session);
    if (optimizedRemainingExpression instanceof ConstantExpression) {
        ConstantExpression constantExpression = (ConstantExpression) optimizedRemainingExpression;
        if (FALSE_CONSTANT.equals(constantExpression) || constantExpression.getValue() == null) {
            return new ConnectorPushdownFilterResult(EMPTY_TABLE_LAYOUT, FALSE_CONSTANT);
        }
    }
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle);
    TupleDomain<ColumnHandle> entireColumnDomain = decomposedFilter.getTupleDomain().transform(subfield -> isEntireColumn(subfield) ? subfield.getRootName() : null).transform(columnHandles::get);
    if (currentLayoutHandle.isPresent()) {
        entireColumnDomain = entireColumnDomain.intersect(((HiveTableLayoutHandle) (currentLayoutHandle.get())).getPartitionColumnPredicate());
    }
    Constraint<ColumnHandle> constraint = new Constraint<>(entireColumnDomain);
    // Extract deterministic conjuncts that apply to partition columns and specify these as Constraint#predicate
    if (!TRUE_CONSTANT.equals(decomposedFilter.getRemainingExpression())) {
        LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(rowExpressionService.getDeterminismEvaluator(), functionResolution, functionMetadataManager);
        RowExpression deterministicPredicate = logicalRowExpressions.filterDeterministicConjuncts(decomposedFilter.getRemainingExpression());
        if (!TRUE_CONSTANT.equals(deterministicPredicate)) {
            ConstraintEvaluator evaluator = new ConstraintEvaluator(rowExpressionService, session, columnHandles, deterministicPredicate);
            constraint = new Constraint<>(entireColumnDomain, evaluator::isCandidate);
        }
    }
    HivePartitionResult hivePartitionResult = partitionManager.getPartitions(metastore, tableHandle, constraint, session);
    TupleDomain<Subfield> domainPredicate = withColumnDomains(ImmutableMap.<Subfield, Domain>builder().putAll(hivePartitionResult.getUnenforcedConstraint().transform(HiveFilterPushdown::toSubfield).getDomains().orElse(ImmutableMap.of())).putAll(decomposedFilter.getTupleDomain().transform(subfield -> !isEntireColumn(subfield) ? subfield : null).getDomains().orElse(ImmutableMap.of())).build());
    Set<String> predicateColumnNames = new HashSet<>();
    domainPredicate.getDomains().get().keySet().stream().map(Subfield::getRootName).forEach(predicateColumnNames::add);
    // Include only columns referenced in the optimized expression. Although the expression is sent to the worker node
    // unoptimized, the worker is expected to optimize the expression before executing.
    extractAll(optimizedRemainingExpression).stream().map(VariableReferenceExpression::getName).forEach(predicateColumnNames::add);
    Map<String, HiveColumnHandle> predicateColumns = predicateColumnNames.stream().map(columnHandles::get).map(HiveColumnHandle.class::cast).collect(toImmutableMap(HiveColumnHandle::getName, Functions.identity()));
    SchemaTableName tableName = ((HiveTableHandle) tableHandle).getSchemaTableName();
    LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(rowExpressionService.getDeterminismEvaluator(), functionResolution, functionMetadataManager);
    List<RowExpression> conjuncts = extractConjuncts(decomposedFilter.getRemainingExpression());
    RowExpression dynamicFilterExpression = extractDynamicConjuncts(conjuncts, logicalRowExpressions);
    RowExpression remainingExpression = extractStaticConjuncts(conjuncts, logicalRowExpressions);
    remainingExpression = removeNestedDynamicFilters(remainingExpression);
    Table table = metastore.getTable(new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), isUserDefinedTypeEncodingEnabled(session), metastore.getColumnConverterProvider()), tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    return new ConnectorPushdownFilterResult(metadata.getTableLayout(session, new HiveTableLayoutHandle(tableName, table.getStorage().getLocation(), hivePartitionResult.getPartitionColumns(), // remove comments to optimize serialization costs
    pruneColumnComments(hivePartitionResult.getDataColumns()), hivePartitionResult.getTableParameters(), hivePartitionResult.getPartitions(), domainPredicate, remainingExpression, predicateColumns, hivePartitionResult.getEnforcedConstraint(), hivePartitionResult.getBucketHandle(), hivePartitionResult.getBucketFilter(), true, createTableLayoutString(session, rowExpressionService, tableName, hivePartitionResult.getBucketHandle(), hivePartitionResult.getBucketFilter(), remainingExpression, domainPredicate), currentLayoutHandle.map(layout -> ((HiveTableLayoutHandle) layout).getRequestedColumns()).orElse(Optional.empty()), false)), dynamicFilterExpression);
}

Example 42 with TableHandle

use of com.facebook.presto.spi.TableHandle in project presto by prestodb.

the class AddColumnTask method execute.

@Override
public ListenableFuture<?> execute(AddColumn statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, Session session, List<Expression> parameters, WarningCollector warningCollector) {
    QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getName());
    Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName);
    if (!tableHandle.isPresent()) {
        if (!statement.isTableExists()) {
            throw new SemanticException(MISSING_TABLE, statement, "Table '%s' does not exist", tableName);
        }
        return immediateFuture(null);
    }
    Optional<ConnectorMaterializedViewDefinition> optionalMaterializedView = metadata.getMaterializedView(session, tableName);
    if (optionalMaterializedView.isPresent()) {
        if (!statement.isTableExists()) {
            throw new SemanticException(NOT_SUPPORTED, statement, "'%s' is a materialized view, and add column is not supported", tableName);
        }
        return immediateFuture(null);
    }
    ConnectorId connectorId = metadata.getCatalogHandle(session, tableName.getCatalogName()).orElseThrow(() -> new PrestoException(NOT_FOUND, "Catalog does not exist: " + tableName.getCatalogName()));
    accessControl.checkCanAddColumns(session.getRequiredTransactionId(), session.getIdentity(), session.getAccessControlContext(), tableName);
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle.get());
    ColumnDefinition element = statement.getColumn();
    Type type;
    try {
        type = metadata.getType(parseTypeSignature(element.getType()));
    } catch (IllegalArgumentException e) {
        throw new SemanticException(TYPE_MISMATCH, element, "Unknown type '%s' for column '%s'", element.getType(), element.getName());
    }
    if (type.equals(UNKNOWN)) {
        throw new SemanticException(TYPE_MISMATCH, element, "Unknown type '%s' for column '%s'", element.getType(), element.getName());
    }
    if (columnHandles.containsKey(element.getName().getValue().toLowerCase(ENGLISH))) {
        if (!statement.isColumnNotExists()) {
            throw new SemanticException(COLUMN_ALREADY_EXISTS, statement, "Column '%s' already exists", element.getName());
        }
        return immediateFuture(null);
    }
    if (!element.isNullable() && !metadata.getConnectorCapabilities(session, connectorId).contains(NOT_NULL_COLUMN_CONSTRAINT)) {
        throw new SemanticException(NOT_SUPPORTED, element, "Catalog '%s' does not support NOT NULL for column '%s'", connectorId.getCatalogName(), element.getName());
    }
    Map<String, Expression> sqlProperties = mapFromProperties(element.getProperties());
    Map<String, Object> columnProperties = metadata.getColumnPropertyManager().getProperties(connectorId, tableName.getCatalogName(), sqlProperties, session, metadata, parameters);
    ColumnMetadata column = new ColumnMetadata(element.getName().getValue(), type, element.isNullable(), element.getComment().orElse(null), null, false, columnProperties);
    metadata.addColumn(session, tableHandle.get(), column);
    return immediateFuture(null);
}

Example 43 with TableHandle

use of com.facebook.presto.spi.TableHandle in project presto by prestodb.

the class DropColumnTask method execute.

@Override
public ListenableFuture<?> execute(DropColumn statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, Session session, List<Expression> parameters, WarningCollector warningCollector) {
    QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getTable());
    Optional<TableHandle> tableHandleOptional = metadata.getTableHandle(session, tableName);
    if (!tableHandleOptional.isPresent()) {
        if (!statement.isTableExists()) {
            throw new SemanticException(MISSING_TABLE, statement, "Table '%s' does not exist", tableName);
        }
        return immediateFuture(null);
    }
    Optional<ConnectorMaterializedViewDefinition> optionalMaterializedView = metadata.getMaterializedView(session, tableName);
    if (optionalMaterializedView.isPresent()) {
        if (!statement.isTableExists()) {
            throw new SemanticException(NOT_SUPPORTED, statement, "'%s' is a materialized view, and drop column is not supported", tableName);
        }
        return immediateFuture(null);
    }
    TableHandle tableHandle = tableHandleOptional.get();
    String column = statement.getColumn().getValue().toLowerCase(ENGLISH);
    accessControl.checkCanDropColumn(session.getRequiredTransactionId(), session.getIdentity(), session.getAccessControlContext(), tableName);
    ColumnHandle columnHandle = metadata.getColumnHandles(session, tableHandle).get(column);
    if (columnHandle == null) {
        if (!statement.isColumnExists()) {
            throw new SemanticException(MISSING_COLUMN, statement, "Column '%s' does not exist", column);
        }
        return immediateFuture(null);
    }
    if (metadata.getColumnMetadata(session, tableHandle, columnHandle).isHidden()) {
        throw new SemanticException(NOT_SUPPORTED, statement, "Cannot drop hidden column");
    }
    if (metadata.getTableMetadata(session, tableHandle).getColumns().stream().filter(info -> !info.isHidden()).count() <= 1) {
        throw new SemanticException(NOT_SUPPORTED, statement, "Cannot drop the only column in a table");
    }
    metadata.dropColumn(session, tableHandle, columnHandle);
    return immediateFuture(null);
}

Example 44 with TableHandle

use of com.facebook.presto.spi.TableHandle in project presto by prestodb.

the class DropTableTask method execute.

@Override
public ListenableFuture<?> execute(DropTable statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, Session session, List<Expression> parameters, WarningCollector warningCollector) {
    QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getTableName());
    Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName);
    if (!tableHandle.isPresent()) {
        if (!statement.isExists()) {
            throw new SemanticException(MISSING_TABLE, statement, "Table '%s' does not exist", tableName);
        }
        return immediateFuture(null);
    }
    Optional<ConnectorMaterializedViewDefinition> optionalMaterializedView = metadata.getMaterializedView(session, tableName);
    if (optionalMaterializedView.isPresent()) {
        if (!statement.isExists()) {
            throw new SemanticException(NOT_SUPPORTED, statement, "'%s' is a materialized view, not a table. Use DROP MATERIALIZED VIEW to drop.", tableName);
        }
        return immediateFuture(null);
    }
    accessControl.checkCanDropTable(session.getRequiredTransactionId(), session.getIdentity(), session.getAccessControlContext(), tableName);
    metadata.dropTable(session, tableHandle.get());
    return immediateFuture(null);
}

Example 45 with TableHandle

use of com.facebook.presto.spi.TableHandle in project presto by prestodb.

the class MaterializedViewQueryOptimizer method visitQuerySpecification.

@Override
protected Node visitQuerySpecification(QuerySpecification node, Void context) {
    if (!node.getFrom().isPresent()) {
        throw new IllegalStateException("Query with no From clause is not rewritable by materialized view");
    }
    Relation relation = node.getFrom().get();
    if (relation instanceof AliasedRelation) {
        removablePrefix = Optional.of(((AliasedRelation) relation).getAlias());
        relation = ((AliasedRelation) relation).getRelation();
    }
    if (!(relation instanceof Table)) {
        throw new SemanticException(NOT_SUPPORTED, node, "Relation other than Table is not supported in query optimizer");
    }
    Table baseTable = (Table) relation;
    if (!removablePrefix.isPresent()) {
        removablePrefix = Optional.of(new Identifier(baseTable.getName().toString()));
    }
    if (node.getGroupBy().isPresent()) {
        ImmutableSet.Builder<Expression> expressionsInGroupByBuilder = ImmutableSet.builder();
        for (GroupingElement element : node.getGroupBy().get().getGroupingElements()) {
            element = removeGroupingElementPrefix(element, removablePrefix);
            Optional<Set<Expression>> groupByOfMaterializedView = materializedViewInfo.getGroupBy();
            if (groupByOfMaterializedView.isPresent()) {
                for (Expression expression : element.getExpressions()) {
                    if (!groupByOfMaterializedView.get().contains(expression) || !materializedViewInfo.getBaseToViewColumnMap().containsKey(expression)) {
                        throw new IllegalStateException(format("Grouping element %s is not present in materialized view groupBy field", element));
                    }
                }
            }
            expressionsInGroupByBuilder.addAll(element.getExpressions());
        }
        expressionsInGroupBy = Optional.of(expressionsInGroupByBuilder.build());
    }
    // TODO: Add HAVING validation to the validator https://github.com/prestodb/presto/issues/16406
    if (node.getHaving().isPresent()) {
        throw new SemanticException(NOT_SUPPORTED, node, "Having clause is not supported in query optimizer");
    }
    if (materializedViewInfo.getWhereClause().isPresent()) {
        if (!node.getWhere().isPresent()) {
            throw new IllegalStateException("Query with no where clause is not rewritable by materialized view with where clause");
        }
        QualifiedObjectName baseTableName = createQualifiedObjectName(session, baseTable, baseTable.getName());
        Optional<TableHandle> tableHandle = metadata.getTableHandle(session, baseTableName);
        if (!tableHandle.isPresent()) {
            throw new SemanticException(MISSING_TABLE, node, "Table does not exist");
        }
        ImmutableList.Builder<Field> fields = ImmutableList.builder();
        for (ColumnHandle columnHandle : metadata.getColumnHandles(session, tableHandle.get()).values()) {
            ColumnMetadata columnMetadata = metadata.getColumnMetadata(session, tableHandle.get(), columnHandle);
            fields.add(Field.newUnqualified(materializedViewInfo.getWhereClause().get().getLocation(), columnMetadata.getName(), columnMetadata.getType()));
        }
        Scope scope = Scope.builder().withRelationType(RelationId.anonymous(), new RelationType(fields.build())).build();
        // Given base query's filter condition and materialized view's filter condition, the goal is to check if MV's filters contain Base's filters (Base implies MV).
        // Let base query's filter condition be A, and MV's filter condition be B.
        // One way to evaluate A implies B is to evaluate logical expression A^~B and check if the output domain is none.
        // If the resulting domain is none, then A^~B is false. Thus A implies B.
        // For more information and examples: https://fb.quip.com/WwmxA40jLMxR
        // TODO: Implement method that utilizes external SAT solver libraries. https://github.com/prestodb/presto/issues/16536
        RowExpression materializedViewWhereCondition = convertToRowExpression(materializedViewInfo.getWhereClause().get(), scope);
        RowExpression baseQueryWhereCondition = convertToRowExpression(node.getWhere().get(), scope);
        RowExpression rewriteLogicExpression = and(baseQueryWhereCondition, call(baseQueryWhereCondition.getSourceLocation(), "not", new FunctionResolution(metadata.getFunctionAndTypeManager()).notFunction(), materializedViewWhereCondition.getType(), materializedViewWhereCondition));
        RowExpression disjunctiveNormalForm = logicalRowExpressions.convertToDisjunctiveNormalForm(rewriteLogicExpression);
        ExtractionResult<VariableReferenceExpression> result = domainTranslator.fromPredicate(session.toConnectorSession(), disjunctiveNormalForm, BASIC_COLUMN_EXTRACTOR);
        if (!result.getTupleDomain().equals(TupleDomain.none())) {
            throw new IllegalStateException("View filter condition does not contain base query's filter condition");
        }
    }
    return new QuerySpecification((Select) process(node.getSelect(), context), node.getFrom().map(from -> (Relation) process(from, context)), node.getWhere().map(where -> (Expression) process(where, context)), node.getGroupBy().map(groupBy -> (GroupBy) process(groupBy, context)), node.getHaving().map(having -> (Expression) process(having, context)), node.getOrderBy().map(orderBy -> (OrderBy) process(orderBy, context)), node.getOffset(), node.getLimit());
}