Examples with HivePartitionManager com.facebook.presto.hive.HivePartitionManager used on opensource projects

Example 1 with HivePartitionManager

use of com.facebook.presto.hive.HivePartitionManager in project presto by prestodb.

the class HiveFilterPushdown method pushdownFilter.

@VisibleForTesting
public static ConnectorPushdownFilterResult pushdownFilter(ConnectorSession session, ConnectorMetadata metadata, SemiTransactionalHiveMetastore metastore, RowExpressionService rowExpressionService, StandardFunctionResolution functionResolution, HivePartitionManager partitionManager, FunctionMetadataManager functionMetadataManager, ConnectorTableHandle tableHandle, RowExpression filter, Optional<ConnectorTableLayoutHandle> currentLayoutHandle) {
    checkArgument(!FALSE_CONSTANT.equals(filter), "Cannot pushdown filter that is always false");
    if (TRUE_CONSTANT.equals(filter) && currentLayoutHandle.isPresent()) {
        return new ConnectorPushdownFilterResult(metadata.getTableLayout(session, currentLayoutHandle.get()), TRUE_CONSTANT);
    }
    // Split the filter into 3 groups of conjuncts:
    // - range filters that apply to entire columns,
    // - range filters that apply to subfields,
    // - the rest. Intersect these with possibly pre-existing filters.
    DomainTranslator.ExtractionResult<Subfield> decomposedFilter = rowExpressionService.getDomainTranslator().fromPredicate(session, filter, new SubfieldExtractor(functionResolution, rowExpressionService.getExpressionOptimizer(), session).toColumnExtractor());
    if (currentLayoutHandle.isPresent()) {
        HiveTableLayoutHandle currentHiveLayout = (HiveTableLayoutHandle) currentLayoutHandle.get();
        decomposedFilter = intersectExtractionResult(new DomainTranslator.ExtractionResult(currentHiveLayout.getDomainPredicate(), currentHiveLayout.getRemainingPredicate()), decomposedFilter);
    }
    if (decomposedFilter.getTupleDomain().isNone()) {
        return new ConnectorPushdownFilterResult(EMPTY_TABLE_LAYOUT, FALSE_CONSTANT);
    }
    RowExpression optimizedRemainingExpression = rowExpressionService.getExpressionOptimizer().optimize(decomposedFilter.getRemainingExpression(), OPTIMIZED, session);
    if (optimizedRemainingExpression instanceof ConstantExpression) {
        ConstantExpression constantExpression = (ConstantExpression) optimizedRemainingExpression;
        if (FALSE_CONSTANT.equals(constantExpression) || constantExpression.getValue() == null) {
            return new ConnectorPushdownFilterResult(EMPTY_TABLE_LAYOUT, FALSE_CONSTANT);
        }
    }
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle);
    TupleDomain<ColumnHandle> entireColumnDomain = decomposedFilter.getTupleDomain().transform(subfield -> isEntireColumn(subfield) ? subfield.getRootName() : null).transform(columnHandles::get);
    if (currentLayoutHandle.isPresent()) {
        entireColumnDomain = entireColumnDomain.intersect(((HiveTableLayoutHandle) (currentLayoutHandle.get())).getPartitionColumnPredicate());
    }
    Constraint<ColumnHandle> constraint = new Constraint<>(entireColumnDomain);
    // Extract deterministic conjuncts that apply to partition columns and specify these as Constraint#predicate
    if (!TRUE_CONSTANT.equals(decomposedFilter.getRemainingExpression())) {
        LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(rowExpressionService.getDeterminismEvaluator(), functionResolution, functionMetadataManager);
        RowExpression deterministicPredicate = logicalRowExpressions.filterDeterministicConjuncts(decomposedFilter.getRemainingExpression());
        if (!TRUE_CONSTANT.equals(deterministicPredicate)) {
            ConstraintEvaluator evaluator = new ConstraintEvaluator(rowExpressionService, session, columnHandles, deterministicPredicate);
            constraint = new Constraint<>(entireColumnDomain, evaluator::isCandidate);
        }
    }
    HivePartitionResult hivePartitionResult = partitionManager.getPartitions(metastore, tableHandle, constraint, session);
    TupleDomain<Subfield> domainPredicate = withColumnDomains(ImmutableMap.<Subfield, Domain>builder().putAll(hivePartitionResult.getUnenforcedConstraint().transform(HiveFilterPushdown::toSubfield).getDomains().orElse(ImmutableMap.of())).putAll(decomposedFilter.getTupleDomain().transform(subfield -> !isEntireColumn(subfield) ? subfield : null).getDomains().orElse(ImmutableMap.of())).build());
    Set<String> predicateColumnNames = new HashSet<>();
    domainPredicate.getDomains().get().keySet().stream().map(Subfield::getRootName).forEach(predicateColumnNames::add);
    // Include only columns referenced in the optimized expression. Although the expression is sent to the worker node
    // unoptimized, the worker is expected to optimize the expression before executing.
    extractAll(optimizedRemainingExpression).stream().map(VariableReferenceExpression::getName).forEach(predicateColumnNames::add);
    Map<String, HiveColumnHandle> predicateColumns = predicateColumnNames.stream().map(columnHandles::get).map(HiveColumnHandle.class::cast).collect(toImmutableMap(HiveColumnHandle::getName, Functions.identity()));
    SchemaTableName tableName = ((HiveTableHandle) tableHandle).getSchemaTableName();
    LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(rowExpressionService.getDeterminismEvaluator(), functionResolution, functionMetadataManager);
    List<RowExpression> conjuncts = extractConjuncts(decomposedFilter.getRemainingExpression());
    RowExpression dynamicFilterExpression = extractDynamicConjuncts(conjuncts, logicalRowExpressions);
    RowExpression remainingExpression = extractStaticConjuncts(conjuncts, logicalRowExpressions);
    remainingExpression = removeNestedDynamicFilters(remainingExpression);
    Table table = metastore.getTable(new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), isUserDefinedTypeEncodingEnabled(session), metastore.getColumnConverterProvider()), tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    return new ConnectorPushdownFilterResult(metadata.getTableLayout(session, new HiveTableLayoutHandle(tableName, table.getStorage().getLocation(), hivePartitionResult.getPartitionColumns(), // remove comments to optimize serialization costs
    pruneColumnComments(hivePartitionResult.getDataColumns()), hivePartitionResult.getTableParameters(), hivePartitionResult.getPartitions(), domainPredicate, remainingExpression, predicateColumns, hivePartitionResult.getEnforcedConstraint(), hivePartitionResult.getBucketHandle(), hivePartitionResult.getBucketFilter(), true, createTableLayoutString(session, rowExpressionService, tableName, hivePartitionResult.getBucketHandle(), hivePartitionResult.getBucketFilter(), remainingExpression, domainPredicate), currentLayoutHandle.map(layout -> ((HiveTableLayoutHandle) layout).getRequestedColumns()).orElse(Optional.empty()), false)), dynamicFilterExpression);
}