Search in sources :

Example 1 with RowExpressionDomainTranslator

use of io.prestosql.sql.relational.RowExpressionDomainTranslator in project hetu-core by openlookeng.

the class TableDeleteOperator method getOutput.

@Override
public Page getOutput() {
    if (state == State.FINISHED || (sourceLayout.isPresent() && state != State.FINISHING)) {
        return null;
    }
    state = State.FINISHED;
    TableHandle tableHandleForDelete = tableHandle;
    if (sourceLayout.isPresent()) {
        // Replace the values from subqueries in filter predicates
        VariablesExtractor.extractUnique(filter.get()).stream().forEach(expr -> symbolExpressionMap.putIfAbsent(new Symbol(expr.getName()), expr));
        RowExpression rowExpression = RowExpressionVariableInliner.inlineVariables(node -> symbolExpressionMap.get(new Symbol(node.getName())), filter.get());
        // Create the tuple domain based on filter and values from source;
        RowExpressionDomainTranslator.ExtractionResult<VariableReferenceExpression> decomposedPredicate = (new RowExpressionDomainTranslator(metadata)).fromPredicate(session.toConnectorSession(), rowExpression, RowExpressionDomainTranslator.BASIC_COLUMN_EXTRACTOR);
        TupleDomain<ColumnHandle> tupleDomain = decomposedPredicate.getTupleDomain().transform(variableName -> assignments.get(new Symbol(variableName.getName())));
        Constraint constraint = new Constraint(tupleDomain);
        // Apply the constraint on the table handle
        Optional<TableHandle> tableHandleDelete = metadata.applyDelete(session, this.tableHandle, constraint);
        if (tableHandleDelete.isPresent()) {
            tableHandleForDelete = tableHandleDelete.get();
        }
    }
    OptionalLong rowsDeletedCount = metadata.executeDelete(session, tableHandleForDelete);
    // output page will only be constructed once,
    // so a new PageBuilder is constructed (instead of using PageBuilder.reset)
    PageBuilder page = new PageBuilder(1, TYPES);
    BlockBuilder rowsBuilder = page.getBlockBuilder(0);
    page.declarePosition();
    if (rowsDeletedCount.isPresent()) {
        BIGINT.writeLong(rowsBuilder, rowsDeletedCount.getAsLong());
    } else {
        rowsBuilder.appendNull();
    }
    return page.build();
}
Also used : ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Constraint(io.prestosql.spi.connector.Constraint) Symbol(io.prestosql.spi.plan.Symbol) RowExpressionDomainTranslator(io.prestosql.sql.relational.RowExpressionDomainTranslator) RowExpression(io.prestosql.spi.relation.RowExpression) PageBuilder(io.prestosql.spi.PageBuilder) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) OptionalLong(java.util.OptionalLong) TableHandle(io.prestosql.spi.metadata.TableHandle) BlockBuilder(io.prestosql.spi.block.BlockBuilder)

Example 2 with RowExpressionDomainTranslator

use of io.prestosql.sql.relational.RowExpressionDomainTranslator in project hetu-core by openlookeng.

the class PushPredicateIntoTableScan method pushPredicateIntoTableScan.

/**
 * For RowExpression {@param predicate}
 */
public static Optional<PlanNode> pushPredicateIntoTableScan(TableScanNode node, RowExpression predicate, boolean pruneWithPredicateExpression, Session session, PlanNodeIdAllocator idAllocator, PlanSymbolAllocator planSymbolAllocator, Metadata metadata, RowExpressionDomainTranslator domainTranslator, boolean pushPartitionsOnly) {
    // don't include non-deterministic predicates
    LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(metadata), new FunctionResolution(metadata.getFunctionAndTypeManager()), metadata.getFunctionAndTypeManager());
    RowExpression deterministicPredicate = logicalRowExpressions.filterDeterministicConjuncts(predicate);
    RowExpressionDomainTranslator.ExtractionResult<VariableReferenceExpression> decomposedPredicate = domainTranslator.fromPredicate(session.toConnectorSession(), deterministicPredicate);
    TupleDomain<ColumnHandle> newDomain = decomposedPredicate.getTupleDomain().transform(variableName -> node.getAssignments().get(new Symbol(variableName.getName()))).intersect(node.getEnforcedConstraint());
    Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse();
    Set<ColumnHandle> allColumnHandles = new HashSet<>();
    assignments.keySet().stream().forEach(allColumnHandles::add);
    Constraint constraint;
    List<Constraint> disjunctConstraints = ImmutableList.of();
    if (!pushPartitionsOnly) {
        List<RowExpression> orSet = LogicalRowExpressions.extractDisjuncts(decomposedPredicate.getRemainingExpression());
        List<RowExpressionDomainTranslator.ExtractionResult<VariableReferenceExpression>> disjunctPredicates = orSet.stream().map(e -> domainTranslator.fromPredicate(session.toConnectorSession(), e)).collect(Collectors.toList());
        /* Check if any Branch yeild all records; then no need to process OR branches */
        if (!disjunctPredicates.stream().anyMatch(e -> e.getTupleDomain().isAll())) {
            List<TupleDomain<ColumnHandle>> orDomains = disjunctPredicates.stream().map(er -> er.getTupleDomain().transform(variableName -> node.getAssignments().get(new Symbol(variableName.getName())))).collect(Collectors.toList());
            disjunctConstraints = orDomains.stream().filter(d -> !d.isAll() && !d.isNone()).map(d -> new Constraint(d)).collect(Collectors.toList());
        }
    }
    if (pruneWithPredicateExpression) {
        LayoutConstraintEvaluatorForRowExpression evaluator = new LayoutConstraintEvaluatorForRowExpression(metadata, session, node.getAssignments(), logicalRowExpressions.combineConjuncts(deterministicPredicate, // which would be expensive to evaluate in the call to isCandidate below.
        domainTranslator.toPredicate(newDomain.simplify().transform(column -> {
            if (assignments.size() == 0 || assignments.getOrDefault(column, null) == null) {
                return null;
            } else {
                return new VariableReferenceExpression(assignments.getOrDefault(column, null).getName(), planSymbolAllocator.getSymbols().get(assignments.getOrDefault(column, null)));
            }
        }))));
        constraint = new Constraint(newDomain, evaluator::isCandidate);
    } else {
        // Currently, invoking the expression interpreter is very expensive.
        // TODO invoke the interpreter unconditionally when the interpreter becomes cheap enough.
        constraint = new Constraint(newDomain);
    }
    TableHandle newTable;
    TupleDomain<ColumnHandle> remainingFilter;
    if (!metadata.usesLegacyTableLayouts(session, node.getTable())) {
        if (newDomain.isNone()) {
            // to turn the subtree into a Values node
            return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()));
        }
        Optional<ConstraintApplicationResult<TableHandle>> result = metadata.applyFilter(session, node.getTable(), constraint, disjunctConstraints, allColumnHandles, pushPartitionsOnly);
        if (!result.isPresent()) {
            return Optional.empty();
        }
        newTable = result.get().getHandle();
        if (metadata.getTableProperties(session, newTable).getPredicate().isNone()) {
            return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()));
        }
        remainingFilter = result.get().getRemainingFilter();
    } else {
        Optional<TableLayoutResult> layout = metadata.getLayout(session, node.getTable(), constraint, Optional.of(node.getOutputSymbols().stream().map(node.getAssignments()::get).collect(toImmutableSet())));
        if (!layout.isPresent() || layout.get().getTableProperties().getPredicate().isNone()) {
            return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()));
        }
        newTable = layout.get().getNewTableHandle();
        remainingFilter = layout.get().getUnenforcedConstraint();
    }
    TableScanNode tableScan = new TableScanNode(node.getId(), newTable, node.getOutputSymbols(), node.getAssignments(), computeEnforced(newDomain, remainingFilter), Optional.of(deterministicPredicate), node.getStrategy(), node.getReuseTableScanMappingId(), 0, node.isForDelete());
    // The order of the arguments to combineConjuncts matters:
    // * Unenforced constraints go first because they can only be simple column references,
    // which are not prone to logic errors such as out-of-bound access, div-by-zero, etc.
    // * Conjuncts in non-deterministic expressions and non-TupleDomain-expressible expressions should
    // retain their original (maybe intermixed) order from the input predicate. However, this is not implemented yet.
    // * Short of implementing the previous bullet point, the current order of non-deterministic expressions
    // and non-TupleDomain-expressible expressions should be retained. Changing the order can lead
    // to failures of previously successful queries.
    RowExpression resultingPredicate;
    if (remainingFilter.isAll() && newTable.getConnectorHandle().hasDisjunctFiltersPushdown()) {
        resultingPredicate = logicalRowExpressions.combineConjuncts(domainTranslator.toPredicate(remainingFilter.transform(assignments::get), planSymbolAllocator.getSymbols()), logicalRowExpressions.filterNonDeterministicConjuncts(predicate));
    } else {
        resultingPredicate = logicalRowExpressions.combineConjuncts(domainTranslator.toPredicate(remainingFilter.transform(assignments::get), planSymbolAllocator.getSymbols()), logicalRowExpressions.filterNonDeterministicConjuncts(predicate), decomposedPredicate.getRemainingExpression());
    }
    if (!TRUE_CONSTANT.equals(resultingPredicate)) {
        return Optional.of(new FilterNode(idAllocator.getNextId(), tableScan, resultingPredicate));
    }
    return Optional.of(tableScan);
}
Also used : ConstantExpression(io.prestosql.spi.relation.ConstantExpression) LogicalRowExpressions(io.prestosql.expressions.LogicalRowExpressions) TryFunction(io.prestosql.operator.scalar.TryFunction) NullableValue(io.prestosql.spi.predicate.NullableValue) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Capture.newCapture(io.prestosql.matching.Capture.newCapture) FilterNode(io.prestosql.spi.plan.FilterNode) Map(java.util.Map) Constraint(io.prestosql.spi.connector.Constraint) ConstraintApplicationResult(io.prestosql.spi.connector.ConstraintApplicationResult) RowExpressionDeterminismEvaluator(io.prestosql.sql.relational.RowExpressionDeterminismEvaluator) SymbolsExtractor(io.prestosql.sql.planner.SymbolsExtractor) ImmutableMap(com.google.common.collect.ImmutableMap) TableScanNode(io.prestosql.spi.plan.TableScanNode) Set(java.util.Set) PlanNode(io.prestosql.spi.plan.PlanNode) RowExpressionDomainTranslator(io.prestosql.sql.relational.RowExpressionDomainTranslator) Collectors(java.util.stream.Collectors) Metadata(io.prestosql.metadata.Metadata) PlanSymbolAllocator(io.prestosql.sql.planner.PlanSymbolAllocator) Objects(java.util.Objects) Captures(io.prestosql.matching.Captures) RowExpressionInterpreter(io.prestosql.sql.planner.RowExpressionInterpreter) List(java.util.List) FunctionResolution(io.prestosql.sql.relational.FunctionResolution) TableLayoutResult(io.prestosql.metadata.TableLayoutResult) Capture(io.prestosql.matching.Capture) Optional(java.util.Optional) TRUE_CONSTANT(io.prestosql.expressions.LogicalRowExpressions.TRUE_CONSTANT) Patterns.source(io.prestosql.sql.planner.plan.Patterns.source) Logger(io.airlift.log.Logger) Pattern(io.prestosql.matching.Pattern) TableHandle(io.prestosql.spi.metadata.TableHandle) Function(java.util.function.Function) TableLayoutResult.computeEnforced(io.prestosql.metadata.TableLayoutResult.computeEnforced) OPTIMIZED(io.prestosql.sql.planner.RowExpressionInterpreter.Level.OPTIMIZED) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) HashSet(java.util.HashSet) ImmutableList(com.google.common.collect.ImmutableList) Objects.requireNonNull(java.util.Objects.requireNonNull) Session(io.prestosql.Session) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) Symbol(io.prestosql.spi.plan.Symbol) Rule(io.prestosql.sql.planner.iterative.Rule) Patterns.filter(io.prestosql.sql.planner.plan.Patterns.filter) TupleDomain(io.prestosql.spi.predicate.TupleDomain) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) ValuesNode(io.prestosql.spi.plan.ValuesNode) VariableResolver(io.prestosql.sql.planner.VariableResolver) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Sets.intersection(com.google.common.collect.Sets.intersection) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) RowExpression(io.prestosql.spi.relation.RowExpression) Patterns.tableScan(io.prestosql.sql.planner.plan.Patterns.tableScan) ValuesNode(io.prestosql.spi.plan.ValuesNode) Constraint(io.prestosql.spi.connector.Constraint) Symbol(io.prestosql.spi.plan.Symbol) FilterNode(io.prestosql.spi.plan.FilterNode) TableLayoutResult(io.prestosql.metadata.TableLayoutResult) FunctionResolution(io.prestosql.sql.relational.FunctionResolution) ConstraintApplicationResult(io.prestosql.spi.connector.ConstraintApplicationResult) HashSet(java.util.HashSet) RowExpressionDeterminismEvaluator(io.prestosql.sql.relational.RowExpressionDeterminismEvaluator) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) LogicalRowExpressions(io.prestosql.expressions.LogicalRowExpressions) RowExpressionDomainTranslator(io.prestosql.sql.relational.RowExpressionDomainTranslator) RowExpression(io.prestosql.spi.relation.RowExpression) TupleDomain(io.prestosql.spi.predicate.TupleDomain) TableScanNode(io.prestosql.spi.plan.TableScanNode) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) TableHandle(io.prestosql.spi.metadata.TableHandle)

Example 3 with RowExpressionDomainTranslator

use of io.prestosql.sql.relational.RowExpressionDomainTranslator in project hetu-core by openlookeng.

the class PushPredicateIntoTableScan method apply.

@Override
public Result apply(FilterNode filterNode, Captures captures, Context context) {
    TableScanNode tableScan = captures.get(TABLE_SCAN);
    Optional<PlanNode> rewritten = pushPredicateIntoTableScan(tableScan, filterNode.getPredicate(), false, context.getSession(), context.getIdAllocator(), context.getSymbolAllocator(), metadata, new RowExpressionDomainTranslator(metadata), pushPartitionsOnly);
    if (!rewritten.isPresent() || arePlansSame(filterNode, tableScan, rewritten.get())) {
        return Result.empty();
    }
    return Result.ofPlanNode(rewritten.get());
}
Also used : PlanNode(io.prestosql.spi.plan.PlanNode) TableScanNode(io.prestosql.spi.plan.TableScanNode) RowExpressionDomainTranslator(io.prestosql.sql.relational.RowExpressionDomainTranslator)

Example 4 with RowExpressionDomainTranslator

use of io.prestosql.sql.relational.RowExpressionDomainTranslator in project hetu-core by openlookeng.

the class PredicatePushDown method optimize.

@Override
public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, PlanSymbolAllocator planSymbolAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) {
    requireNonNull(plan, "plan is null");
    requireNonNull(session, "session is null");
    requireNonNull(types, "types is null");
    requireNonNull(idAllocator, "idAllocator is null");
    RowExpressionPredicateExtractor predicateExtractor = new RowExpressionPredicateExtractor(new RowExpressionDomainTranslator(metadata), metadata, planSymbolAllocator, useTableProperties);
    Memo memo = new Memo(idAllocator, plan);
    Lookup lookup = Lookup.from(planNode -> Stream.of(memo.resolve(planNode)));
    StatsProvider statsProvider = new CachingStatsProvider(costCalculationHandle.getStatsCalculator(), Optional.of(memo), lookup, session, planSymbolAllocator.getTypes(), true);
    CostProvider costProvider = new CachingCostProvider(costCalculationHandle.getCostCalculator(), statsProvider, Optional.of(memo), session, planSymbolAllocator.getTypes());
    FilterPushdownForCTEHandler filterPushdownForCTEHandler = new FilterPushdownForCTEHandler(pushdownForCTE, costProvider, costCalculationHandle.getCostComparator());
    Rewriter rewriter = new Rewriter(planSymbolAllocator, idAllocator, metadata, predicateExtractor, typeAnalyzer, session, dynamicFiltering, filterPushdownForCTEHandler);
    PlanNode rewrittenNode = SimplePlanRewriter.rewriteWith(rewriter, plan, TRUE_CONSTANT);
    if (rewriter.isSecondTraverseRequired()) {
        return SimplePlanRewriter.rewriteWith(rewriter, rewrittenNode, TRUE_CONSTANT);
    }
    return rewrittenNode;
}
Also used : CachingStatsProvider(io.prestosql.cost.CachingStatsProvider) PlanNode(io.prestosql.spi.plan.PlanNode) StatsProvider(io.prestosql.cost.StatsProvider) CachingStatsProvider(io.prestosql.cost.CachingStatsProvider) CachingCostProvider(io.prestosql.cost.CachingCostProvider) RowExpressionDomainTranslator(io.prestosql.sql.relational.RowExpressionDomainTranslator) CostProvider(io.prestosql.cost.CostProvider) CachingCostProvider(io.prestosql.cost.CachingCostProvider) SimplePlanRewriter(io.prestosql.sql.planner.plan.SimplePlanRewriter) Lookup(io.prestosql.sql.planner.iterative.Lookup) Memo(io.prestosql.sql.planner.iterative.Memo) RowExpressionPredicateExtractor(io.prestosql.sql.planner.RowExpressionPredicateExtractor)

Aggregations

RowExpressionDomainTranslator (io.prestosql.sql.relational.RowExpressionDomainTranslator)4 PlanNode (io.prestosql.spi.plan.PlanNode)3 ColumnHandle (io.prestosql.spi.connector.ColumnHandle)2 Constraint (io.prestosql.spi.connector.Constraint)2 TableHandle (io.prestosql.spi.metadata.TableHandle)2 Symbol (io.prestosql.spi.plan.Symbol)2 TableScanNode (io.prestosql.spi.plan.TableScanNode)2 RowExpression (io.prestosql.spi.relation.RowExpression)2 VariableReferenceExpression (io.prestosql.spi.relation.VariableReferenceExpression)2 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)1 ImmutableBiMap (com.google.common.collect.ImmutableBiMap)1 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ImmutableSet.toImmutableSet (com.google.common.collect.ImmutableSet.toImmutableSet)1 Sets.intersection (com.google.common.collect.Sets.intersection)1 Logger (io.airlift.log.Logger)1 Session (io.prestosql.Session)1 CachingCostProvider (io.prestosql.cost.CachingCostProvider)1 CachingStatsProvider (io.prestosql.cost.CachingStatsProvider)1 CostProvider (io.prestosql.cost.CostProvider)1