Search in sources :

Example 1 with JoinStatistics

use of io.trino.spi.connector.JoinStatistics in project trino by trinodb.

the class PushJoinIntoTableScan method getJoinStatistics.

private JoinStatistics getJoinStatistics(JoinNode join, TableScanNode left, TableScanNode right, Context context) {
    return new JoinStatistics() {

        @Override
        public Optional<BasicRelationStatistics> getLeftStatistics() {
            return getBasicRelationStats(left, left.getOutputSymbols(), context);
        }

        @Override
        public Optional<BasicRelationStatistics> getRightStatistics() {
            return getBasicRelationStats(right, right.getOutputSymbols(), context);
        }

        @Override
        public Optional<BasicRelationStatistics> getJoinStatistics() {
            return getBasicRelationStats(join, join.getOutputSymbols(), context);
        }

        private Optional<BasicRelationStatistics> getBasicRelationStats(PlanNode node, List<Symbol> outputSymbols, Context context) {
            PlanNodeStatsEstimate stats = context.getStatsProvider().getStats(node);
            TypeProvider types = context.getSymbolAllocator().getTypes();
            double outputRowCount = stats.getOutputRowCount();
            double outputSize = stats.getOutputSizeInBytes(outputSymbols, types);
            if (isNaN(outputRowCount) || isNaN(outputSize)) {
                return Optional.empty();
            }
            return Optional.of(new BasicRelationStatistics((long) outputRowCount, (long) outputSize));
        }
    };
}
Also used : JoinStatistics(io.trino.spi.connector.JoinStatistics) PlanNode(io.trino.sql.planner.plan.PlanNode) PlanNodeStatsEstimate(io.trino.cost.PlanNodeStatsEstimate) TypeProvider(io.trino.sql.planner.TypeProvider) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) BasicRelationStatistics(io.trino.spi.connector.BasicRelationStatistics)

Example 2 with JoinStatistics

use of io.trino.spi.connector.JoinStatistics in project trino by trinodb.

the class PushJoinIntoTableScan method apply.

@Override
public Result apply(JoinNode joinNode, Captures captures, Context context) {
    if (joinNode.isCrossJoin()) {
        return Result.empty();
    }
    TableScanNode left = captures.get(LEFT_TABLE_SCAN);
    TableScanNode right = captures.get(RIGHT_TABLE_SCAN);
    verify(!left.isUpdateTarget() && !right.isUpdateTarget(), "Unexpected Join over for-update table scan");
    Expression effectiveFilter = getEffectiveFilter(joinNode);
    FilterSplitResult filterSplitResult = splitFilter(effectiveFilter, left.getOutputSymbols(), right.getOutputSymbols(), context);
    if (!filterSplitResult.getRemainingFilter().equals(BooleanLiteral.TRUE_LITERAL)) {
        // TODO add extra filter node above join
        return Result.empty();
    }
    if (left.getEnforcedConstraint().isNone() || right.getEnforcedConstraint().isNone()) {
        // enforced constraint harder below.
        return Result.empty();
    }
    Map<String, ColumnHandle> leftAssignments = left.getAssignments().entrySet().stream().collect(toImmutableMap(entry -> entry.getKey().getName(), Map.Entry::getValue));
    Map<String, ColumnHandle> rightAssignments = right.getAssignments().entrySet().stream().collect(toImmutableMap(entry -> entry.getKey().getName(), Map.Entry::getValue));
    /*
         * We are (lazily) computing estimated statistics for join node and left and right table
         * and passing those to connector via applyJoin.
         *
         * There are a couple reasons for this approach:
         * - the engine knows how to estimate join and connector may not
         * - the engine may have cached stats for the table scans (within context.getStatsProvider()), so can be able to provide information more inexpensively
         * - in the future, the engine may be able to provide stats for table scan even in case when connector no longer can (see https://github.com/trinodb/trino/issues/6998)
         * - the pushdown feasibility assessment logic may be different (or configured differently) for different connectors/catalogs.
         */
    JoinStatistics joinStatistics = getJoinStatistics(joinNode, left, right, context);
    Optional<JoinApplicationResult<TableHandle>> joinApplicationResult = metadata.applyJoin(context.getSession(), getJoinType(joinNode), left.getTable(), right.getTable(), filterSplitResult.getPushableConditions(), // TODO we could pass only subset of assignments here, those which are needed to resolve filterSplitResult.getPushableConditions
    leftAssignments, rightAssignments, joinStatistics);
    if (joinApplicationResult.isEmpty()) {
        return Result.empty();
    }
    TableHandle handle = joinApplicationResult.get().getTableHandle();
    Map<ColumnHandle, ColumnHandle> leftColumnHandlesMapping = joinApplicationResult.get().getLeftColumnHandles();
    Map<ColumnHandle, ColumnHandle> rightColumnHandlesMapping = joinApplicationResult.get().getRightColumnHandles();
    ImmutableMap.Builder<Symbol, ColumnHandle> assignmentsBuilder = ImmutableMap.builder();
    assignmentsBuilder.putAll(left.getAssignments().entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, entry -> leftColumnHandlesMapping.get(entry.getValue()))));
    assignmentsBuilder.putAll(right.getAssignments().entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, entry -> rightColumnHandlesMapping.get(entry.getValue()))));
    Map<Symbol, ColumnHandle> assignments = assignmentsBuilder.buildOrThrow();
    // convert enforced constraint
    JoinNode.Type joinType = joinNode.getType();
    TupleDomain<ColumnHandle> leftConstraint = deriveConstraint(left.getEnforcedConstraint(), leftColumnHandlesMapping, joinType == RIGHT || joinType == FULL);
    TupleDomain<ColumnHandle> rightConstraint = deriveConstraint(right.getEnforcedConstraint(), rightColumnHandlesMapping, joinType == LEFT || joinType == FULL);
    TupleDomain<ColumnHandle> newEnforcedConstraint = TupleDomain.withColumnDomains(ImmutableMap.<ColumnHandle, Domain>builder().putAll(leftConstraint.getDomains().orElseThrow()).putAll(rightConstraint.getDomains().orElseThrow()).buildOrThrow());
    return Result.ofPlanNode(new ProjectNode(context.getIdAllocator().getNextId(), new TableScanNode(joinNode.getId(), handle, ImmutableList.copyOf(assignments.keySet()), assignments, newEnforcedConstraint, deriveTableStatisticsForPushdown(context.getStatsProvider(), context.getSession(), joinApplicationResult.get().isPrecalculateStatistics(), joinNode), false, Optional.empty()), Assignments.identity(joinNode.getOutputSymbols())));
}
Also used : PlanNode(io.trino.sql.planner.plan.PlanNode) LEFT(io.trino.sql.planner.plan.JoinNode.Type.LEFT) BooleanLiteral(io.trino.sql.tree.BooleanLiteral) Map(java.util.Map) JoinNode(io.trino.sql.planner.plan.JoinNode) TableScanNode(io.trino.sql.planner.plan.TableScanNode) PlanNodeStatsEstimate(io.trino.cost.PlanNodeStatsEstimate) Rules.deriveTableStatisticsForPushdown(io.trino.sql.planner.iterative.rule.Rules.deriveTableStatisticsForPushdown) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Domain(io.trino.spi.predicate.Domain) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Assignments(io.trino.sql.planner.plan.Assignments) Set(java.util.Set) BasicRelationStatistics(io.trino.spi.connector.BasicRelationStatistics) Patterns.tableScan(io.trino.sql.planner.plan.Patterns.tableScan) ComparisonExpression(io.trino.sql.tree.ComparisonExpression) List(java.util.List) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) Pattern(io.trino.matching.Pattern) SystemSessionProperties.isAllowPushdownIntoConnectors(io.trino.SystemSessionProperties.isAllowPushdownIntoConnectors) RIGHT(io.trino.sql.planner.plan.JoinNode.Type.RIGHT) SymbolReference(io.trino.sql.tree.SymbolReference) Optional(java.util.Optional) Expression(io.trino.sql.tree.Expression) ExpressionUtils.extractConjuncts(io.trino.sql.ExpressionUtils.extractConjuncts) Session(io.trino.Session) JoinCondition(io.trino.spi.connector.JoinCondition) Patterns(io.trino.sql.planner.plan.Patterns) Variable(io.trino.spi.expression.Variable) Capture.newCapture(io.trino.matching.Capture.newCapture) JoinStatistics(io.trino.spi.connector.JoinStatistics) ImmutableList(com.google.common.collect.ImmutableList) JoinType(io.trino.spi.connector.JoinType) Verify.verify(com.google.common.base.Verify.verify) Objects.requireNonNull(java.util.Objects.requireNonNull) ColumnHandle(io.trino.spi.connector.ColumnHandle) Rule(io.trino.sql.planner.iterative.Rule) Join.left(io.trino.sql.planner.plan.Patterns.Join.left) ProjectNode(io.trino.sql.planner.plan.ProjectNode) ExpressionUtils(io.trino.sql.ExpressionUtils) Symbol(io.trino.sql.planner.Symbol) FULL(io.trino.sql.planner.plan.JoinNode.Type.FULL) TupleDomain(io.trino.spi.predicate.TupleDomain) Capture(io.trino.matching.Capture) JoinApplicationResult(io.trino.spi.connector.JoinApplicationResult) TableHandle(io.trino.metadata.TableHandle) Join.right(io.trino.sql.planner.plan.Patterns.Join.right) ExpressionUtils.and(io.trino.sql.ExpressionUtils.and) Double.isNaN(java.lang.Double.isNaN) Captures(io.trino.matching.Captures) Metadata(io.trino.metadata.Metadata) TypeProvider(io.trino.sql.planner.TypeProvider) Domain.onlyNull(io.trino.spi.predicate.Domain.onlyNull) ColumnHandle(io.trino.spi.connector.ColumnHandle) Symbol(io.trino.sql.planner.Symbol) JoinNode(io.trino.sql.planner.plan.JoinNode) JoinApplicationResult(io.trino.spi.connector.JoinApplicationResult) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) JoinStatistics(io.trino.spi.connector.JoinStatistics) TableScanNode(io.trino.sql.planner.plan.TableScanNode) ComparisonExpression(io.trino.sql.tree.ComparisonExpression) Expression(io.trino.sql.tree.Expression) TableHandle(io.trino.metadata.TableHandle) ProjectNode(io.trino.sql.planner.plan.ProjectNode) Domain(io.trino.spi.predicate.Domain) TupleDomain(io.trino.spi.predicate.TupleDomain) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap)

Aggregations

ImmutableList (com.google.common.collect.ImmutableList)2 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)2 PlanNodeStatsEstimate (io.trino.cost.PlanNodeStatsEstimate)2 BasicRelationStatistics (io.trino.spi.connector.BasicRelationStatistics)2 JoinStatistics (io.trino.spi.connector.JoinStatistics)2 TypeProvider (io.trino.sql.planner.TypeProvider)2 PlanNode (io.trino.sql.planner.plan.PlanNode)2 List (java.util.List)2 Verify.verify (com.google.common.base.Verify.verify)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ImmutableMap.toImmutableMap (com.google.common.collect.ImmutableMap.toImmutableMap)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 Session (io.trino.Session)1 SystemSessionProperties.isAllowPushdownIntoConnectors (io.trino.SystemSessionProperties.isAllowPushdownIntoConnectors)1 Capture (io.trino.matching.Capture)1 Capture.newCapture (io.trino.matching.Capture.newCapture)1 Captures (io.trino.matching.Captures)1 Pattern (io.trino.matching.Pattern)1 Metadata (io.trino.metadata.Metadata)1 TableHandle (io.trino.metadata.TableHandle)1