Examples with StatsProvider com.facebook.presto.cost.StatsProvider used on opensource projects

Example 11 with StatsProvider

use of com.facebook.presto.cost.StatsProvider in project presto by prestodb.

the class RuntimeReorderJoinSides method apply.

@Override
public Result apply(JoinNode joinNode, Captures captures, Context context) {
    // Early exit if the leaves of the joinNode subtree include non tableScan nodes.
    if (searchFrom(joinNode, context.getLookup()).where(node -> node.getSources().isEmpty() && !(node instanceof TableScanNode)).matches()) {
        return Result.empty();
    }
    double leftOutputSizeInBytes = Double.NaN;
    double rightOutputSizeInBytes = Double.NaN;
    StatsProvider statsProvider = context.getStatsProvider();
    if (searchFrom(joinNode, context.getLookup()).where(node -> !(node instanceof TableScanNode) && !(node instanceof ExchangeNode)).findAll().size() == 1) {
        // Simple plan is characterized as Join directly on tableScanNodes only with exchangeNode in between.
        // For simple plans, directly fetch the overall table sizes as the size of the join sides to have
        // accurate input bytes statistics and meanwhile avoid non-negligible cost of collecting and processing
        // per-column statistics.
        leftOutputSizeInBytes = statsProvider.getStats(joinNode.getLeft()).getOutputSizeInBytes();
        rightOutputSizeInBytes = statsProvider.getStats(joinNode.getRight()).getOutputSizeInBytes();
    }
    if (Double.isNaN(leftOutputSizeInBytes) || Double.isNaN(rightOutputSizeInBytes)) {
        // Per-column estimate left and right output size for complex plans or when size statistics is unavailable.
        leftOutputSizeInBytes = statsProvider.getStats(joinNode.getLeft()).getOutputSizeInBytes(joinNode.getLeft().getOutputVariables());
        rightOutputSizeInBytes = statsProvider.getStats(joinNode.getRight()).getOutputSizeInBytes(joinNode.getRight().getOutputVariables());
    }
    if (Double.isNaN(leftOutputSizeInBytes) || Double.isNaN(rightOutputSizeInBytes)) {
        return Result.empty();
    }
    if (rightOutputSizeInBytes <= leftOutputSizeInBytes) {
        return Result.empty();
    }
    // Check if the swapped join is valid.
    if (!isSwappedJoinValid(joinNode)) {
        return Result.empty();
    }
    JoinNode swapped = joinNode.flipChildren();
    PlanNode newLeft = swapped.getLeft();
    Optional<VariableReferenceExpression> leftHashVariable = swapped.getLeftHashVariable();
    // Remove unnecessary LocalExchange in the current probe side. If the immediate left child (new probe side) of the join node
    // is a localExchange, there are two cases: an Exchange introduced by the current probe side (previous build side); or it is a UnionNode.
    // If the exchangeNode has more than 1 sources, it corresponds to the second case, otherwise it corresponds to the first case and could be safe to remove
    PlanNode resolvedSwappedLeft = context.getLookup().resolve(newLeft);
    if (resolvedSwappedLeft instanceof ExchangeNode && resolvedSwappedLeft.getSources().size() == 1) {
        // Ensure the new probe after skipping the local exchange will satisfy the required probe side property
        if (checkProbeSidePropertySatisfied(resolvedSwappedLeft.getSources().get(0), context)) {
            newLeft = resolvedSwappedLeft.getSources().get(0);
            // it as the leftHashVariable of the swapped join node.
            if (swapped.getLeftHashVariable().isPresent()) {
                int hashVariableIndex = resolvedSwappedLeft.getOutputVariables().indexOf(swapped.getLeftHashVariable().get());
                leftHashVariable = Optional.of(resolvedSwappedLeft.getSources().get(0).getOutputVariables().get(hashVariableIndex));
                // This is against typical iterativeOptimizer behavior and given this case is rare, just abort the swapping for this scenario.
                if (swapped.getOutputVariables().contains(swapped.getLeftHashVariable().get())) {
                    return Result.empty();
                }
            }
        }
    }
    // Add additional localExchange if the new build side does not satisfy the partitioning conditions.
    List<VariableReferenceExpression> buildJoinVariables = swapped.getCriteria().stream().map(JoinNode.EquiJoinClause::getRight).collect(toImmutableList());
    PlanNode newRight = swapped.getRight();
    if (!checkBuildSidePropertySatisfied(swapped.getRight(), buildJoinVariables, context)) {
        if (getTaskConcurrency(context.getSession()) > 1) {
            newRight = systemPartitionedExchange(context.getIdAllocator().getNextId(), LOCAL, swapped.getRight(), buildJoinVariables, swapped.getRightHashVariable());
        } else {
            newRight = gatheringExchange(context.getIdAllocator().getNextId(), LOCAL, swapped.getRight());
        }
    }
    JoinNode newJoinNode = new JoinNode(swapped.getSourceLocation(), swapped.getId(), swapped.getType(), newLeft, newRight, swapped.getCriteria(), swapped.getOutputVariables(), swapped.getFilter(), leftHashVariable, swapped.getRightHashVariable(), swapped.getDistributionType(), swapped.getDynamicFilters());
    log.debug(format("Probe size: %.2f is smaller than Build size: %.2f => invoke runtime join swapping on JoinNode ID: %s.", leftOutputSizeInBytes, rightOutputSizeInBytes, newJoinNode.getId()));
    return Result.ofPlanNode(newJoinNode);
}

Example 12 with StatsProvider

use of com.facebook.presto.cost.StatsProvider in project presto by prestodb.

the class PushPartialAggregationThroughExchange method partialAggregationNotUseful.

private boolean partialAggregationNotUseful(AggregationNode aggregationNode, ExchangeNode exchangeNode, Context context) {
    StatsProvider stats = context.getStatsProvider();
    PlanNodeStatsEstimate exchangeStats = stats.getStats(exchangeNode);
    PlanNodeStatsEstimate aggregationStats = stats.getStats(aggregationNode);
    double inputBytes = exchangeStats.getOutputSizeInBytes(exchangeNode.getOutputVariables());
    double outputBytes = aggregationStats.getOutputSizeInBytes(aggregationNode.getOutputVariables());
    double byteReductionThreshold = getPartialAggregationByteReductionThreshold(context.getSession());
    return exchangeStats.isConfident() && outputBytes > inputBytes * byteReductionThreshold;
}

Example 13 with StatsProvider

use of com.facebook.presto.cost.StatsProvider in project presto by prestodb.

the class LogicalPlanner method computeStats.

private StatsAndCosts computeStats(PlanNode root, TypeProvider types) {
    if (explain || isPrintStatsForNonJoinQuery(session) || PlanNodeSearcher.searchFrom(root).where(node -> (node instanceof JoinNode) || (node instanceof SemiJoinNode)).matches()) {
        StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, types);
        CostProvider costProvider = new CachingCostProvider(costCalculator, statsProvider, Optional.empty(), session);
        return StatsAndCosts.create(root, statsProvider, costProvider);
    }
    return StatsAndCosts.empty();
}

Example 14 with StatsProvider

use of com.facebook.presto.cost.StatsProvider in project presto by prestodb.

the class MarkDistinctMatcher method detailMatches.

@Override
public MatchResult detailMatches(PlanNode node, StatsProvider stats, Session session, Metadata metadata, SymbolAliases symbolAliases) {
    checkState(shapeMatches(node), "Plan testing framework error: shapeMatches returned false in detailMatches in %s", this.getClass().getName());
    MarkDistinctNode markDistinctNode = (MarkDistinctNode) node;
    if (!markDistinctNode.getHashVariable().map(variable -> new Symbol(variable.getName())).equals(hashSymbol.map(alias -> alias.toSymbol(symbolAliases)))) {
        return NO_MATCH;
    }
    if (!markDistinctNode.getDistinctVariables().stream().map(VariableReferenceExpression::getName).map(Symbol::new).collect(toImmutableSet()).equals(distinctSymbols.stream().map(alias -> alias.toSymbol(symbolAliases)).collect(toImmutableSet()))) {
        return NO_MATCH;
    }
    return match(markerSymbol.toString(), createSymbolReference(markDistinctNode.getMarkerVariable()));
}

Example 15 with StatsProvider

use of com.facebook.presto.cost.StatsProvider in project presto by prestodb.

the class JoinMatcher method detailMatches.

@Override
public MatchResult detailMatches(PlanNode node, StatsProvider stats, Session session, Metadata metadata, SymbolAliases symbolAliases) {
    checkState(shapeMatches(node), "Plan testing framework error: shapeMatches returned false in detailMatches in %s", this.getClass().getName());
    JoinNode joinNode = (JoinNode) node;
    if (joinNode.getCriteria().size() != equiCriteria.size()) {
        return NO_MATCH;
    }
    if (filter.isPresent()) {
        if (!joinNode.getFilter().isPresent()) {
            return NO_MATCH;
        }
        RowExpression expression = joinNode.getFilter().get();
        if (isExpression(expression)) {
            if (!new ExpressionVerifier(symbolAliases).process(castToExpression(expression), filter.get())) {
                return NO_MATCH;
            }
        } else {
            if (!new RowExpressionVerifier(symbolAliases, metadata, session).process(filter.get(), expression)) {
                return NO_MATCH;
            }
        }
    } else {
        if (joinNode.getFilter().isPresent()) {
            return NO_MATCH;
        }
    }
    if (distributionType.isPresent() && !distributionType.equals(joinNode.getDistributionType())) {
        return NO_MATCH;
    }
    /*
         * Have to use order-independent comparison; there are no guarantees what order
         * the equi criteria will have after planning and optimizing.
         */
    Set<List<String>> actual = joinNode.getCriteria().stream().map(criteria -> ImmutableList.of(criteria.getLeft().getName(), criteria.getRight().getName())).collect(toImmutableSet());
    Set<List<String>> expected = equiCriteria.stream().map(maker -> maker.getExpectedValue(symbolAliases)).map(criteria -> ImmutableList.of(criteria.getLeft().getName(), criteria.getRight().getName())).collect(toImmutableSet());
    if (!expected.equals(actual)) {
        return NO_MATCH;
    }
    if (dynamicFilter.isPresent() && !dynamicFilter.get().match(joinNode, symbolAliases).isMatch()) {
        return NO_MATCH;
    }
    return MatchResult.match();
}