Examples with JoinNode com.facebook.presto.sql.planner.plan.JoinNode used on opensource projects

Example 21 with JoinNode

use of com.facebook.presto.sql.planner.plan.JoinNode in project presto by prestodb.

the class PushAggregationThroughOuterJoin method apply.

@Override
public Result apply(AggregationNode aggregation, Captures captures, Context context) {
    JoinNode join = captures.get(JOIN);
    if (join.getFilter().isPresent() || !(join.getType() == JoinNode.Type.LEFT || join.getType() == JoinNode.Type.RIGHT) || !groupsOnAllColumns(aggregation, getOuterTable(join).getOutputVariables()) || !isDistinct(context.getLookup().resolve(getOuterTable(join)), context.getLookup()::resolve)) {
        return Result.empty();
    }
    List<VariableReferenceExpression> groupingKeys = join.getCriteria().stream().map(join.getType() == JoinNode.Type.RIGHT ? JoinNode.EquiJoinClause::getLeft : JoinNode.EquiJoinClause::getRight).collect(toImmutableList());
    AggregationNode rewrittenAggregation = new AggregationNode(aggregation.getSourceLocation(), aggregation.getId(), getInnerTable(join), aggregation.getAggregations(), singleGroupingSet(groupingKeys), ImmutableList.of(), aggregation.getStep(), aggregation.getHashVariable(), aggregation.getGroupIdVariable());
    JoinNode rewrittenJoin;
    if (join.getType() == JoinNode.Type.LEFT) {
        rewrittenJoin = new JoinNode(join.getSourceLocation(), join.getId(), join.getType(), join.getLeft(), rewrittenAggregation, join.getCriteria(), ImmutableList.<VariableReferenceExpression>builder().addAll(join.getLeft().getOutputVariables()).addAll(rewrittenAggregation.getAggregations().keySet()).build(), join.getFilter(), join.getLeftHashVariable(), join.getRightHashVariable(), join.getDistributionType(), join.getDynamicFilters());
    } else {
        rewrittenJoin = new JoinNode(join.getSourceLocation(), join.getId(), join.getType(), rewrittenAggregation, join.getRight(), join.getCriteria(), ImmutableList.<VariableReferenceExpression>builder().addAll(rewrittenAggregation.getAggregations().keySet()).addAll(join.getRight().getOutputVariables()).build(), join.getFilter(), join.getLeftHashVariable(), join.getRightHashVariable(), join.getDistributionType(), join.getDynamicFilters());
    }
    Optional<PlanNode> resultNode = coalesceWithNullAggregation(rewrittenAggregation, rewrittenJoin, context.getVariableAllocator(), context.getIdAllocator(), context.getLookup());
    if (!resultNode.isPresent()) {
        return Result.empty();
    }
    return Result.ofPlanNode(resultNode.get());
}

Example 22 with JoinNode

use of com.facebook.presto.sql.planner.plan.JoinNode in project presto by prestodb.

the class PushAggregationThroughOuterJoin method coalesceWithNullAggregation.

// When the aggregation is done after the join, there will be a null value that gets aggregated over
// where rows did not exist in the inner table.  For some aggregate functions, such as count, the result
// of an aggregation over a single null row is one or zero rather than null. In order to ensure correct results,
// we add a coalesce function with the output of the new outer join and the agggregation performed over a single
// null row.
private Optional<PlanNode> coalesceWithNullAggregation(AggregationNode aggregationNode, PlanNode outerJoin, PlanVariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, Lookup lookup) {
    // Create an aggregation node over a row of nulls.
    Optional<MappedAggregationInfo> aggregationOverNullInfoResultNode = createAggregationOverNull(aggregationNode, variableAllocator, idAllocator, lookup);
    if (!aggregationOverNullInfoResultNode.isPresent()) {
        return Optional.empty();
    }
    MappedAggregationInfo aggregationOverNullInfo = aggregationOverNullInfoResultNode.get();
    AggregationNode aggregationOverNull = aggregationOverNullInfo.getAggregation();
    Map<VariableReferenceExpression, VariableReferenceExpression> sourceAggregationToOverNullMapping = aggregationOverNullInfo.getVariableMapping();
    // Do a cross join with the aggregation over null
    JoinNode crossJoin = new JoinNode(outerJoin.getSourceLocation(), idAllocator.getNextId(), JoinNode.Type.INNER, outerJoin, aggregationOverNull, ImmutableList.of(), ImmutableList.<VariableReferenceExpression>builder().addAll(outerJoin.getOutputVariables()).addAll(aggregationOverNull.getOutputVariables()).build(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), ImmutableMap.of());
    // Add coalesce expressions for all aggregation functions
    Assignments.Builder assignmentsBuilder = Assignments.builder();
    for (VariableReferenceExpression variable : outerJoin.getOutputVariables()) {
        if (aggregationNode.getAggregations().keySet().contains(variable)) {
            assignmentsBuilder.put(variable, coalesce(ImmutableList.of(variable, sourceAggregationToOverNullMapping.get(variable))));
        } else {
            assignmentsBuilder.put(variable, variable);
        }
    }
    return Optional.of(new ProjectNode(idAllocator.getNextId(), crossJoin, assignmentsBuilder.build()));
}

Example 23 with JoinNode

use of com.facebook.presto.sql.planner.plan.JoinNode in project presto by prestodb.

the class RuntimeReorderJoinSides method apply.

@Override
public Result apply(JoinNode joinNode, Captures captures, Context context) {
    // Early exit if the leaves of the joinNode subtree include non tableScan nodes.
    if (searchFrom(joinNode, context.getLookup()).where(node -> node.getSources().isEmpty() && !(node instanceof TableScanNode)).matches()) {
        return Result.empty();
    }
    double leftOutputSizeInBytes = Double.NaN;
    double rightOutputSizeInBytes = Double.NaN;
    StatsProvider statsProvider = context.getStatsProvider();
    if (searchFrom(joinNode, context.getLookup()).where(node -> !(node instanceof TableScanNode) && !(node instanceof ExchangeNode)).findAll().size() == 1) {
        // Simple plan is characterized as Join directly on tableScanNodes only with exchangeNode in between.
        // For simple plans, directly fetch the overall table sizes as the size of the join sides to have
        // accurate input bytes statistics and meanwhile avoid non-negligible cost of collecting and processing
        // per-column statistics.
        leftOutputSizeInBytes = statsProvider.getStats(joinNode.getLeft()).getOutputSizeInBytes();
        rightOutputSizeInBytes = statsProvider.getStats(joinNode.getRight()).getOutputSizeInBytes();
    }
    if (Double.isNaN(leftOutputSizeInBytes) || Double.isNaN(rightOutputSizeInBytes)) {
        // Per-column estimate left and right output size for complex plans or when size statistics is unavailable.
        leftOutputSizeInBytes = statsProvider.getStats(joinNode.getLeft()).getOutputSizeInBytes(joinNode.getLeft().getOutputVariables());
        rightOutputSizeInBytes = statsProvider.getStats(joinNode.getRight()).getOutputSizeInBytes(joinNode.getRight().getOutputVariables());
    }
    if (Double.isNaN(leftOutputSizeInBytes) || Double.isNaN(rightOutputSizeInBytes)) {
        return Result.empty();
    }
    if (rightOutputSizeInBytes <= leftOutputSizeInBytes) {
        return Result.empty();
    }
    // Check if the swapped join is valid.
    if (!isSwappedJoinValid(joinNode)) {
        return Result.empty();
    }
    JoinNode swapped = joinNode.flipChildren();
    PlanNode newLeft = swapped.getLeft();
    Optional<VariableReferenceExpression> leftHashVariable = swapped.getLeftHashVariable();
    // Remove unnecessary LocalExchange in the current probe side. If the immediate left child (new probe side) of the join node
    // is a localExchange, there are two cases: an Exchange introduced by the current probe side (previous build side); or it is a UnionNode.
    // If the exchangeNode has more than 1 sources, it corresponds to the second case, otherwise it corresponds to the first case and could be safe to remove
    PlanNode resolvedSwappedLeft = context.getLookup().resolve(newLeft);
    if (resolvedSwappedLeft instanceof ExchangeNode && resolvedSwappedLeft.getSources().size() == 1) {
        // Ensure the new probe after skipping the local exchange will satisfy the required probe side property
        if (checkProbeSidePropertySatisfied(resolvedSwappedLeft.getSources().get(0), context)) {
            newLeft = resolvedSwappedLeft.getSources().get(0);
            // it as the leftHashVariable of the swapped join node.
            if (swapped.getLeftHashVariable().isPresent()) {
                int hashVariableIndex = resolvedSwappedLeft.getOutputVariables().indexOf(swapped.getLeftHashVariable().get());
                leftHashVariable = Optional.of(resolvedSwappedLeft.getSources().get(0).getOutputVariables().get(hashVariableIndex));
                // This is against typical iterativeOptimizer behavior and given this case is rare, just abort the swapping for this scenario.
                if (swapped.getOutputVariables().contains(swapped.getLeftHashVariable().get())) {
                    return Result.empty();
                }
            }
        }
    }
    // Add additional localExchange if the new build side does not satisfy the partitioning conditions.
    List<VariableReferenceExpression> buildJoinVariables = swapped.getCriteria().stream().map(JoinNode.EquiJoinClause::getRight).collect(toImmutableList());
    PlanNode newRight = swapped.getRight();
    if (!checkBuildSidePropertySatisfied(swapped.getRight(), buildJoinVariables, context)) {
        if (getTaskConcurrency(context.getSession()) > 1) {
            newRight = systemPartitionedExchange(context.getIdAllocator().getNextId(), LOCAL, swapped.getRight(), buildJoinVariables, swapped.getRightHashVariable());
        } else {
            newRight = gatheringExchange(context.getIdAllocator().getNextId(), LOCAL, swapped.getRight());
        }
    }
    JoinNode newJoinNode = new JoinNode(swapped.getSourceLocation(), swapped.getId(), swapped.getType(), newLeft, newRight, swapped.getCriteria(), swapped.getOutputVariables(), swapped.getFilter(), leftHashVariable, swapped.getRightHashVariable(), swapped.getDistributionType(), swapped.getDynamicFilters());
    log.debug(format("Probe size: %.2f is smaller than Build size: %.2f => invoke runtime join swapping on JoinNode ID: %s.", leftOutputSizeInBytes, rightOutputSizeInBytes, newJoinNode.getId()));
    return Result.ofPlanNode(newJoinNode);
}

Example 24 with JoinNode

use of com.facebook.presto.sql.planner.plan.JoinNode in project presto by prestodb.

the class PushLimitThroughOuterJoin method apply.

@Override
public Result apply(LimitNode parent, Captures captures, Context context) {
    if (!isPushLimitThroughOuterJoin(context.getSession())) {
        return Result.empty();
    }
    JoinNode joinNode = captures.get(CHILD);
    PlanNode left = joinNode.getLeft();
    PlanNode right = joinNode.getRight();
    if (joinNode.getType() == LEFT && !isLimited(left, context.getLookup(), parent.getCount())) {
        left = new LimitNode(parent.getSourceLocation(), context.getIdAllocator().getNextId(), left, parent.getCount(), PARTIAL);
    }
    if (joinNode.getType() == RIGHT && !isLimited(right, context.getLookup(), parent.getCount())) {
        right = new LimitNode(parent.getSourceLocation(), context.getIdAllocator().getNextId(), right, parent.getCount(), PARTIAL);
    }
    if (joinNode.getLeft() != left || joinNode.getRight() != right) {
        return Result.ofPlanNode(parent.replaceChildren(ImmutableList.of(joinNode.replaceChildren(ImmutableList.of(left, right)))));
    }
    return Result.empty();
}

Example 25 with JoinNode

use of com.facebook.presto.sql.planner.plan.JoinNode in project presto by prestodb.

the class PruneCrossJoinColumns method pushDownProjectOff.

@Override
protected Optional<PlanNode> pushDownProjectOff(PlanNodeIdAllocator idAllocator, PlanVariableAllocator variableAllocator, JoinNode joinNode, Set<VariableReferenceExpression> referencedOutputs) {
    Optional<PlanNode> newLeft = restrictOutputs(idAllocator, joinNode.getLeft(), referencedOutputs, false);
    Optional<PlanNode> newRight = restrictOutputs(idAllocator, joinNode.getRight(), referencedOutputs, false);
    if (!newLeft.isPresent() && !newRight.isPresent()) {
        return Optional.empty();
    }
    ImmutableList.Builder<VariableReferenceExpression> outputVariableBuilder = ImmutableList.builder();
    outputVariableBuilder.addAll(newLeft.orElse(joinNode.getLeft()).getOutputVariables());
    outputVariableBuilder.addAll(newRight.orElse(joinNode.getRight()).getOutputVariables());
    return Optional.of(new JoinNode(joinNode.getSourceLocation(), idAllocator.getNextId(), joinNode.getType(), newLeft.orElse(joinNode.getLeft()), newRight.orElse(joinNode.getRight()), joinNode.getCriteria(), outputVariableBuilder.build(), joinNode.getFilter(), joinNode.getLeftHashVariable(), joinNode.getRightHashVariable(), joinNode.getDistributionType(), joinNode.getDynamicFilters()));
}