use of com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT in project presto by prestodb.
the class TestDetermineJoinDistributionType method testChoosesLeftWhenCriteriaEmpty.
@Test
public void testChoosesLeftWhenCriteriaEmpty() {
int aRows = 1_000__00;
int bRows = 1_0;
assertDetermineJoinDistributionType(new CostComparator(75, 10, 15)).setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "1PB").overrideStats("valuesA", PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "A1", BIGINT), new VariableStatsEstimate(0, 100, 0, 640000, 100))).build()).overrideStats("valuesB", PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "B1", BIGINT), new VariableStatsEstimate(0, 100, 0, 640000, 100))).build()).on(p -> p.join(RIGHT, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), ImmutableList.of(), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())).matches(join(LEFT, ImmutableList.of(), Optional.empty(), Optional.of(REPLICATED), values(ImmutableMap.of("B1", 0)), values(ImmutableMap.of("A1", 0))));
}
use of com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT in project presto by prestodb.
the class TestDetermineJoinDistributionType method testFlipAndReplicateRightOuterJoinWhenJoinCardinalityUnknown.
@Test
public void testFlipAndReplicateRightOuterJoinWhenJoinCardinalityUnknown() {
int aRows = 10;
int bRows = 1_000_000;
assertDetermineJoinDistributionType(new CostComparator(75, 10, 15)).setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).overrideStats("valuesA", PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "A1", BIGINT), VariableStatsEstimate.unknown())).build()).overrideStats("valuesB", PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "B1", BIGINT), VariableStatsEstimate.unknown())).build()).on(p -> p.join(RIGHT, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())).matches(join(LEFT, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
}
use of com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT in project presto by prestodb.
the class RuntimeReorderJoinSides method apply.
@Override
public Result apply(JoinNode joinNode, Captures captures, Context context) {
// Early exit if the leaves of the joinNode subtree include non tableScan nodes.
if (searchFrom(joinNode, context.getLookup()).where(node -> node.getSources().isEmpty() && !(node instanceof TableScanNode)).matches()) {
return Result.empty();
}
double leftOutputSizeInBytes = Double.NaN;
double rightOutputSizeInBytes = Double.NaN;
StatsProvider statsProvider = context.getStatsProvider();
if (searchFrom(joinNode, context.getLookup()).where(node -> !(node instanceof TableScanNode) && !(node instanceof ExchangeNode)).findAll().size() == 1) {
// Simple plan is characterized as Join directly on tableScanNodes only with exchangeNode in between.
// For simple plans, directly fetch the overall table sizes as the size of the join sides to have
// accurate input bytes statistics and meanwhile avoid non-negligible cost of collecting and processing
// per-column statistics.
leftOutputSizeInBytes = statsProvider.getStats(joinNode.getLeft()).getOutputSizeInBytes();
rightOutputSizeInBytes = statsProvider.getStats(joinNode.getRight()).getOutputSizeInBytes();
}
if (Double.isNaN(leftOutputSizeInBytes) || Double.isNaN(rightOutputSizeInBytes)) {
// Per-column estimate left and right output size for complex plans or when size statistics is unavailable.
leftOutputSizeInBytes = statsProvider.getStats(joinNode.getLeft()).getOutputSizeInBytes(joinNode.getLeft().getOutputVariables());
rightOutputSizeInBytes = statsProvider.getStats(joinNode.getRight()).getOutputSizeInBytes(joinNode.getRight().getOutputVariables());
}
if (Double.isNaN(leftOutputSizeInBytes) || Double.isNaN(rightOutputSizeInBytes)) {
return Result.empty();
}
if (rightOutputSizeInBytes <= leftOutputSizeInBytes) {
return Result.empty();
}
// Check if the swapped join is valid.
if (!isSwappedJoinValid(joinNode)) {
return Result.empty();
}
JoinNode swapped = joinNode.flipChildren();
PlanNode newLeft = swapped.getLeft();
Optional<VariableReferenceExpression> leftHashVariable = swapped.getLeftHashVariable();
// Remove unnecessary LocalExchange in the current probe side. If the immediate left child (new probe side) of the join node
// is a localExchange, there are two cases: an Exchange introduced by the current probe side (previous build side); or it is a UnionNode.
// If the exchangeNode has more than 1 sources, it corresponds to the second case, otherwise it corresponds to the first case and could be safe to remove
PlanNode resolvedSwappedLeft = context.getLookup().resolve(newLeft);
if (resolvedSwappedLeft instanceof ExchangeNode && resolvedSwappedLeft.getSources().size() == 1) {
// Ensure the new probe after skipping the local exchange will satisfy the required probe side property
if (checkProbeSidePropertySatisfied(resolvedSwappedLeft.getSources().get(0), context)) {
newLeft = resolvedSwappedLeft.getSources().get(0);
// it as the leftHashVariable of the swapped join node.
if (swapped.getLeftHashVariable().isPresent()) {
int hashVariableIndex = resolvedSwappedLeft.getOutputVariables().indexOf(swapped.getLeftHashVariable().get());
leftHashVariable = Optional.of(resolvedSwappedLeft.getSources().get(0).getOutputVariables().get(hashVariableIndex));
// This is against typical iterativeOptimizer behavior and given this case is rare, just abort the swapping for this scenario.
if (swapped.getOutputVariables().contains(swapped.getLeftHashVariable().get())) {
return Result.empty();
}
}
}
}
// Add additional localExchange if the new build side does not satisfy the partitioning conditions.
List<VariableReferenceExpression> buildJoinVariables = swapped.getCriteria().stream().map(JoinNode.EquiJoinClause::getRight).collect(toImmutableList());
PlanNode newRight = swapped.getRight();
if (!checkBuildSidePropertySatisfied(swapped.getRight(), buildJoinVariables, context)) {
if (getTaskConcurrency(context.getSession()) > 1) {
newRight = systemPartitionedExchange(context.getIdAllocator().getNextId(), LOCAL, swapped.getRight(), buildJoinVariables, swapped.getRightHashVariable());
} else {
newRight = gatheringExchange(context.getIdAllocator().getNextId(), LOCAL, swapped.getRight());
}
}
JoinNode newJoinNode = new JoinNode(swapped.getSourceLocation(), swapped.getId(), swapped.getType(), newLeft, newRight, swapped.getCriteria(), swapped.getOutputVariables(), swapped.getFilter(), leftHashVariable, swapped.getRightHashVariable(), swapped.getDistributionType(), swapped.getDynamicFilters());
log.debug(format("Probe size: %.2f is smaller than Build size: %.2f => invoke runtime join swapping on JoinNode ID: %s.", leftOutputSizeInBytes, rightOutputSizeInBytes, newJoinNode.getId()));
return Result.ofPlanNode(newJoinNode);
}
use of com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT in project presto by prestodb.
the class TestDetermineJoinDistributionType method testChoosesRightWhenFallsBackToSyntactic.
@Test
public void testChoosesRightWhenFallsBackToSyntactic() {
int aRows = 1_000__00;
int bRows = 1_0;
assertDetermineJoinDistributionType(new CostComparator(75, 10, 15)).setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "100MB").overrideStats("valuesA", PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "A1", BIGINT), new VariableStatsEstimate(0, 100, 0, 640000, 100))).build()).overrideStats("valuesB", PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "B1", BIGINT), new VariableStatsEstimate(0, 100, 0, 640000, 100))).build()).on(p -> p.join(RIGHT, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), ImmutableList.of(), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())).matches(join(RIGHT, ImmutableList.of(), Optional.empty(), Optional.of(PARTITIONED), values(ImmutableMap.of("B1", 0)), values(ImmutableMap.of("A1", 0))));
}
use of com.facebook.presto.sql.planner.plan.JoinNode.Type.RIGHT in project presto by prestodb.
the class TestDetermineJoinDistributionType method testFlipAndReplicateRightOuterJoin.
@Test
public void testFlipAndReplicateRightOuterJoin() {
int aRows = 10;
int bRows = 1_000_000;
assertDetermineJoinDistributionType(new CostComparator(75, 10, 15)).setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).overrideStats("valuesA", PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "A1", BIGINT), new VariableStatsEstimate(0, 100, 0, 640000, 100))).build()).overrideStats("valuesB", PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "B1", BIGINT), new VariableStatsEstimate(0, 100, 0, 640000, 100))).build()).on(p -> p.join(RIGHT, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())).matches(join(LEFT, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
}
Aggregations