use of com.facebook.presto.spi.plan.TableScanNode in project presto by prestodb.
the class RuntimeReorderJoinSides method apply.
@Override
public Result apply(JoinNode joinNode, Captures captures, Context context) {
// Early exit if the leaves of the joinNode subtree include non tableScan nodes.
if (searchFrom(joinNode, context.getLookup()).where(node -> node.getSources().isEmpty() && !(node instanceof TableScanNode)).matches()) {
return Result.empty();
}
double leftOutputSizeInBytes = Double.NaN;
double rightOutputSizeInBytes = Double.NaN;
StatsProvider statsProvider = context.getStatsProvider();
if (searchFrom(joinNode, context.getLookup()).where(node -> !(node instanceof TableScanNode) && !(node instanceof ExchangeNode)).findAll().size() == 1) {
// Simple plan is characterized as Join directly on tableScanNodes only with exchangeNode in between.
// For simple plans, directly fetch the overall table sizes as the size of the join sides to have
// accurate input bytes statistics and meanwhile avoid non-negligible cost of collecting and processing
// per-column statistics.
leftOutputSizeInBytes = statsProvider.getStats(joinNode.getLeft()).getOutputSizeInBytes();
rightOutputSizeInBytes = statsProvider.getStats(joinNode.getRight()).getOutputSizeInBytes();
}
if (Double.isNaN(leftOutputSizeInBytes) || Double.isNaN(rightOutputSizeInBytes)) {
// Per-column estimate left and right output size for complex plans or when size statistics is unavailable.
leftOutputSizeInBytes = statsProvider.getStats(joinNode.getLeft()).getOutputSizeInBytes(joinNode.getLeft().getOutputVariables());
rightOutputSizeInBytes = statsProvider.getStats(joinNode.getRight()).getOutputSizeInBytes(joinNode.getRight().getOutputVariables());
}
if (Double.isNaN(leftOutputSizeInBytes) || Double.isNaN(rightOutputSizeInBytes)) {
return Result.empty();
}
if (rightOutputSizeInBytes <= leftOutputSizeInBytes) {
return Result.empty();
}
// Check if the swapped join is valid.
if (!isSwappedJoinValid(joinNode)) {
return Result.empty();
}
JoinNode swapped = joinNode.flipChildren();
PlanNode newLeft = swapped.getLeft();
Optional<VariableReferenceExpression> leftHashVariable = swapped.getLeftHashVariable();
// Remove unnecessary LocalExchange in the current probe side. If the immediate left child (new probe side) of the join node
// is a localExchange, there are two cases: an Exchange introduced by the current probe side (previous build side); or it is a UnionNode.
// If the exchangeNode has more than 1 sources, it corresponds to the second case, otherwise it corresponds to the first case and could be safe to remove
PlanNode resolvedSwappedLeft = context.getLookup().resolve(newLeft);
if (resolvedSwappedLeft instanceof ExchangeNode && resolvedSwappedLeft.getSources().size() == 1) {
// Ensure the new probe after skipping the local exchange will satisfy the required probe side property
if (checkProbeSidePropertySatisfied(resolvedSwappedLeft.getSources().get(0), context)) {
newLeft = resolvedSwappedLeft.getSources().get(0);
// it as the leftHashVariable of the swapped join node.
if (swapped.getLeftHashVariable().isPresent()) {
int hashVariableIndex = resolvedSwappedLeft.getOutputVariables().indexOf(swapped.getLeftHashVariable().get());
leftHashVariable = Optional.of(resolvedSwappedLeft.getSources().get(0).getOutputVariables().get(hashVariableIndex));
// This is against typical iterativeOptimizer behavior and given this case is rare, just abort the swapping for this scenario.
if (swapped.getOutputVariables().contains(swapped.getLeftHashVariable().get())) {
return Result.empty();
}
}
}
}
// Add additional localExchange if the new build side does not satisfy the partitioning conditions.
List<VariableReferenceExpression> buildJoinVariables = swapped.getCriteria().stream().map(JoinNode.EquiJoinClause::getRight).collect(toImmutableList());
PlanNode newRight = swapped.getRight();
if (!checkBuildSidePropertySatisfied(swapped.getRight(), buildJoinVariables, context)) {
if (getTaskConcurrency(context.getSession()) > 1) {
newRight = systemPartitionedExchange(context.getIdAllocator().getNextId(), LOCAL, swapped.getRight(), buildJoinVariables, swapped.getRightHashVariable());
} else {
newRight = gatheringExchange(context.getIdAllocator().getNextId(), LOCAL, swapped.getRight());
}
}
JoinNode newJoinNode = new JoinNode(swapped.getSourceLocation(), swapped.getId(), swapped.getType(), newLeft, newRight, swapped.getCriteria(), swapped.getOutputVariables(), swapped.getFilter(), leftHashVariable, swapped.getRightHashVariable(), swapped.getDistributionType(), swapped.getDynamicFilters());
log.debug(format("Probe size: %.2f is smaller than Build size: %.2f => invoke runtime join swapping on JoinNode ID: %s.", leftOutputSizeInBytes, rightOutputSizeInBytes, newJoinNode.getId()));
return Result.ofPlanNode(newJoinNode);
}
use of com.facebook.presto.spi.plan.TableScanNode in project presto by prestodb.
the class TestCostCalculator method testMemoryCostJoinAboveJoin.
@Test
public void testMemoryCostJoinAboveJoin() {
// join
// / \
// ts1 join23
// / \
// ts2 ts3
TableScanNode ts1 = tableScan("ts1", "key1");
TableScanNode ts2 = tableScan("ts2", "key2");
TableScanNode ts3 = tableScan("ts3", "key3");
JoinNode join23 = join("join23", ts2, ts3, JoinNode.DistributionType.PARTITIONED, "key2", "key3");
JoinNode join = join("join", ts1, join23, JoinNode.DistributionType.PARTITIONED, "key1", "key2");
Map<String, PlanCostEstimate> costs = ImmutableMap.of("ts1", new PlanCostEstimate(0, 128, 128, 0), "ts2", new PlanCostEstimate(0, 64, 64, 0), "ts3", new PlanCostEstimate(0, 32, 32, 0));
Map<String, PlanNodeStatsEstimate> stats = ImmutableMap.of("join", statsEstimate(join, 10_000), "join23", statsEstimate(join23, 2_000), "ts1", statsEstimate(ts1, 10_000), "ts2", statsEstimate(ts2, 1_000), "ts3", statsEstimate(ts3, 100));
Map<String, Type> types = ImmutableMap.of("key1", BIGINT, "key2", BIGINT, "key3", BIGINT);
assertCost(join23, costs, stats).memory(// join23 memory footprint
100 * IS_NULL_OVERHEAD + 64 + // ts2, ts3 memory footprint
32).memoryWhenOutputting(// join23 memory footprint
100 * IS_NULL_OVERHEAD + // ts2 memory footprint
64);
assertCost(join, costs, stats).memory(// join memory footprint
2000 * IS_NULL_OVERHEAD + 100 * IS_NULL_OVERHEAD + // join23 total memory when outputting
64 + // ts1 memory footprint
128).memoryWhenOutputting(// join memory footprint
2000 * IS_NULL_OVERHEAD + // ts1 memory footprint
128);
assertCostEstimatedExchanges(join23, costs, stats).memory(// join23 memory footprint
100 * IS_NULL_OVERHEAD + 64 + // ts2, ts3 memory footprint
32).memoryWhenOutputting(// join23 memory footprint
100 * IS_NULL_OVERHEAD + // ts2 memory footprint
64);
assertCostEstimatedExchanges(join, costs, stats).memory(// join memory footprint
2000 * IS_NULL_OVERHEAD + 100 * IS_NULL_OVERHEAD + // join23 total memory when outputting
64 + // ts1 memory footprint
128).memoryWhenOutputting(// join memory footprint
2000 * IS_NULL_OVERHEAD + // ts1 memory footprint
128);
assertCostSingleStageFragmentedPlan(join23, costs, stats, types).memory(// join23 memory footprint
100 * IS_NULL_OVERHEAD + 64 + // ts2, ts3 memory footprint
32).memoryWhenOutputting(// join23 memory footprint
100 * IS_NULL_OVERHEAD + // ts2 memory footprint
64);
assertCostSingleStageFragmentedPlan(join, costs, stats, types).memory(// join memory footprint
2000 * IS_NULL_OVERHEAD + 100 * IS_NULL_OVERHEAD + // join23 total memory when outputting
64 + // ts1 memory footprint
128).memoryWhenOutputting(// join memory footprint
2000 * IS_NULL_OVERHEAD + // ts1 memory footprint
128);
}
use of com.facebook.presto.spi.plan.TableScanNode in project presto by prestodb.
the class TestCostCalculator method testProject.
@Test
public void testProject() {
TableScanNode tableScan = tableScan("ts", "orderkey");
PlanNode project = project("project", tableScan, new VariableReferenceExpression(Optional.empty(), "string", VARCHAR), new Cast(new SymbolReference("orderkey"), "VARCHAR"));
Map<String, PlanCostEstimate> costs = ImmutableMap.of("ts", cpuCost(1000));
Map<String, PlanNodeStatsEstimate> stats = ImmutableMap.of("project", statsEstimate(project, 4000), "ts", statsEstimate(tableScan, 1000));
Map<String, Type> types = ImmutableMap.of("orderkey", BIGINT, "string", VARCHAR);
assertCost(project, costs, stats).cpu(1000 + 4000 * OFFSET_AND_IS_NULL_OVERHEAD).memory(0).network(0);
assertCostEstimatedExchanges(project, costs, stats).cpu(1000 + 4000 * OFFSET_AND_IS_NULL_OVERHEAD).memory(0).network(0);
assertCostSingleStageFragmentedPlan(project, costs, stats, types).cpu(1000 + 4000 * OFFSET_AND_IS_NULL_OVERHEAD).memory(0).network(0);
assertCostHasUnknownComponentsForUnknownStats(project);
}
use of com.facebook.presto.spi.plan.TableScanNode in project presto by prestodb.
the class TestCostCalculator method testRepartitionedJoin.
@Test
public void testRepartitionedJoin() {
TableScanNode ts1 = tableScan("ts1", "orderkey");
TableScanNode ts2 = tableScan("ts2", "orderkey_0");
JoinNode join = join("join", ts1, ts2, JoinNode.DistributionType.PARTITIONED, "orderkey", "orderkey_0");
Map<String, PlanCostEstimate> costs = ImmutableMap.of("ts1", cpuCost(6000), "ts2", cpuCost(1000));
Map<String, PlanNodeStatsEstimate> stats = ImmutableMap.of("join", statsEstimate(join, 12000), "ts1", statsEstimate(ts1, 6000), "ts2", statsEstimate(ts2, 1000));
Map<String, Type> types = ImmutableMap.of("orderkey", BIGINT, "orderkey_0", BIGINT);
assertCost(join, costs, stats).cpu(6000 + 1000 + (12000 + 6000 + 1000) * IS_NULL_OVERHEAD).memory(1000 * IS_NULL_OVERHEAD).network(0);
assertCostEstimatedExchanges(join, costs, stats).cpu(6000 + 1000 + (12000 + 6000 + 1000 + 6000 + 1000 + 1000) * IS_NULL_OVERHEAD).memory(1000 * IS_NULL_OVERHEAD).network((6000 + 1000) * IS_NULL_OVERHEAD);
assertCostSingleStageFragmentedPlan(join, costs, stats, types).cpu(6000 + 1000 + (12000 + 6000 + 1000) * IS_NULL_OVERHEAD).memory(1000 * IS_NULL_OVERHEAD).network(0);
assertCostHasUnknownComponentsForUnknownStats(join);
}
use of com.facebook.presto.spi.plan.TableScanNode in project presto by prestodb.
the class TestCostCalculator method testRepartitionedJoinWithExchange.
@Test
public void testRepartitionedJoinWithExchange() {
TableScanNode ts1 = tableScan("ts1", "orderkey");
TableScanNode ts2 = tableScan("ts2", "orderkey_0");
PlanNode p1 = project("p1", ts1, variable("orderkey_1", BIGINT), new SymbolReference("orderkey"));
ExchangeNode remoteExchange1 = systemPartitionedExchange(new PlanNodeId("re1"), REMOTE_STREAMING, p1, ImmutableList.of(new VariableReferenceExpression(Optional.empty(), "orderkey_1", BIGINT)), Optional.empty());
ExchangeNode remoteExchange2 = systemPartitionedExchange(new PlanNodeId("re2"), REMOTE_STREAMING, ts2, ImmutableList.of(new VariableReferenceExpression(Optional.empty(), "orderkey_0", BIGINT)), Optional.empty());
ExchangeNode localExchange = systemPartitionedExchange(new PlanNodeId("le"), LOCAL, remoteExchange2, ImmutableList.of(new VariableReferenceExpression(Optional.empty(), "orderkey_0", BIGINT)), Optional.empty());
JoinNode join = join("join", remoteExchange1, localExchange, JoinNode.DistributionType.PARTITIONED, "orderkey_1", "orderkey_0");
Map<String, PlanNodeStatsEstimate> stats = ImmutableMap.<String, PlanNodeStatsEstimate>builder().put("join", statsEstimate(join, 12000)).put("re1", statsEstimate(remoteExchange1, 10000)).put("re2", statsEstimate(remoteExchange2, 10000)).put("le", statsEstimate(localExchange, 6000)).put("p1", statsEstimate(p1, 6000)).put("ts1", statsEstimate(ts1, 6000)).put("ts2", statsEstimate(ts2, 1000)).build();
Map<String, Type> types = ImmutableMap.of("orderkey", BIGINT, "orderkey_1", BIGINT, "orderkey_0", BIGINT);
assertFragmentedEqualsUnfragmented(join, stats, types);
}
Aggregations