use of io.prestosql.cost.PlanNodeStatsEstimate in project hetu-core by openlookeng.
the class TestReorderJoins method testReplicatesWhenNotRestricted.
@Test
public void testReplicatesWhenNotRestricted() {
// variable width so that average row size is respected
Type symbolType = createUnboundedVarcharType();
int aRows = 10_000;
int bRows = 10;
PlanNodeStatsEstimate probeSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000, 10))).build();
PlanNodeStatsEstimate buildSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000, 10))).build();
// B table is small enough to be replicated in AUTOMATIC_RESTRICTED mode
assertReorderJoins().setSystemProperty(JOIN_DISTRIBUTION_TYPE, AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "100MB").on(p -> {
Symbol a1 = p.symbol("A1", symbolType);
Symbol b1 = p.symbol("B1", symbolType);
return p.join(INNER, p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), ImmutableList.of(new EquiJoinClause(a1, b1)), ImmutableList.of(a1, b1), Optional.empty());
}).overrideStats("valuesA", probeSideStatsEstimate).overrideStats("valuesB", buildSideStatsEstimate).matches(join(INNER, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
probeSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000d * 10000, 10))).build();
buildSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000d * 10000, 10))).build();
// B table exceeds AUTOMATIC_RESTRICTED limit therefore it is partitioned
assertReorderJoins().setSystemProperty(JOIN_DISTRIBUTION_TYPE, AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "100MB").on(p -> {
Symbol a1 = p.symbol("A1", symbolType);
Symbol b1 = p.symbol("B1", symbolType);
return p.join(INNER, p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), ImmutableList.of(new EquiJoinClause(a1, b1)), ImmutableList.of(a1, b1), Optional.empty());
}).overrideStats("valuesA", probeSideStatsEstimate).overrideStats("valuesB", buildSideStatsEstimate).matches(join(INNER, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(PARTITIONED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
}
use of io.prestosql.cost.PlanNodeStatsEstimate in project hetu-core by openlookeng.
the class TestMemo method testEvictStatsOnReplace.
@Test
public void testEvictStatsOnReplace() {
PlanNode y = node();
PlanNode x = node(y);
Memo memo = new Memo(idAllocator, x);
int xGroup = memo.getRootGroup();
int yGroup = getChildGroup(memo, memo.getRootGroup());
PlanNodeStatsEstimate xStats = PlanNodeStatsEstimate.builder().setOutputRowCount(42).build();
PlanNodeStatsEstimate yStats = PlanNodeStatsEstimate.builder().setOutputRowCount(55).build();
memo.storeStats(yGroup, yStats);
memo.storeStats(xGroup, xStats);
assertEquals(memo.getStats(yGroup), Optional.of(yStats));
assertEquals(memo.getStats(xGroup), Optional.of(xStats));
memo.replace(yGroup, node(), "rule");
assertEquals(memo.getStats(yGroup), Optional.empty());
assertEquals(memo.getStats(xGroup), Optional.empty());
}
use of io.prestosql.cost.PlanNodeStatsEstimate in project hetu-core by openlookeng.
the class TestDetermineSemiJoinDistributionType method testReplicatesWhenNotRestricted.
@Test
public void testReplicatesWhenNotRestricted() {
// variable width so that average row size is respected
Type symbolType = createUnboundedVarcharType();
int aRows = 10_000;
int bRows = 10;
PlanNodeStatsEstimate probeSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000, 10))).build();
PlanNodeStatsEstimate buildSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000, 10))).build();
// B table is small enough to be replicated in AUTOMATIC_RESTRICTED mode
assertDetermineSemiJoinDistributionType().setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "100MB").overrideStats("valuesA", probeSideStatsEstimate).overrideStats("valuesB", buildSideStatsEstimate).on(p -> {
Symbol a1 = p.symbol("A1", symbolType);
Symbol b1 = p.symbol("B1", symbolType);
return p.semiJoin(p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), a1, b1, p.symbol("output"), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty());
}).matches(semiJoin("A1", "B1", "output", Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
probeSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000d * 10000, 10))).build();
buildSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000d * 10000, 10))).build();
// B table exceeds AUTOMATIC_RESTRICTED limit therefore it is partitioned
assertDetermineSemiJoinDistributionType().setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "100MB").overrideStats("valuesA", probeSideStatsEstimate).overrideStats("valuesB", buildSideStatsEstimate).on(p -> {
Symbol a1 = p.symbol("A1", symbolType);
Symbol b1 = p.symbol("B1", symbolType);
return p.semiJoin(p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), a1, b1, p.symbol("output"), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty());
}).matches(semiJoin("A1", "B1", "output", Optional.of(PARTITIONED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
}
use of io.prestosql.cost.PlanNodeStatsEstimate in project hetu-core by openlookeng.
the class TextRenderer method printEstimates.
private String printEstimates(PlanRepresentation plan, NodeRepresentation node) {
if (node.getEstimatedStats().stream().allMatch(PlanNodeStatsEstimate::isOutputRowCountUnknown) && node.getEstimatedCost().stream().allMatch(c -> c.equals(PlanCostEstimate.unknown()))) {
return "";
}
StringBuilder output = new StringBuilder();
int estimateCount = node.getEstimatedStats().size();
output.append("Estimates: ");
for (int i = 0; i < estimateCount; i++) {
PlanNodeStatsEstimate stats = node.getEstimatedStats().get(i);
PlanCostEstimate cost = node.getEstimatedCost().get(i);
List<Symbol> outputSymbols = node.getOutputs().stream().map(TypedSymbol::getSymbol).collect(toList());
output.append(format("{rows: %s (%s), cpu: %s, memory: %s, network: %s}", formatAsLong(stats.getOutputRowCount()), formatAsDataSize(stats.getOutputSizeInBytes(outputSymbols, plan.getTypes())), formatAsCpuCost(cost.getCpuCost()), formatAsDataSize(cost.getMaxMemory()), formatAsDataSize(cost.getNetworkCost())));
if (i < estimateCount - 1) {
output.append("/");
}
}
output.append("\n");
return output.toString();
}
use of io.prestosql.cost.PlanNodeStatsEstimate in project hetu-core by openlookeng.
the class TablePushdown method getNewIntermediateTreeAfterInnerTableUpdate.
/**
* @param newInnerJoinNode is the recently created join node after the outer table has been pushed into subquery
* @param stack is the stack of nodes from the original captured JoinNode and subquery TableScanNode
* @return the PlanNode after pulling the subquery table's Aggregation and Group By above the join
*/
private PlanNode getNewIntermediateTreeAfterInnerTableUpdate(JoinNode newInnerJoinNode, Stack<NodeWithTreeDirection> stack) {
TableScanNode subqueryTableNode = (TableScanNode) stack.peek().getNode();
/*
* create assignment builder for a new intermediate ProjectNode between the newInnerJoinNode and
* TableScanNode for subquery table
* */
Assignments.Builder assignmentsBuilder = Assignments.builder();
/*
* All symbols from TableScanNode are directly copied over
* */
for (Map.Entry<Symbol, ColumnHandle> tableEntry : subqueryTableNode.getAssignments().entrySet()) {
Symbol s = tableEntry.getKey();
assignmentsBuilder.put(s, castToRowExpression(new SymbolReference(s.getName())));
}
ProjectNode parentOfSubqueryTableNode = new ProjectNode(ruleContext.getIdAllocator().getNextId(), subqueryTableNode, assignmentsBuilder.build());
List<Symbol> parentOfSubqueryTableNodeOutputSymbols = parentOfSubqueryTableNode.getOutputSymbols();
/*
* Recreate the inner joinNode using the new ProjectNode as one of its sources.
* */
PlanNodeStatsEstimate leftSourceStats = ruleContext.getStatsProvider().getStats(newInnerJoinNode.getLeft());
PlanNodeStatsEstimate rightSourceStats = ruleContext.getStatsProvider().getStats(parentOfSubqueryTableNode);
JoinNode newJoinNode;
if (leftSourceStats.isOutputRowCountUnknown()) {
/*
* CAUTION: the stats are not available, so source reordering is not allowed. Query may fail
* */
newJoinNode = new JoinNode(newInnerJoinNode.getId(), newInnerJoinNode.getType(), newInnerJoinNode.getLeft(), parentOfSubqueryTableNode, newInnerJoinNode.getCriteria(), ImmutableList.<Symbol>builder().addAll(parentOfSubqueryTableNodeOutputSymbols).build(), newInnerJoinNode.getFilter(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), newInnerJoinNode.getDynamicFilters());
} else {
double leftSourceRowsCount = leftSourceStats.getOutputRowCount();
double rightSourceRowsCount = rightSourceStats.getOutputRowCount();
if (leftSourceRowsCount <= rightSourceRowsCount) {
// We reorder the children of this new join node such that the table with more rows is on the left
List<JoinNode.EquiJoinClause> newInnerJoinCriteria = newInnerJoinNode.getCriteria().stream().map(JoinNode.EquiJoinClause::flip).collect(toImmutableList());
newJoinNode = new JoinNode(newInnerJoinNode.getId(), newInnerJoinNode.getType(), parentOfSubqueryTableNode, newInnerJoinNode.getLeft(), newInnerJoinCriteria, ImmutableList.<Symbol>builder().addAll(parentOfSubqueryTableNodeOutputSymbols).build(), newInnerJoinNode.getFilter(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), newInnerJoinNode.getDynamicFilters());
} else {
newJoinNode = new JoinNode(newInnerJoinNode.getId(), newInnerJoinNode.getType(), newInnerJoinNode.getLeft(), parentOfSubqueryTableNode, newInnerJoinNode.getCriteria(), ImmutableList.<Symbol>builder().addAll(parentOfSubqueryTableNodeOutputSymbols).build(), newInnerJoinNode.getFilter(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), newInnerJoinNode.getDynamicFilters());
}
}
// Remove the TableScanNode from the stack
stack.pop();
AggregationNode newAggNode = (AggregationNode) stack.peek().getNode().replaceChildren(ImmutableList.of(newJoinNode));
return stack.firstElement().getNode().replaceChildren(ImmutableList.of(newAggNode));
}
Aggregations