use of org.voltdb.plannodes.HashAggregatePlanNode in project voltdb by VoltDB.
the class TestPlansDistinct method checkDistinctWithGroupbyPlans.
/**
*
* @param distinctSQL Group by query with distinct
* @param groupbySQL Group by query without distinct
*/
protected void checkDistinctWithGroupbyPlans(String distinctSQL, String groupbySQL, boolean limitPushdown) {
List<AbstractPlanNode> pns1 = compileToFragments(distinctSQL);
List<AbstractPlanNode> pns2 = compileToFragments(groupbySQL);
//printExplainPlan(pns1);
//printExplainPlan(pns2);
assertTrue(pns1.get(0) instanceof SendPlanNode);
assertTrue(pns2.get(0) instanceof SendPlanNode);
AbstractPlanNode apn1, apn2;
apn1 = pns1.get(0).getChild(0);
apn2 = pns2.get(0).getChild(0);
boolean hasTopProjection1 = false;
if (apn1 instanceof ProjectionPlanNode) {
apn1 = apn1.getChild(0);
hasTopProjection1 = true;
}
boolean hasTopProjection2 = false;
if (apn2 instanceof ProjectionPlanNode) {
apn2 = apn2.getChild(0);
hasTopProjection2 = true;
}
// DISTINCT plan node is rewrote with GROUP BY and adds above the original GROUP BY node
// there may be another projection node in between for complex aggregation case
boolean hasOrderby = false, hasLimit = false;
boolean groupByMergeReceive = false;
// infer the ORDERBY/LIMIT information from the base line query
if (apn2 instanceof OrderByPlanNode) {
hasOrderby = true;
if (apn2.getInlinePlanNode(PlanNodeType.LIMIT) != null) {
hasLimit = true;
}
apn2 = apn2.getChild(0);
} else if (apn2 instanceof LimitPlanNode) {
hasLimit = true;
apn2 = apn2.getChild(0);
} else if (apn2 instanceof MergeReceivePlanNode) {
assertTrue(apn2.getInlinePlanNode(PlanNodeType.ORDERBY) != null);
hasOrderby = true;
hasLimit = apn2.getInlinePlanNode(PlanNodeType.LIMIT) != null;
groupByMergeReceive = true;
}
// check the DISTINCT query plan
boolean distinctMergeReceive = false;
if (hasOrderby) {
if (apn1 instanceof OrderByPlanNode) {
assertTrue(apn1 instanceof OrderByPlanNode);
if (hasLimit) {
// check inline limit
assertNotNull(apn1.getInlinePlanNode(PlanNodeType.LIMIT));
}
apn1 = apn1.getChild(0);
} else if (apn1 instanceof MergeReceivePlanNode) {
distinctMergeReceive = true;
assertNotNull(apn1.getInlinePlanNode(PlanNodeType.ORDERBY));
assertEquals(0, apn1.getChildCount());
} else {
fail("The distinctSQL top node is not OrderBy or MergeReceive.");
}
} else if (hasLimit) {
assertTrue(apn1 instanceof LimitPlanNode);
apn1 = apn1.getChild(0);
}
// Check DISTINCT group by plan node
if (distinctMergeReceive) {
AbstractPlanNode aggr = AggregatePlanNode.getInlineAggregationNode(apn1);
assertTrue(aggr instanceof AggregatePlanNode);
assertEquals(0, ((AggregatePlanNode) aggr).getAggregateTypesSize());
assertEquals(pns1.get(0).getOutputSchema().getColumns().size(), ((AggregatePlanNode) aggr).getGroupByExpressionsSize());
if (hasLimit) {
// check inline limit
assertNotNull(aggr.getInlinePlanNode(PlanNodeType.LIMIT));
}
} else {
assertTrue(apn1 instanceof HashAggregatePlanNode);
assertEquals(0, ((HashAggregatePlanNode) apn1).getAggregateTypesSize());
assertEquals(pns1.get(0).getOutputSchema().getColumns().size(), ((HashAggregatePlanNode) apn1).getGroupByExpressionsSize());
apn1 = apn1.getChild(0);
}
// check projection node for complex aggregation case
if (apn1 instanceof ProjectionPlanNode) {
apn1 = apn1.getChild(0);
assertFalse(hasTopProjection1);
}
if (apn2 instanceof ProjectionPlanNode) {
apn2 = apn2.getChild(0);
assertFalse(hasTopProjection2);
}
// check the rest plan nodes.
if (distinctMergeReceive == false && groupByMergeReceive == false) {
assertEquals(apn1.toExplainPlanString(), apn2.toExplainPlanString());
} else if (distinctMergeReceive == true && groupByMergeReceive == true) {
// In case of applied MergeReceive optimization the apn1 and apn2 nodes
// should not have any children
assertEquals(0, apn1.getChildCount());
assertEquals(0, apn2.getChildCount());
}
// Distributed DISTINCT GROUP BY
if (pns1.size() > 1) {
if (!limitPushdown) {
assertEquals(pns1.get(1).toExplainPlanString(), pns2.get(1).toExplainPlanString());
return;
}
assertTrue(pns1.get(1) instanceof SendPlanNode);
assertTrue(pns2.get(1) instanceof SendPlanNode);
apn1 = pns1.get(1).getChild(0);
apn2 = pns2.get(1).getChild(0);
// ignore the ORDER BY/LIMIT pushdown plan node
// because DISTINCT case can not be pushed down
assertTrue(apn2 instanceof OrderByPlanNode);
assertNotNull(apn2.getInlinePlanNode(PlanNodeType.LIMIT));
apn2 = apn2.getChild(0);
// winners may produce completely different paths.
if (distinctMergeReceive == false && groupByMergeReceive == false) {
assertEquals(apn1.toExplainPlanString(), apn2.toExplainPlanString());
}
}
}
use of org.voltdb.plannodes.HashAggregatePlanNode in project voltdb by VoltDB.
the class TestPlansGroupBy method testDistinctA1_Subquery.
public void testDistinctA1_Subquery() {
AbstractPlanNode p;
List<AbstractPlanNode> pns;
// Distinct rewrote with group by
pns = compileToFragments("select * from (SELECT DISTINCT A1 FROM T1) temp");
p = pns.get(0).getChild(0);
assertTrue(p instanceof SeqScanPlanNode);
assertTrue(p.getChild(0) instanceof HashAggregatePlanNode);
assertTrue(p.getChild(0).getChild(0) instanceof ReceivePlanNode);
p = pns.get(1).getChild(0);
assertTrue(p instanceof AbstractScanPlanNode);
assertNotNull(p.getInlinePlanNode(PlanNodeType.HASHAGGREGATE));
}
use of org.voltdb.plannodes.HashAggregatePlanNode in project voltdb by VoltDB.
the class PlanAssembler method getNextSelectPlan.
private CompiledPlan getNextSelectPlan() {
assert (m_subAssembler != null);
// A matview reaggregation template plan may have been initialized
// with a post-predicate expression moved from the statement's
// join tree prior to any subquery planning.
// Since normally subquery planning is driven from the join tree,
// any subqueries that are moved out of the join tree would need
// to be planned separately.
// This planning would need to be done prior to calling
// m_subAssembler.nextPlan()
// because it can have query partitioning implications.
// Under the current query limitations, the partitioning implications
// are very simple -- subqueries are not allowed in multipartition
// queries against partitioned data, so detection of a subquery in
// the same query as a matview reaggregation can just return an error,
// without any need for subquery planning here.
HashAggregatePlanNode reAggNode = null;
HashAggregatePlanNode mvReAggTemplate = m_parsedSelect.m_mvFixInfo.getReAggregationPlanNode();
if (mvReAggTemplate != null) {
reAggNode = new HashAggregatePlanNode(mvReAggTemplate);
AbstractExpression postPredicate = reAggNode.getPostPredicate();
if (postPredicate != null && postPredicate.hasSubquerySubexpression()) {
// For now, this is just a special case violation of the limitation on
// use of subquery expressions in MP queries on partitioned data.
// That special case was going undetected when we didn't flag it here.
m_recentErrorMsg = IN_EXISTS_SCALAR_ERROR_MESSAGE;
return null;
}
// // Something more along these lines would have to be enabled
// // to allow expression subqueries to be used in multi-partition
// // matview queries.
// if (!getBestCostPlanForExpressionSubQueries(subqueryExprs)) {
// // There was at least one sub-query and we should have a compiled plan for it
// return null;
// }
}
AbstractPlanNode subSelectRoot = m_subAssembler.nextPlan();
if (subSelectRoot == null) {
m_recentErrorMsg = m_subAssembler.m_recentErrorMsg;
return null;
}
AbstractPlanNode root = subSelectRoot;
boolean mvFixNeedsProjection = false;
/*
* If the access plan for the table in the join order was for a
* distributed table scan there must be a send/receive pair at the top
* EXCEPT for the special outer join case in which a replicated table
* was on the OUTER side of an outer join across from the (joined) scan
* of the partitioned table(s) (all of them) in the query. In that case,
* the one required send/receive pair is already in the plan below the
* inner side of a NestLoop join.
*/
if (m_partitioning.requiresTwoFragments()) {
boolean mvFixInfoCoordinatorNeeded = true;
boolean mvFixInfoEdgeCaseOuterJoin = false;
ArrayList<AbstractPlanNode> receivers = root.findAllNodesOfClass(AbstractReceivePlanNode.class);
if (receivers.size() == 1) {
// Edge cases: left outer join with replicated table.
if (m_parsedSelect.m_mvFixInfo.needed()) {
mvFixInfoCoordinatorNeeded = false;
AbstractPlanNode receiveNode = receivers.get(0);
if (receiveNode.getParent(0) instanceof NestLoopPlanNode) {
if (subSelectRoot.hasInlinedIndexScanOfTable(m_parsedSelect.m_mvFixInfo.getMVTableName())) {
return getNextSelectPlan();
}
List<AbstractPlanNode> nljs = receiveNode.findAllNodesOfType(PlanNodeType.NESTLOOP);
List<AbstractPlanNode> nlijs = receiveNode.findAllNodesOfType(PlanNodeType.NESTLOOPINDEX);
// This is like a single table case.
if (nljs.size() + nlijs.size() == 0) {
mvFixInfoEdgeCaseOuterJoin = true;
}
root = handleMVBasedMultiPartQuery(reAggNode, root, mvFixInfoEdgeCaseOuterJoin);
}
}
} else {
if (receivers.size() > 0) {
throw new PlanningErrorException("This special case join between an outer replicated table and " + "an inner partitioned table is too complex and is not supported.");
}
root = SubPlanAssembler.addSendReceivePair(root);
// Root is a receive node here.
assert (root instanceof ReceivePlanNode);
if (m_parsedSelect.mayNeedAvgPushdown()) {
m_parsedSelect.switchOptimalSuiteForAvgPushdown();
}
if (m_parsedSelect.m_tableList.size() > 1 && m_parsedSelect.m_mvFixInfo.needed() && subSelectRoot.hasInlinedIndexScanOfTable(m_parsedSelect.m_mvFixInfo.getMVTableName())) {
// So, in-lined index scan of Nested loop index join can not be possible.
return getNextSelectPlan();
}
}
root = handleAggregationOperators(root);
// Process the re-aggregate plan node and insert it into the plan.
if (m_parsedSelect.m_mvFixInfo.needed() && mvFixInfoCoordinatorNeeded) {
AbstractPlanNode tmpRoot = root;
root = handleMVBasedMultiPartQuery(reAggNode, root, mvFixInfoEdgeCaseOuterJoin);
if (root != tmpRoot) {
mvFixNeedsProjection = true;
}
}
} else {
/*
* There is no receive node and root is a single partition plan.
*/
// If there is no receive plan node and no distributed plan has been generated,
// the fix set for MV is not needed.
m_parsedSelect.m_mvFixInfo.setNeeded(false);
root = handleAggregationOperators(root);
}
// add a PartitionByPlanNode here.
if (m_parsedSelect.hasWindowFunctionExpression()) {
root = handleWindowedOperators(root);
}
if (m_parsedSelect.hasOrderByColumns()) {
root = handleOrderBy(m_parsedSelect, root);
if (m_parsedSelect.isComplexOrderBy() && root instanceof OrderByPlanNode) {
AbstractPlanNode child = root.getChild(0);
AbstractPlanNode grandChild = child.getChild(0);
// swap the ORDER BY and complex aggregate Projection node
if (child instanceof ProjectionPlanNode) {
root.unlinkChild(child);
child.unlinkChild(grandChild);
child.addAndLinkChild(root);
root.addAndLinkChild(grandChild);
// update the new root
root = child;
} else if (m_parsedSelect.hasDistinctWithGroupBy() && child.getPlanNodeType() == PlanNodeType.HASHAGGREGATE && grandChild.getPlanNodeType() == PlanNodeType.PROJECTION) {
AbstractPlanNode grandGrandChild = grandChild.getChild(0);
child.clearParents();
root.clearChildren();
grandGrandChild.clearParents();
grandChild.clearChildren();
grandChild.addAndLinkChild(root);
root.addAndLinkChild(grandGrandChild);
root = child;
}
}
}
// node.
if (mvFixNeedsProjection || needProjectionNode(root)) {
root = addProjection(root);
}
if (m_parsedSelect.hasLimitOrOffset()) {
root = handleSelectLimitOperator(root);
}
CompiledPlan plan = new CompiledPlan();
plan.rootPlanGraph = root;
plan.setReadOnly(true);
boolean orderIsDeterministic = m_parsedSelect.isOrderDeterministic();
boolean hasLimitOrOffset = m_parsedSelect.hasLimitOrOffset();
String contentDeterminismMessage = m_parsedSelect.getContentDeterminismMessage();
plan.statementGuaranteesDeterminism(hasLimitOrOffset, orderIsDeterministic, contentDeterminismMessage);
// Apply the micro-optimization:
// LIMIT push down, Table count / Counting Index, Optimized Min/Max
MicroOptimizationRunner.applyAll(plan, m_parsedSelect);
return plan;
}
use of org.voltdb.plannodes.HashAggregatePlanNode in project voltdb by VoltDB.
the class PlanAssembler method handleDistinctWithGroupby.
/**
* Handle DISTINCT with GROUP BY if it is not redundant with the
* aggregation/grouping.
* DISTINCT is basically rewritten with GROUP BY to benefit from
* all kinds of GROUP BY optimizations.
* Trivial case DISTINCT in a statement with no GROUP BY has been
* rewritten very early at query parsing time.
* In the non-trivial case, where an existing GROUP BY column is NOT
* in the select list, DISTINCT can be implemented via a final aggregation
* (never pushed down) added to the top of the plan.
* @param root can be an aggregate plan node or projection plan node
* @return
*/
private AbstractPlanNode handleDistinctWithGroupby(AbstractPlanNode root) {
if (!m_parsedSelect.hasDistinctWithGroupBy()) {
return root;
}
assert (m_parsedSelect.isGrouped());
// all of the grouping columns are present in the display columns.
if (m_parsedSelect.displayColumnsContainAllGroupByColumns()) {
return root;
}
// Now non complex aggregation cases are handled already
assert (m_parsedSelect.hasComplexAgg());
AggregatePlanNode distinctAggNode = new HashAggregatePlanNode();
distinctAggNode.setOutputSchema(m_parsedSelect.getDistinctProjectionSchema());
for (ParsedColInfo col : m_parsedSelect.distinctGroupByColumns()) {
distinctAggNode.addGroupByExpression(col.expression);
}
// TODO(xin): push down the DISTINCT for certain cases
// Ticket: ENG-7360
/*
boolean pushedDown = false;
boolean canPushdownDistinctAgg =
m_parsedSelect.hasPartitionColumnInDistinctGroupby();
//
// disable pushdown, DISTINCT push down turns out complex
//
canPushdownDistinctAgg = false;
if (canPushdownDistinctAgg && !m_parsedSelect.m_mvFixInfo.needed()) {
assert(m_parsedSelect.hasPartitionColumnInGroupby());
AbstractPlanNode receive = root;
if (receive instanceof ReceivePlanNode) {
// Temporarily strip send/receive pair
AbstractPlanNode distNode = receive.getChild(0).getChild(0);
receive.getChild(0).unlinkChild(distNode);
distinctAggNode.addAndLinkChild(distNode);
receive.getChild(0).addAndLinkChild(distinctAggNode);
pushedDown = true;
}
}*/
distinctAggNode.addAndLinkChild(root);
root = distinctAggNode;
return root;
}
use of org.voltdb.plannodes.HashAggregatePlanNode in project voltdb by VoltDB.
the class PlanAssembler method handleAggregationOperators.
private AbstractPlanNode handleAggregationOperators(AbstractPlanNode root) {
/*
* "Select A from T group by A" is grouped but has no aggregate operator
* expressions. Catch that case by checking the grouped flag
*/
if (m_parsedSelect.hasAggregateOrGroupby()) {
AggregatePlanNode aggNode = null;
// i.e., on the coordinator
AggregatePlanNode topAggNode = null;
IndexGroupByInfo gbInfo = new IndexGroupByInfo();
if (root instanceof AbstractReceivePlanNode) {
// for distinct that does not group by partition column
if (!m_parsedSelect.hasAggregateDistinct() || m_parsedSelect.hasPartitionColumnInGroupby()) {
AbstractPlanNode candidate = root.getChild(0).getChild(0);
gbInfo.m_multiPartition = true;
switchToIndexScanForGroupBy(candidate, gbInfo);
}
} else if (switchToIndexScanForGroupBy(root, gbInfo)) {
root = gbInfo.m_indexAccess;
}
boolean needHashAgg = gbInfo.needHashAggregator(root, m_parsedSelect);
// Construct the aggregate nodes
if (needHashAgg) {
if (m_parsedSelect.m_mvFixInfo.needed()) {
// TODO: may optimize this edge case in future
aggNode = new HashAggregatePlanNode();
} else {
if (gbInfo.isChangedToSerialAggregate()) {
assert (root instanceof ReceivePlanNode);
aggNode = new AggregatePlanNode();
} else if (gbInfo.isChangedToPartialAggregate()) {
aggNode = new PartialAggregatePlanNode(gbInfo.m_coveredGroupByColumns);
} else {
aggNode = new HashAggregatePlanNode();
}
topAggNode = new HashAggregatePlanNode();
}
} else {
aggNode = new AggregatePlanNode();
if (!m_parsedSelect.m_mvFixInfo.needed()) {
topAggNode = new AggregatePlanNode();
}
}
NodeSchema agg_schema = new NodeSchema();
NodeSchema top_agg_schema = new NodeSchema();
for (int outputColumnIndex = 0; outputColumnIndex < m_parsedSelect.m_aggResultColumns.size(); outputColumnIndex += 1) {
ParsedColInfo col = m_parsedSelect.m_aggResultColumns.get(outputColumnIndex);
AbstractExpression rootExpr = col.expression;
AbstractExpression agg_input_expr = null;
SchemaColumn schema_col = null;
SchemaColumn top_schema_col = null;
if (rootExpr instanceof AggregateExpression) {
ExpressionType agg_expression_type = rootExpr.getExpressionType();
agg_input_expr = rootExpr.getLeft();
// A bit of a hack: ProjectionNodes after the
// aggregate node need the output columns here to
// contain TupleValueExpressions (effectively on a temp table).
// So we construct one based on the output of the
// aggregate expression, the column alias provided by HSQL,
// and the offset into the output table schema for the
// aggregate node that we're computing.
// Oh, oh, it's magic, you know..
TupleValueExpression tve = new TupleValueExpression(AbstractParsedStmt.TEMP_TABLE_NAME, AbstractParsedStmt.TEMP_TABLE_NAME, "", col.alias, rootExpr, outputColumnIndex);
tve.setDifferentiator(col.differentiator);
boolean is_distinct = ((AggregateExpression) rootExpr).isDistinct();
aggNode.addAggregate(agg_expression_type, is_distinct, outputColumnIndex, agg_input_expr);
schema_col = new SchemaColumn(AbstractParsedStmt.TEMP_TABLE_NAME, AbstractParsedStmt.TEMP_TABLE_NAME, "", col.alias, tve, outputColumnIndex);
top_schema_col = new SchemaColumn(AbstractParsedStmt.TEMP_TABLE_NAME, AbstractParsedStmt.TEMP_TABLE_NAME, "", col.alias, tve, outputColumnIndex);
/*
* Special case count(*), count(), sum(), min() and max() to
* push them down to each partition. It will do the
* push-down if the select columns only contains the listed
* aggregate operators and other group-by columns. If the
* select columns includes any other aggregates, it will not
* do the push-down. - nshi
*/
if (topAggNode != null) {
ExpressionType top_expression_type = agg_expression_type;
/*
* For count(*), count() and sum(), the pushed-down
* aggregate node doesn't change. An extra sum()
* aggregate node is added to the coordinator to sum up
* the numbers from all the partitions. The input schema
* and the output schema of the sum() aggregate node is
* the same as the output schema of the push-down
* aggregate node.
*
* If DISTINCT is specified, don't do push-down for
* count() and sum() when not group by partition column.
* An exception is the aggregation arguments are the
* partition column (ENG-4980).
*/
if (agg_expression_type == ExpressionType.AGGREGATE_COUNT_STAR || agg_expression_type == ExpressionType.AGGREGATE_COUNT || agg_expression_type == ExpressionType.AGGREGATE_SUM) {
if (is_distinct && !(m_parsedSelect.hasPartitionColumnInGroupby() || canPushDownDistinctAggregation((AggregateExpression) rootExpr))) {
topAggNode = null;
} else {
// for aggregate distinct when group by
// partition column, the top aggregate node
// will be dropped later, thus there is no
// effect to assign the top_expression_type.
top_expression_type = ExpressionType.AGGREGATE_SUM;
}
} else /*
* For min() and max(), the pushed-down aggregate node
* doesn't change. An extra aggregate node of the same
* type is added to the coordinator. The input schema
* and the output schema of the top aggregate node is
* the same as the output schema of the pushed-down
* aggregate node.
*
* APPROX_COUNT_DISTINCT can be similarly pushed down, but
* must be split into two different functions, which is
* done later, from pushDownAggregate().
*/
if (agg_expression_type != ExpressionType.AGGREGATE_MIN && agg_expression_type != ExpressionType.AGGREGATE_MAX && agg_expression_type != ExpressionType.AGGREGATE_APPROX_COUNT_DISTINCT) {
/*
* Unsupported aggregate for push-down (AVG for example).
*/
topAggNode = null;
}
if (topAggNode != null) {
/*
* Input column of the top aggregate node is the
* output column of the push-down aggregate node
*/
boolean topDistinctFalse = false;
topAggNode.addAggregate(top_expression_type, topDistinctFalse, outputColumnIndex, tve);
}
}
// end if we have a top agg node
} else {
// has already been broken down.
assert (!rootExpr.hasAnySubexpressionOfClass(AggregateExpression.class));
/*
* These columns are the pass through columns that are not being
* aggregated on. These are the ones from the SELECT list. They
* MUST already exist in the child node's output. Find them and
* add them to the aggregate's output.
*/
schema_col = new SchemaColumn(col.tableName, col.tableAlias, col.columnName, col.alias, col.expression, outputColumnIndex);
AbstractExpression topExpr = null;
if (col.groupBy) {
topExpr = m_parsedSelect.m_groupByExpressions.get(col.alias);
} else {
topExpr = col.expression;
}
top_schema_col = new SchemaColumn(col.tableName, col.tableAlias, col.columnName, col.alias, topExpr, outputColumnIndex);
}
agg_schema.addColumn(schema_col);
top_agg_schema.addColumn(top_schema_col);
}
for (ParsedColInfo col : m_parsedSelect.groupByColumns()) {
aggNode.addGroupByExpression(col.expression);
if (topAggNode != null) {
topAggNode.addGroupByExpression(m_parsedSelect.m_groupByExpressions.get(col.alias));
}
}
aggNode.setOutputSchema(agg_schema);
if (topAggNode != null) {
if (m_parsedSelect.hasComplexGroupby()) {
topAggNode.setOutputSchema(top_agg_schema);
} else {
topAggNode.setOutputSchema(agg_schema);
}
}
// Never push down aggregation for MV fix case.
root = pushDownAggregate(root, aggNode, topAggNode, m_parsedSelect);
}
return handleDistinctWithGroupby(root);
}
Aggregations