use of org.voltdb.plannodes.LimitPlanNode in project voltdb by VoltDB.
the class AbstractParsedStmt method limitPlanNodeFromXml.
/**
* Produce a LimitPlanNode from the given XML
* @param limitXml Volt XML for limit
* @param offsetXml Volt XML for offset
* @return An instance of LimitPlanNode for the given XML
*/
LimitPlanNode limitPlanNodeFromXml(VoltXMLElement limitXml, VoltXMLElement offsetXml) {
if (limitXml == null && offsetXml == null) {
return null;
}
String node;
long limitParameterId = -1;
long offsetParameterId = -1;
long limit = -1;
long offset = 0;
if (limitXml != null) {
// Parse limit
if ((node = limitXml.attributes.get("limit_paramid")) != null) {
limitParameterId = Long.parseLong(node);
} else {
assert (limitXml.children.size() == 1);
VoltXMLElement valueNode = limitXml.children.get(0);
String isParam = valueNode.attributes.get("isparam");
if ((isParam != null) && (isParam.equalsIgnoreCase("true"))) {
limitParameterId = Long.parseLong(valueNode.attributes.get("id"));
} else {
node = limitXml.attributes.get("limit");
assert (node != null);
limit = Long.parseLong(node);
}
}
}
if (offsetXml != null) {
// Parse offset
if ((node = offsetXml.attributes.get("offset_paramid")) != null) {
offsetParameterId = Long.parseLong(node);
} else {
if (offsetXml.children.size() == 1) {
VoltXMLElement valueNode = offsetXml.children.get(0);
String isParam = valueNode.attributes.get("isparam");
if ((isParam != null) && (isParam.equalsIgnoreCase("true"))) {
offsetParameterId = Long.parseLong(valueNode.attributes.get("id"));
} else {
node = offsetXml.attributes.get("offset");
assert (node != null);
offset = Long.parseLong(node);
}
}
}
}
// limit and offset can't have both value and parameter
if (limit != -1)
assert limitParameterId == -1 : "Parsed value and param. limit.";
if (offset != 0)
assert offsetParameterId == -1 : "Parsed value and param. offset.";
LimitPlanNode limitPlanNode = new LimitPlanNode();
limitPlanNode.setLimit((int) limit);
limitPlanNode.setOffset((int) offset);
limitPlanNode.setLimitParameterIndex(parameterCountIndexById(limitParameterId));
limitPlanNode.setOffsetParameterIndex(parameterCountIndexById(offsetParameterId));
return limitPlanNode;
}
use of org.voltdb.plannodes.LimitPlanNode in project voltdb by VoltDB.
the class ReplaceWithIndexLimit method recursivelyApply.
@Override
protected AbstractPlanNode recursivelyApply(AbstractPlanNode plan) {
assert (plan != null);
// depth first:
// Find AggregatePlanNode with exactly one child
// where that child is an AbstractScanPlanNode.
// Replace qualifying SeqScanPlanNode with an
// IndexScanPlanNode with an inlined LimitPlanNode;
// or appending the LimitPlanNode to the existing
// qualified IndexScanPlanNode.
ArrayList<AbstractPlanNode> children = new ArrayList<AbstractPlanNode>();
for (int i = 0; i < plan.getChildCount(); i++) children.add(plan.getChild(i));
for (AbstractPlanNode child : children) {
// TODO this will break when children feed multiple parents
AbstractPlanNode newChild = recursivelyApply(child);
// Do a graft into the (parent) plan only if a replacement for a child was found.
if (newChild == child) {
continue;
}
child.removeFromGraph();
plan.addAndLinkChild(newChild);
}
// check for an aggregation of the right form
if ((plan instanceof AggregatePlanNode) == false)
return plan;
assert (plan.getChildCount() == 1);
AggregatePlanNode aggplan = (AggregatePlanNode) plan;
// handle one single min() / max() now
// TODO: combination of [min(), max(), count()]
SortDirectionType sortDirection = SortDirectionType.INVALID;
if (aggplan.isTableMin()) {
sortDirection = SortDirectionType.ASC;
} else if (aggplan.isTableMax()) {
sortDirection = SortDirectionType.DESC;
} else {
return plan;
}
AbstractPlanNode child = plan.getChild(0);
AbstractExpression aggExpr = aggplan.getFirstAggregateExpression();
// for a SEQSCAN, replace it with a INDEXSCAN node with an inline LIMIT plan node
if (child instanceof SeqScanPlanNode) {
// should have other index access plan if any qualified index found for the predicate
if (((SeqScanPlanNode) child).getPredicate() != null) {
return plan;
}
if (((AbstractScanPlanNode) child).isSubQuery()) {
return plan;
}
// create an empty bindingExprs list, used for store (possible) bindings for adHoc query
ArrayList<AbstractExpression> bindings = new ArrayList<AbstractExpression>();
Index ret = findQualifiedIndex(((SeqScanPlanNode) child), aggExpr, bindings);
if (ret == null) {
return plan;
} else {
// 1. create one INDEXSCAN plan node with inlined LIMIT
// and replace the SEQSCAN node with it
// 2. we know which end row we want to fetch, so it's safe to
// specify sorting direction here
IndexScanPlanNode ispn = new IndexScanPlanNode((SeqScanPlanNode) child, aggplan, ret, sortDirection);
ispn.setBindings(bindings);
assert (ispn.getSearchKeyExpressions().size() == 0);
if (sortDirection == SortDirectionType.ASC) {
assert (aggplan.isTableMin());
ispn.setSkipNullPredicate(0);
}
LimitPlanNode lpn = new LimitPlanNode();
lpn.setLimit(1);
lpn.setOffset(0);
ispn.addInlinePlanNode(lpn);
// remove old SeqScan node and link the new generated IndexScan node
plan.clearChildren();
plan.addAndLinkChild(ispn);
return plan;
}
}
if ((child instanceof IndexScanPlanNode) == false) {
return plan;
}
// already have the IndexScanPlanNode
IndexScanPlanNode ispn = (IndexScanPlanNode) child;
// we added for reverse scan purpose only
if (((IndexScanPlanNode) child).getPredicate() != null && !((IndexScanPlanNode) child).isPredicatesOptimizableForAggregate()) {
return plan;
}
// Guard against (possible future?) cases of indexable subquery.
if (((AbstractScanPlanNode) child).isSubQuery()) {
return plan;
}
// 2. Handle equality filters and one other comparison operator (<, <=, >, >=), see comments below
if (ispn.getLookupType() != IndexLookupType.EQ && Math.abs(ispn.getSearchKeyExpressions().size() - ExpressionUtil.uncombinePredicate(ispn.getEndExpression()).size()) > 1) {
return plan;
}
// exprs will be used as filterExprs to check the index
// For forward scan, the initial value is endExprs and might be changed in different values in variant cases
// For reverse scan, the initial value is initialExprs which is the "old" endExprs
List<AbstractExpression> exprs;
int numOfSearchKeys = ispn.getSearchKeyExpressions().size();
if (ispn.getLookupType() == IndexLookupType.LT || ispn.getLookupType() == IndexLookupType.LTE) {
exprs = ExpressionUtil.uncombinePredicate(ispn.getInitialExpression());
numOfSearchKeys -= 1;
} else {
exprs = ExpressionUtil.uncombinePredicate(ispn.getEndExpression());
}
int numberOfExprs = exprs.size();
/* Retrieve the index expressions from the target index. (ENG-8819, Ethan)
* This is because we found that for the following two queries:
* #1: explain select max(c2/2) from t where c1=1 and c2/2<=3;
* #2: explain select max(c2/2) from t where c1=1 and c2/2<=?;
* We can get an inline limit 1 for #2 but not for #1. This is because all constants in #1 got parameterized.
* The result is that the query cannot pass the bindingToIndexedExpression() tests below
* because we lost all the constant value expressions (cannot attempt to bind a pve to a pve!).
* Those constant values expressions can only be accessed from the idnex.
* We will not add those bindings to the ispn.getBindings() here because they will be added anyway in checkIndex().
* PS: For this case (i.e. index on expressions), checkIndex() will call checkExpressionIndex(),
* where bindings will be added.
*/
Index indexToUse = ispn.getCatalogIndex();
String tableAlias = ispn.getTargetTableAlias();
List<AbstractExpression> indexedExprs = null;
if (!indexToUse.getExpressionsjson().isEmpty()) {
StmtTableScan tableScan = m_parsedStmt.getStmtTableScanByAlias(tableAlias);
try {
indexedExprs = AbstractExpression.fromJSONArrayString(indexToUse.getExpressionsjson(), tableScan);
} catch (JSONException e) {
e.printStackTrace();
assert (false);
return plan;
}
}
/* If there is only 1 difference between searchkeyExprs and endExprs,
* 1. trivial filters can be discarded, 2 possibilities:
* a. SELECT MIN(X) FROM T WHERE [other prefix filters] X < / <= ?
* <=> SELECT MIN(X) FROM T WHERE [other prefix filters] && the X < / <= ? filter
* b. SELECT MAX(X) FROM T WHERE X > / >= ?
* <=> SELECT MAX(X) FROM T with post-filter
* 2. filter should act as equality filter, 2 possibilities
* SELECT MIN(X) FROM T WHERE [other prefix filters] X > / >= ?
* SELECT MAX(X) FROM T WHERE [other prefix filters] X < / <= ?
* check if there is other filters for SELECT MAX(X) FROM T WHERE [other prefix filter AND ] X > / >= ?
* but we should allow SELECT MAX(X) FROM T WHERE X = ?
* This is for queries having MAX() but no ORDER BY. (ENG-8819, Ethan)
* sortDirection == DESC if max, ASC if min. ispn.getSortDirection() == INVALID if no ORDER BY. */
if (sortDirection == SortDirectionType.DESC && ispn.getSortDirection() == SortDirectionType.INVALID) {
/* numberOfExprs = exprs.size(), exprs are initial expressions for reversed index scans (lookupType LT, LTE),
* are end expressions for forward index scans (lookupType GT, GTE, EQ).
* Note, lookupType doesn't decide the scan direction for sure. MIN(X) where X < ? is still a forward scan.
* X < ? will be a post filter for the scan rather than an initial expression. */
if (numberOfExprs == 1) {
// e.g.: explain select max(c2/2) from t where c2/2<=3;
// In this case, as long as the where condition (exprs.get(0)) matches the aggregation argument, continue.
AbstractExpression exprToBind = indexedExprs == null ? exprs.get(0).getLeft() : indexedExprs.get(0);
if (aggExpr.bindingToIndexedExpression(exprToBind) == null) {
return plan;
}
} else if (numberOfExprs > 1) {
// ENG-4016: Optimization for query SELECT MAX(X) FROM T WHERE [other prefix filters] X < / <= ?
// Just keep trying, don't return early.
boolean earlyReturn = true;
for (int i = 0; i < numberOfExprs; ++i) {
AbstractExpression expr = exprs.get(i);
AbstractExpression indexedExpr = indexedExprs == null ? expr.getLeft() : indexedExprs.get(i);
if (aggExpr.bindingToIndexedExpression(indexedExpr) != null && (expr.getExpressionType() == ExpressionType.COMPARE_LESSTHANOREQUALTO || expr.getExpressionType() == ExpressionType.COMPARE_LESSTHAN || expr.getExpressionType() == ExpressionType.COMPARE_EQUAL)) {
earlyReturn = false;
break;
}
}
if (earlyReturn) {
return plan;
}
}
}
// have an upper bound: # of endingExpr is more than # of searchExpr
if (numberOfExprs > numOfSearchKeys) {
AbstractExpression lastEndExpr = exprs.get(numberOfExprs - 1);
// check last ending condition, see whether it is
// SELECT MIN(X) FROM T WHERE [other prefix filters] X < / <= ? or
// other filters will be checked later
AbstractExpression exprToBind = indexedExprs == null ? lastEndExpr.getLeft() : indexedExprs.get(numberOfExprs - 1);
if ((lastEndExpr.getExpressionType() == ExpressionType.COMPARE_LESSTHAN || lastEndExpr.getExpressionType() == ExpressionType.COMPARE_LESSTHANOREQUALTO) && aggExpr.bindingToIndexedExpression(exprToBind) != null) {
exprs.remove(lastEndExpr);
}
}
// and we can take advantage of that
if (checkIndex(ispn.getCatalogIndex(), aggExpr, exprs, ispn.getBindings(), tableAlias)) {
// we know which end we want to fetch, set the sort direction
ispn.setSortDirection(sortDirection);
// for SELECT MIN(X) FROM T WHERE [prefix filters] = ?
if (numberOfExprs == numOfSearchKeys && sortDirection == SortDirectionType.ASC) {
if (ispn.getLookupType() == IndexLookupType.GTE) {
assert (aggplan.isTableMin());
ispn.setSkipNullPredicate(numOfSearchKeys);
}
}
// reset the IndexLookupType, remove "added" searchKey, add back to endExpression, and clear "added" predicate
if (sortDirection == SortDirectionType.ASC && (ispn.getLookupType() == IndexLookupType.LT || ispn.getLookupType() == IndexLookupType.LTE)) {
ispn.setLookupType(IndexLookupType.GTE);
ispn.removeLastSearchKey();
ispn.addEndExpression(ExpressionUtil.uncombinePredicate(ispn.getInitialExpression()).get(numberOfExprs - 1));
ispn.setSkipNullPredicate(numOfSearchKeys);
ispn.resetPredicate();
}
// add an inline LIMIT plan node to this index scan plan node
LimitPlanNode lpn = new LimitPlanNode();
lpn.setLimit(1);
lpn.setOffset(0);
ispn.addInlinePlanNode(lpn);
// |__LimitPlanNode
if (sortDirection == SortDirectionType.DESC && !ispn.getSearchKeyExpressions().isEmpty() && exprs.isEmpty() && ExpressionUtil.uncombinePredicate(ispn.getInitialExpression()).isEmpty()) {
AbstractExpression newPredicate = new ComparisonExpression();
if (ispn.getLookupType() == IndexLookupType.GT)
newPredicate.setExpressionType(ExpressionType.COMPARE_GREATERTHAN);
if (ispn.getLookupType() == IndexLookupType.GTE)
newPredicate.setExpressionType(ExpressionType.COMPARE_GREATERTHANOREQUALTO);
newPredicate.setRight(ispn.getSearchKeyExpressions().get(0));
newPredicate.setLeft(aggExpr);
newPredicate.setValueType(aggExpr.getValueType());
ispn.clearSearchKeyExpression();
aggplan.setPrePredicate(newPredicate);
}
}
return plan;
}
use of org.voltdb.plannodes.LimitPlanNode in project voltdb by VoltDB.
the class PushdownLimits method recursivelyApply.
@Override
protected AbstractPlanNode recursivelyApply(AbstractPlanNode plan) {
assert (plan != null);
// depth first:
// find LimitPlanNodes with exactly one child
// where that child is an AbstractScanPlanNode
// disconnect the LimitPlanNode
// and inline the LimitPlanNode in to the AbstractScanPlanNode
ArrayList<AbstractPlanNode> children = new ArrayList<AbstractPlanNode>();
for (int i = 0; i < plan.getChildCount(); i++) children.add(plan.getChild(i));
plan.clearChildren();
for (AbstractPlanNode child : children) {
// TODO this will break when children feed multiple parents
child = recursivelyApply(child);
child.clearParents();
plan.addAndLinkChild(child);
}
if (!(plan instanceof LimitPlanNode)) {
return plan;
}
if (plan.getChildCount() != 1) {
assert (plan.getChildCount() == 1);
return plan;
}
AbstractPlanNode child = plan.getChild(0);
// push into Scans
if (child instanceof AbstractScanPlanNode) {
// in future, this limit can be aggregate inline node.
if (AggregatePlanNode.getInlineAggregationNode(child) != null) {
return plan;
}
plan.clearChildren();
child.clearParents();
child.addInlinePlanNode(plan);
return child;
}
// == child/projection . recursivelyApply(plan/limit . leaf/whatever)
if (child instanceof ProjectionPlanNode) {
assert (child.getChildCount() == 1);
AbstractPlanNode leaf = child.getChild(0);
leaf.clearParents();
plan.clearChildren();
plan.addAndLinkChild(leaf);
child.clearChildren();
child.clearParents();
child.addAndLinkChild(plan);
return recursivelyApply(child);
}
// push into JOINs
if (child instanceof AbstractJoinPlanNode) {
plan.clearChildren();
child.clearParents();
child.addInlinePlanNode(plan);
// }
return child;
}
return plan;
}
use of org.voltdb.plannodes.LimitPlanNode in project voltdb by VoltDB.
the class ParsedSelectStmt method prepareLimitPlanNode.
public static void prepareLimitPlanNode(AbstractParsedStmt stmt, LimitOffset limitOffset) {
int limitParamIndex = stmt.parameterCountIndexById(limitOffset.m_limitParameterId);
int offsetParamIndex = stmt.parameterCountIndexById(limitOffset.m_offsetParameterId);
// The coordinator's top limit graph fragment for a MP plan.
// If planning "order by ... limit", getNextSelectPlan()
// will have already added an order by to the coordinator frag.
// This is the only limit node in a SP plan
limitOffset.m_limitNodeTop = new LimitPlanNode();
limitOffset.m_limitNodeTop.setLimit((int) limitOffset.m_limit);
limitOffset.m_limitNodeTop.setOffset((int) limitOffset.m_offset);
limitOffset.m_limitNodeTop.setLimitParameterIndex(limitParamIndex);
limitOffset.m_limitNodeTop.setOffsetParameterIndex(offsetParamIndex);
if (limitOffset.m_limitCanPushdown) {
limitOffset.m_limitNodeDist = new LimitPlanNode();
// limit as a pad on the limit.
if (limitOffset.m_limit != -1) {
limitOffset.m_limitNodeDist.setLimit((int) (limitOffset.m_limit + limitOffset.m_offset));
}
if (limitOffset.hasLimitOrOffsetParameters()) {
AbstractExpression left = stmt.getParameterOrConstantAsExpression(limitOffset.m_offsetParameterId, limitOffset.m_offset);
assert (left != null);
AbstractExpression right = stmt.getParameterOrConstantAsExpression(limitOffset.m_limitParameterId, limitOffset.m_limit);
assert (right != null);
OperatorExpression expr = new OperatorExpression(ExpressionType.OPERATOR_PLUS, left, right);
expr.setValueType(VoltType.INTEGER);
expr.setValueSize(VoltType.INTEGER.getLengthInBytesForFixedTypes());
limitOffset.m_limitNodeDist.setLimitExpression(expr);
}
// else let the parameterized forms of offset/limit default to unused/invalid.
}
}
use of org.voltdb.plannodes.LimitPlanNode in project voltdb by VoltDB.
the class TestPlansDistinct method checkDistinctWithGroupbyPlans.
/**
*
* @param distinctSQL Group by query with distinct
* @param groupbySQL Group by query without distinct
*/
protected void checkDistinctWithGroupbyPlans(String distinctSQL, String groupbySQL, boolean limitPushdown) {
List<AbstractPlanNode> pns1 = compileToFragments(distinctSQL);
List<AbstractPlanNode> pns2 = compileToFragments(groupbySQL);
//printExplainPlan(pns1);
//printExplainPlan(pns2);
assertTrue(pns1.get(0) instanceof SendPlanNode);
assertTrue(pns2.get(0) instanceof SendPlanNode);
AbstractPlanNode apn1, apn2;
apn1 = pns1.get(0).getChild(0);
apn2 = pns2.get(0).getChild(0);
boolean hasTopProjection1 = false;
if (apn1 instanceof ProjectionPlanNode) {
apn1 = apn1.getChild(0);
hasTopProjection1 = true;
}
boolean hasTopProjection2 = false;
if (apn2 instanceof ProjectionPlanNode) {
apn2 = apn2.getChild(0);
hasTopProjection2 = true;
}
// DISTINCT plan node is rewrote with GROUP BY and adds above the original GROUP BY node
// there may be another projection node in between for complex aggregation case
boolean hasOrderby = false, hasLimit = false;
boolean groupByMergeReceive = false;
// infer the ORDERBY/LIMIT information from the base line query
if (apn2 instanceof OrderByPlanNode) {
hasOrderby = true;
if (apn2.getInlinePlanNode(PlanNodeType.LIMIT) != null) {
hasLimit = true;
}
apn2 = apn2.getChild(0);
} else if (apn2 instanceof LimitPlanNode) {
hasLimit = true;
apn2 = apn2.getChild(0);
} else if (apn2 instanceof MergeReceivePlanNode) {
assertTrue(apn2.getInlinePlanNode(PlanNodeType.ORDERBY) != null);
hasOrderby = true;
hasLimit = apn2.getInlinePlanNode(PlanNodeType.LIMIT) != null;
groupByMergeReceive = true;
}
// check the DISTINCT query plan
boolean distinctMergeReceive = false;
if (hasOrderby) {
if (apn1 instanceof OrderByPlanNode) {
assertTrue(apn1 instanceof OrderByPlanNode);
if (hasLimit) {
// check inline limit
assertNotNull(apn1.getInlinePlanNode(PlanNodeType.LIMIT));
}
apn1 = apn1.getChild(0);
} else if (apn1 instanceof MergeReceivePlanNode) {
distinctMergeReceive = true;
assertNotNull(apn1.getInlinePlanNode(PlanNodeType.ORDERBY));
assertEquals(0, apn1.getChildCount());
} else {
fail("The distinctSQL top node is not OrderBy or MergeReceive.");
}
} else if (hasLimit) {
assertTrue(apn1 instanceof LimitPlanNode);
apn1 = apn1.getChild(0);
}
// Check DISTINCT group by plan node
if (distinctMergeReceive) {
AbstractPlanNode aggr = AggregatePlanNode.getInlineAggregationNode(apn1);
assertTrue(aggr instanceof AggregatePlanNode);
assertEquals(0, ((AggregatePlanNode) aggr).getAggregateTypesSize());
assertEquals(pns1.get(0).getOutputSchema().getColumns().size(), ((AggregatePlanNode) aggr).getGroupByExpressionsSize());
if (hasLimit) {
// check inline limit
assertNotNull(aggr.getInlinePlanNode(PlanNodeType.LIMIT));
}
} else {
assertTrue(apn1 instanceof HashAggregatePlanNode);
assertEquals(0, ((HashAggregatePlanNode) apn1).getAggregateTypesSize());
assertEquals(pns1.get(0).getOutputSchema().getColumns().size(), ((HashAggregatePlanNode) apn1).getGroupByExpressionsSize());
apn1 = apn1.getChild(0);
}
// check projection node for complex aggregation case
if (apn1 instanceof ProjectionPlanNode) {
apn1 = apn1.getChild(0);
assertFalse(hasTopProjection1);
}
if (apn2 instanceof ProjectionPlanNode) {
apn2 = apn2.getChild(0);
assertFalse(hasTopProjection2);
}
// check the rest plan nodes.
if (distinctMergeReceive == false && groupByMergeReceive == false) {
assertEquals(apn1.toExplainPlanString(), apn2.toExplainPlanString());
} else if (distinctMergeReceive == true && groupByMergeReceive == true) {
// In case of applied MergeReceive optimization the apn1 and apn2 nodes
// should not have any children
assertEquals(0, apn1.getChildCount());
assertEquals(0, apn2.getChildCount());
}
// Distributed DISTINCT GROUP BY
if (pns1.size() > 1) {
if (!limitPushdown) {
assertEquals(pns1.get(1).toExplainPlanString(), pns2.get(1).toExplainPlanString());
return;
}
assertTrue(pns1.get(1) instanceof SendPlanNode);
assertTrue(pns2.get(1) instanceof SendPlanNode);
apn1 = pns1.get(1).getChild(0);
apn2 = pns2.get(1).getChild(0);
// ignore the ORDER BY/LIMIT pushdown plan node
// because DISTINCT case can not be pushed down
assertTrue(apn2 instanceof OrderByPlanNode);
assertNotNull(apn2.getInlinePlanNode(PlanNodeType.LIMIT));
apn2 = apn2.getChild(0);
// winners may produce completely different paths.
if (distinctMergeReceive == false && groupByMergeReceive == false) {
assertEquals(apn1.toExplainPlanString(), apn2.toExplainPlanString());
}
}
}
Aggregations