use of org.voltdb.types.SortDirectionType in project voltdb by VoltDB.
the class AbstractParsedStmt method parseWindowedAggregationExpression.
/**
* Parse a windowed expression. This actually just returns a TVE. The
* WindowFunctionExpression is squirreled away in the m_windowFunctionExpressions
* object, though, because we will need it later.
*
* @param exprNode
* @return
*/
private AbstractExpression parseWindowedAggregationExpression(VoltXMLElement exprNode) {
int id = Integer.parseInt(exprNode.attributes.get("id"));
String optypeName = exprNode.attributes.get("optype");
ExpressionType optype = ExpressionType.get(optypeName);
if (optype == ExpressionType.INVALID) {
throw new PlanningErrorException("Undefined windowed function call " + optypeName);
}
// the windowed expression, then this is an error.
if (!m_parsingInDisplayColumns) {
if (m_windowFunctionExpressions.size() > 0) {
WindowFunctionExpression we = m_windowFunctionExpressions.get(0);
if (we.getXMLID() == id) {
// away in the windowed expression.
return we.getDisplayListExpression();
}
}
throw new PlanningErrorException("Windowed function call expressions can only appear in the selection list of a query or subquery.");
}
// Parse individual aggregate expressions
List<AbstractExpression> partitionbyExprs = new ArrayList<>();
List<AbstractExpression> orderbyExprs = new ArrayList<>();
List<SortDirectionType> orderbyDirs = new ArrayList<>();
List<AbstractExpression> aggParams = new ArrayList<>();
for (VoltXMLElement childEle : exprNode.children) {
if (childEle.name.equals("winspec")) {
for (VoltXMLElement ele : childEle.children) {
if (ele.name.equals("partitionbyList")) {
for (VoltXMLElement childNode : ele.children) {
AbstractExpression expr = parseExpressionNode(childNode);
ExpressionUtil.finalizeValueTypes(expr);
partitionbyExprs.add(expr);
}
} else if (ele.name.equals("orderbyList")) {
for (VoltXMLElement childNode : ele.children) {
SortDirectionType sortDir = Boolean.valueOf(childNode.attributes.get("descending")) ? SortDirectionType.DESC : SortDirectionType.ASC;
AbstractExpression expr = parseExpressionNode(childNode.children.get(0));
ExpressionUtil.finalizeValueTypes(expr);
orderbyExprs.add(expr);
orderbyDirs.add(sortDir);
}
}
}
} else {
AbstractExpression aggParam = parseExpressionNode(childEle);
if (aggParam != null) {
aggParam.finalizeValueTypes();
aggParams.add(aggParam);
}
}
}
String alias = WINDOWED_AGGREGATE_COLUMN_NAME;
if (exprNode.attributes.containsKey("alias")) {
alias = exprNode.attributes.get("alias");
}
WindowFunctionExpression rankExpr = new WindowFunctionExpression(optype, partitionbyExprs, orderbyExprs, orderbyDirs, aggParams, id);
ExpressionUtil.finalizeValueTypes(rankExpr);
// Only offset 0 is useful. But we keep the index anyway.
int offset = m_windowFunctionExpressions.size();
m_windowFunctionExpressions.add(rankExpr);
TupleValueExpression tve = new TupleValueExpression(TEMP_TABLE_NAME, TEMP_TABLE_NAME, alias, alias, rankExpr, offset);
// This tve does not ever need a differentiator.
tve.setNeedsNoDifferentiation();
rankExpr.setDisplayListExpression(tve);
return tve;
}
use of org.voltdb.types.SortDirectionType in project voltdb by VoltDB.
the class ReplaceWithIndexLimit method recursivelyApply.
@Override
protected AbstractPlanNode recursivelyApply(AbstractPlanNode plan) {
assert (plan != null);
// depth first:
// Find AggregatePlanNode with exactly one child
// where that child is an AbstractScanPlanNode.
// Replace qualifying SeqScanPlanNode with an
// IndexScanPlanNode with an inlined LimitPlanNode;
// or appending the LimitPlanNode to the existing
// qualified IndexScanPlanNode.
ArrayList<AbstractPlanNode> children = new ArrayList<AbstractPlanNode>();
for (int i = 0; i < plan.getChildCount(); i++) children.add(plan.getChild(i));
for (AbstractPlanNode child : children) {
// TODO this will break when children feed multiple parents
AbstractPlanNode newChild = recursivelyApply(child);
// Do a graft into the (parent) plan only if a replacement for a child was found.
if (newChild == child) {
continue;
}
child.removeFromGraph();
plan.addAndLinkChild(newChild);
}
// check for an aggregation of the right form
if ((plan instanceof AggregatePlanNode) == false)
return plan;
assert (plan.getChildCount() == 1);
AggregatePlanNode aggplan = (AggregatePlanNode) plan;
// handle one single min() / max() now
// TODO: combination of [min(), max(), count()]
SortDirectionType sortDirection = SortDirectionType.INVALID;
if (aggplan.isTableMin()) {
sortDirection = SortDirectionType.ASC;
} else if (aggplan.isTableMax()) {
sortDirection = SortDirectionType.DESC;
} else {
return plan;
}
AbstractPlanNode child = plan.getChild(0);
AbstractExpression aggExpr = aggplan.getFirstAggregateExpression();
// for a SEQSCAN, replace it with a INDEXSCAN node with an inline LIMIT plan node
if (child instanceof SeqScanPlanNode) {
// should have other index access plan if any qualified index found for the predicate
if (((SeqScanPlanNode) child).getPredicate() != null) {
return plan;
}
if (((AbstractScanPlanNode) child).isSubQuery()) {
return plan;
}
// create an empty bindingExprs list, used for store (possible) bindings for adHoc query
ArrayList<AbstractExpression> bindings = new ArrayList<AbstractExpression>();
Index ret = findQualifiedIndex(((SeqScanPlanNode) child), aggExpr, bindings);
if (ret == null) {
return plan;
} else {
// 1. create one INDEXSCAN plan node with inlined LIMIT
// and replace the SEQSCAN node with it
// 2. we know which end row we want to fetch, so it's safe to
// specify sorting direction here
IndexScanPlanNode ispn = new IndexScanPlanNode((SeqScanPlanNode) child, aggplan, ret, sortDirection);
ispn.setBindings(bindings);
assert (ispn.getSearchKeyExpressions().size() == 0);
if (sortDirection == SortDirectionType.ASC) {
assert (aggplan.isTableMin());
ispn.setSkipNullPredicate(0);
}
LimitPlanNode lpn = new LimitPlanNode();
lpn.setLimit(1);
lpn.setOffset(0);
ispn.addInlinePlanNode(lpn);
// remove old SeqScan node and link the new generated IndexScan node
plan.clearChildren();
plan.addAndLinkChild(ispn);
return plan;
}
}
if ((child instanceof IndexScanPlanNode) == false) {
return plan;
}
// already have the IndexScanPlanNode
IndexScanPlanNode ispn = (IndexScanPlanNode) child;
// we added for reverse scan purpose only
if (((IndexScanPlanNode) child).getPredicate() != null && !((IndexScanPlanNode) child).isPredicatesOptimizableForAggregate()) {
return plan;
}
// Guard against (possible future?) cases of indexable subquery.
if (((AbstractScanPlanNode) child).isSubQuery()) {
return plan;
}
// 2. Handle equality filters and one other comparison operator (<, <=, >, >=), see comments below
if (ispn.getLookupType() != IndexLookupType.EQ && Math.abs(ispn.getSearchKeyExpressions().size() - ExpressionUtil.uncombinePredicate(ispn.getEndExpression()).size()) > 1) {
return plan;
}
// exprs will be used as filterExprs to check the index
// For forward scan, the initial value is endExprs and might be changed in different values in variant cases
// For reverse scan, the initial value is initialExprs which is the "old" endExprs
List<AbstractExpression> exprs;
int numOfSearchKeys = ispn.getSearchKeyExpressions().size();
if (ispn.getLookupType() == IndexLookupType.LT || ispn.getLookupType() == IndexLookupType.LTE) {
exprs = ExpressionUtil.uncombinePredicate(ispn.getInitialExpression());
numOfSearchKeys -= 1;
} else {
exprs = ExpressionUtil.uncombinePredicate(ispn.getEndExpression());
}
int numberOfExprs = exprs.size();
/* Retrieve the index expressions from the target index. (ENG-8819, Ethan)
* This is because we found that for the following two queries:
* #1: explain select max(c2/2) from t where c1=1 and c2/2<=3;
* #2: explain select max(c2/2) from t where c1=1 and c2/2<=?;
* We can get an inline limit 1 for #2 but not for #1. This is because all constants in #1 got parameterized.
* The result is that the query cannot pass the bindingToIndexedExpression() tests below
* because we lost all the constant value expressions (cannot attempt to bind a pve to a pve!).
* Those constant values expressions can only be accessed from the idnex.
* We will not add those bindings to the ispn.getBindings() here because they will be added anyway in checkIndex().
* PS: For this case (i.e. index on expressions), checkIndex() will call checkExpressionIndex(),
* where bindings will be added.
*/
Index indexToUse = ispn.getCatalogIndex();
String tableAlias = ispn.getTargetTableAlias();
List<AbstractExpression> indexedExprs = null;
if (!indexToUse.getExpressionsjson().isEmpty()) {
StmtTableScan tableScan = m_parsedStmt.getStmtTableScanByAlias(tableAlias);
try {
indexedExprs = AbstractExpression.fromJSONArrayString(indexToUse.getExpressionsjson(), tableScan);
} catch (JSONException e) {
e.printStackTrace();
assert (false);
return plan;
}
}
/* If there is only 1 difference between searchkeyExprs and endExprs,
* 1. trivial filters can be discarded, 2 possibilities:
* a. SELECT MIN(X) FROM T WHERE [other prefix filters] X < / <= ?
* <=> SELECT MIN(X) FROM T WHERE [other prefix filters] && the X < / <= ? filter
* b. SELECT MAX(X) FROM T WHERE X > / >= ?
* <=> SELECT MAX(X) FROM T with post-filter
* 2. filter should act as equality filter, 2 possibilities
* SELECT MIN(X) FROM T WHERE [other prefix filters] X > / >= ?
* SELECT MAX(X) FROM T WHERE [other prefix filters] X < / <= ?
* check if there is other filters for SELECT MAX(X) FROM T WHERE [other prefix filter AND ] X > / >= ?
* but we should allow SELECT MAX(X) FROM T WHERE X = ?
* This is for queries having MAX() but no ORDER BY. (ENG-8819, Ethan)
* sortDirection == DESC if max, ASC if min. ispn.getSortDirection() == INVALID if no ORDER BY. */
if (sortDirection == SortDirectionType.DESC && ispn.getSortDirection() == SortDirectionType.INVALID) {
/* numberOfExprs = exprs.size(), exprs are initial expressions for reversed index scans (lookupType LT, LTE),
* are end expressions for forward index scans (lookupType GT, GTE, EQ).
* Note, lookupType doesn't decide the scan direction for sure. MIN(X) where X < ? is still a forward scan.
* X < ? will be a post filter for the scan rather than an initial expression. */
if (numberOfExprs == 1) {
// e.g.: explain select max(c2/2) from t where c2/2<=3;
// In this case, as long as the where condition (exprs.get(0)) matches the aggregation argument, continue.
AbstractExpression exprToBind = indexedExprs == null ? exprs.get(0).getLeft() : indexedExprs.get(0);
if (aggExpr.bindingToIndexedExpression(exprToBind) == null) {
return plan;
}
} else if (numberOfExprs > 1) {
// ENG-4016: Optimization for query SELECT MAX(X) FROM T WHERE [other prefix filters] X < / <= ?
// Just keep trying, don't return early.
boolean earlyReturn = true;
for (int i = 0; i < numberOfExprs; ++i) {
AbstractExpression expr = exprs.get(i);
AbstractExpression indexedExpr = indexedExprs == null ? expr.getLeft() : indexedExprs.get(i);
if (aggExpr.bindingToIndexedExpression(indexedExpr) != null && (expr.getExpressionType() == ExpressionType.COMPARE_LESSTHANOREQUALTO || expr.getExpressionType() == ExpressionType.COMPARE_LESSTHAN || expr.getExpressionType() == ExpressionType.COMPARE_EQUAL)) {
earlyReturn = false;
break;
}
}
if (earlyReturn) {
return plan;
}
}
}
// have an upper bound: # of endingExpr is more than # of searchExpr
if (numberOfExprs > numOfSearchKeys) {
AbstractExpression lastEndExpr = exprs.get(numberOfExprs - 1);
// check last ending condition, see whether it is
// SELECT MIN(X) FROM T WHERE [other prefix filters] X < / <= ? or
// other filters will be checked later
AbstractExpression exprToBind = indexedExprs == null ? lastEndExpr.getLeft() : indexedExprs.get(numberOfExprs - 1);
if ((lastEndExpr.getExpressionType() == ExpressionType.COMPARE_LESSTHAN || lastEndExpr.getExpressionType() == ExpressionType.COMPARE_LESSTHANOREQUALTO) && aggExpr.bindingToIndexedExpression(exprToBind) != null) {
exprs.remove(lastEndExpr);
}
}
// and we can take advantage of that
if (checkIndex(ispn.getCatalogIndex(), aggExpr, exprs, ispn.getBindings(), tableAlias)) {
// we know which end we want to fetch, set the sort direction
ispn.setSortDirection(sortDirection);
// for SELECT MIN(X) FROM T WHERE [prefix filters] = ?
if (numberOfExprs == numOfSearchKeys && sortDirection == SortDirectionType.ASC) {
if (ispn.getLookupType() == IndexLookupType.GTE) {
assert (aggplan.isTableMin());
ispn.setSkipNullPredicate(numOfSearchKeys);
}
}
// reset the IndexLookupType, remove "added" searchKey, add back to endExpression, and clear "added" predicate
if (sortDirection == SortDirectionType.ASC && (ispn.getLookupType() == IndexLookupType.LT || ispn.getLookupType() == IndexLookupType.LTE)) {
ispn.setLookupType(IndexLookupType.GTE);
ispn.removeLastSearchKey();
ispn.addEndExpression(ExpressionUtil.uncombinePredicate(ispn.getInitialExpression()).get(numberOfExprs - 1));
ispn.setSkipNullPredicate(numOfSearchKeys);
ispn.resetPredicate();
}
// add an inline LIMIT plan node to this index scan plan node
LimitPlanNode lpn = new LimitPlanNode();
lpn.setLimit(1);
lpn.setOffset(0);
ispn.addInlinePlanNode(lpn);
// |__LimitPlanNode
if (sortDirection == SortDirectionType.DESC && !ispn.getSearchKeyExpressions().isEmpty() && exprs.isEmpty() && ExpressionUtil.uncombinePredicate(ispn.getInitialExpression()).isEmpty()) {
AbstractExpression newPredicate = new ComparisonExpression();
if (ispn.getLookupType() == IndexLookupType.GT)
newPredicate.setExpressionType(ExpressionType.COMPARE_GREATERTHAN);
if (ispn.getLookupType() == IndexLookupType.GTE)
newPredicate.setExpressionType(ExpressionType.COMPARE_GREATERTHANOREQUALTO);
newPredicate.setRight(ispn.getSearchKeyExpressions().get(0));
newPredicate.setLeft(aggExpr);
newPredicate.setValueType(aggExpr.getValueType());
ispn.clearSearchKeyExpression();
aggplan.setPrePredicate(newPredicate);
}
}
return plan;
}
use of org.voltdb.types.SortDirectionType in project voltdb by VoltDB.
the class PlanAssembler method handleWindowedOperators.
/**
* Create nodes for windowed operations.
*
* @param root
* @return
*/
private AbstractPlanNode handleWindowedOperators(AbstractPlanNode root) {
// Get the windowed expression. We need to set its output
// schema from the display list.
WindowFunctionExpression winExpr = m_parsedSelect.getWindowFunctionExpressions().get(0);
assert (winExpr != null);
// This will set the output schema to contain the
// windowed schema column only. In generateOutputSchema
// we will add the input columns.
WindowFunctionPlanNode pnode = new WindowFunctionPlanNode();
pnode.setWindowFunctionExpression(winExpr);
// We always need an order by plan node, even if the sort
// is optimized away by an index. This may be turned
// into an inline order by in a MergeReceivePlanNode.
IndexUseForOrderBy scanNode = findScanNodeForWindowFunction(root);
AbstractPlanNode cnode = null;
int winfunc = (scanNode == null) ? SubPlanAssembler.NO_INDEX_USE : scanNode.getWindowFunctionUsesIndex();
// statement level order by ordering.
if ((SubPlanAssembler.STATEMENT_LEVEL_ORDER_BY_INDEX == winfunc) || (SubPlanAssembler.NO_INDEX_USE == winfunc)) {
// No index. Calculate the expression order here and stuff it into
// the order by node. Note that if we support more than one window
// function this would be the case when scanNode.getWindowFunctionUsesIndex()
// returns a window function number which is different from the number
// of winExpr.
List<AbstractExpression> partitionByExpressions = winExpr.getPartitionByExpressions();
// If the order by expression list contains a partition by expression then
// we won't have to sort by it twice. We sort by the partition by expressions
// first, and we don't care what order we sort by them. So, find the
// sort direction in the order by list and use that in the partition by
// list, and then mark that it was deleted in the order by
// list.
//
// We choose to make this dontsort rather than dosort because the
// Java default value for boolean is false, and we want to sort by
// default.
boolean[] dontsort = new boolean[winExpr.getOrderbySize()];
List<AbstractExpression> orderByExpressions = winExpr.getOrderByExpressions();
List<SortDirectionType> orderByDirections = winExpr.getOrderByDirections();
OrderByPlanNode onode = new OrderByPlanNode();
for (int idx = 0; idx < winExpr.getPartitionbySize(); ++idx) {
SortDirectionType pdir = SortDirectionType.ASC;
AbstractExpression partitionByExpression = partitionByExpressions.get(idx);
int sidx = winExpr.getSortIndexOfOrderByExpression(partitionByExpression);
if (0 <= sidx) {
pdir = orderByDirections.get(sidx);
dontsort[sidx] = true;
}
onode.addSort(partitionByExpression, pdir);
}
for (int idx = 0; idx < winExpr.getOrderbySize(); ++idx) {
if (!dontsort[idx]) {
AbstractExpression orderByExpr = orderByExpressions.get(idx);
SortDirectionType orderByDir = orderByDirections.get(idx);
onode.addSort(orderByExpr, orderByDir);
}
}
onode.addAndLinkChild(root);
cnode = onode;
} else {
assert (scanNode != null);
// inline order by node of a MergeReceive node.
assert (0 == scanNode.getWindowFunctionUsesIndex());
if (m_partitioning.requiresTwoFragments()) {
OrderByPlanNode onode = new OrderByPlanNode();
SortDirectionType dir = scanNode.getSortOrderFromIndexScan();
assert (dir != SortDirectionType.INVALID);
// This was created when the index was determined.
// We cached it in the scan node.
List<AbstractExpression> orderExprs = scanNode.getFinalExpressionOrderFromIndexScan();
assert (orderExprs != null);
for (AbstractExpression ae : orderExprs) {
onode.addSort(ae, dir);
}
// Link in the OrderByNode.
onode.addAndLinkChild(root);
cnode = onode;
} else {
// Don't create and link in the order by node.
cnode = root;
}
}
pnode.addAndLinkChild(cnode);
return pnode;
}
use of org.voltdb.types.SortDirectionType in project voltdb by VoltDB.
the class PlannerTestCase method verifyOrderByPlanNode.
/**
* Call this function to verify that an order by plan node has the
* sort expressions and directions we expect.
*
* @param orderByPlanNode The plan node to test.
* @param columnDescrs Pairs of expressions and sort directions. There
* must be an even number of these, the even
* numbered ones must be expressions and the odd
* numbered ones must be sort directions. This is
* numbering starting at 0. So, they must be in
* the order expr, direction, expr, direction, and
* so forth.
*/
protected void verifyOrderByPlanNode(OrderByPlanNode orderByPlanNode, Object... columnDescrs) {
// We should have an even number of columns
assertEquals(0, columnDescrs.length % 2);
List<AbstractExpression> exprs = orderByPlanNode.getSortExpressions();
List<SortDirectionType> dirs = orderByPlanNode.getSortDirections();
assertEquals(exprs.size(), dirs.size());
assertEquals(columnDescrs.length / 2, exprs.size());
for (int idx = 0; idx < exprs.size(); ++idx) {
// Assert that an expected one-part name matches a tve by column name
// and an expected two-part name matches a tve by table and column name.
AbstractExpression expr = exprs.get(idx);
assertTrue(expr instanceof TupleValueExpression);
TupleValueExpression tve = (TupleValueExpression) expr;
String[] expectedNames = ((String) columnDescrs[2 * idx]).split("\\.");
String columnName = null;
int nParts = expectedNames.length;
if (nParts > 1) {
assertEquals(2, nParts);
String tableName = expectedNames[0].toUpperCase();
assertEquals(tableName, tve.getTableName().toUpperCase());
}
// In either case, the column name must match the LAST part.
columnName = expectedNames[nParts - 1].toUpperCase();
assertEquals(columnName, tve.getColumnName().toUpperCase());
SortDirectionType dir = dirs.get(idx);
assertEquals(columnDescrs[2 * idx + 1], dir);
}
}
use of org.voltdb.types.SortDirectionType in project voltdb by VoltDB.
the class TestWindowedFunctions method validateWindowedFunctionPlan.
/**
* Validate that each similar windowed query in testRank produces a similar
* plan, with the expected minor variation to its ORDER BY node.
* @param windowedQuery a variant of a test query of a known basic format
* @param nSorts the expected number of sort criteria that should have been
* extracted from the variant query's PARTITION BY and ORDER BY.
* @param descSortIndex the position among the sort criteria of the original
* ORDER BY column, always distinguishable by its "DESC" direction.
**/
private void validateWindowedFunctionPlan(String windowedQuery, int nSorts, int descSortIndex, int numPartitionExprs, ExpressionType winOpType) {
// Sometimes we get multi-fragment nodes when we
// expect single fragment nodes. Keeping all the fragments
// helps to diagnose the problem.
List<AbstractPlanNode> nodes = compileToFragments(windowedQuery);
assertEquals(1, nodes.size());
AbstractPlanNode node = nodes.get(0);
// The plan should look like:
// SendNode -> ProjectionPlanNode -> PartitionByPlanNode -> OrderByPlanNode -> SeqScanNode
// We also do some sanity checking on the PartitionPlan node.
// First dissect the plan.
assertTrue(node instanceof SendPlanNode);
AbstractPlanNode projPlanNode = node.getChild(0);
assertTrue(projPlanNode instanceof ProjectionPlanNode);
AbstractPlanNode windowFuncPlanNode = projPlanNode.getChild(0);
assertTrue(windowFuncPlanNode instanceof WindowFunctionPlanNode);
AbstractPlanNode abstractOrderByNode = windowFuncPlanNode.getChild(0);
assertTrue(abstractOrderByNode instanceof OrderByPlanNode);
OrderByPlanNode orderByNode = (OrderByPlanNode) abstractOrderByNode;
NodeSchema input_schema = orderByNode.getOutputSchema();
assertNotNull(input_schema);
AbstractPlanNode seqScanNode = orderByNode.getChild(0);
assertTrue(seqScanNode instanceof SeqScanPlanNode || seqScanNode instanceof NestLoopPlanNode);
WindowFunctionPlanNode wfPlanNode = (WindowFunctionPlanNode) windowFuncPlanNode;
NodeSchema schema = wfPlanNode.getOutputSchema();
//
// Check that the window function plan node's output schema is correct.
// Look at the first expression, to verify that it's the windowed expression.
// Then check that the TVEs all make sense.
//
SchemaColumn column = schema.getColumns().get(0);
assertEquals("ARANK", column.getColumnAlias());
assertEquals(numPartitionExprs, wfPlanNode.getPartitionByExpressions().size());
validateTVEs(input_schema, wfPlanNode, false);
//
// Check that the operation is what we expect.
//
assertTrue(wfPlanNode.getAggregateTypes().size() > 0);
assertEquals(winOpType, wfPlanNode.getAggregateTypes().get(0));
//
for (List<AbstractExpression> exprs : wfPlanNode.getAggregateExpressions()) {
if (exprs != null) {
for (AbstractExpression expr : exprs) {
assertNotNull(expr.getValueType());
}
}
}
//
// Check that the order by node has the right number of expressions.
// and that they have the correct order.
//
assertEquals(nSorts, orderByNode.getSortExpressions().size());
int sortIndex = 0;
for (SortDirectionType direction : orderByNode.getSortDirections()) {
SortDirectionType expected = (sortIndex == descSortIndex) ? SortDirectionType.DESC : SortDirectionType.ASC;
assertEquals(expected, direction);
++sortIndex;
}
}
Aggregations