use of org.voltdb.expressions.AbstractExpression in project voltdb by VoltDB.
the class JoinNode method applyTransitiveEquivalence.
private static List<AbstractExpression> applyTransitiveEquivalence(List<AbstractExpression> singleTableExprs, List<AbstractExpression> twoTableExprs) {
ArrayList<AbstractExpression> simplifiedExprs = new ArrayList<>();
HashMap<AbstractExpression, Set<AbstractExpression>> eqMap1 = new HashMap<>();
ExpressionUtil.collectPartitioningFilters(singleTableExprs, eqMap1);
for (AbstractExpression expr : twoTableExprs) {
if (!expr.isColumnEquivalenceFilter()) {
continue;
}
AbstractExpression leftExpr = expr.getLeft();
AbstractExpression rightExpr = expr.getRight();
assert (leftExpr instanceof TupleValueExpression && rightExpr instanceof TupleValueExpression);
Set<AbstractExpression> eqSet1 = eqMap1.get(leftExpr);
AbstractExpression singleExpr = leftExpr;
if (eqSet1 == null) {
eqSet1 = eqMap1.get(rightExpr);
if (eqSet1 == null) {
continue;
}
singleExpr = rightExpr;
}
for (AbstractExpression eqExpr : eqSet1) {
if (eqExpr instanceof ConstantValueExpression) {
if (singleExpr == leftExpr) {
expr.setLeft(eqExpr);
} else {
expr.setRight(eqExpr);
}
simplifiedExprs.add(expr);
// much sense, right?
break;
}
}
}
twoTableExprs.removeAll(simplifiedExprs);
return simplifiedExprs;
}
use of org.voltdb.expressions.AbstractExpression in project voltdb by VoltDB.
the class SubPlanAssembler method getRelevantAccessPathsForTable.
/**
* Generate all possible access paths for given sets of join and filter
* expressions for a table.
* The list includes the naive (scan) pass and possible index scans
*
* @param table Table to generate access path for
* @param joinExprs join expressions this table is part of
* @param filterExprs filter expressions this table is part of
* @param postExprs post expressions this table is part of
* @return List of valid access paths
*/
protected ArrayList<AccessPath> getRelevantAccessPathsForTable(StmtTableScan tableScan, List<AbstractExpression> joinExprs, List<AbstractExpression> filterExprs, List<AbstractExpression> postExprs) {
ArrayList<AccessPath> paths = new ArrayList<>();
List<AbstractExpression> allJoinExprs = new ArrayList<>();
List<AbstractExpression> allExprs = new ArrayList<>();
// add the empty seq-scan access path
if (joinExprs != null) {
allExprs.addAll(joinExprs);
allJoinExprs.addAll(joinExprs);
}
if (postExprs != null) {
allJoinExprs.addAll(postExprs);
}
if (filterExprs != null) {
allExprs.addAll(filterExprs);
}
AccessPath naivePath = getRelevantNaivePath(allJoinExprs, filterExprs);
paths.add(naivePath);
Collection<Index> indexes = tableScan.getIndexes();
for (Index index : indexes) {
AccessPath path = getRelevantAccessPathForIndex(tableScan, allExprs, index);
// Process the index WHERE clause into a list of anded
// sub-expressions and process each sub-expression, searching the
// query (or matview) WHERE clause for an expression to cover each
// of them. Coverage can be in the form of an identical filter or
// a more restrictive filter. Specifically, comparison filters like
// "X > 1" cover the index predicate filter "X is not null" but are
// not an exact match. As such, they are not completely optimized
// out by use of the partial index.
// The partial index's WHERE sub-expressions must be covered to
// allow ANY use of the index.
// Of the covering sub-expressions (from the query or view),
// those that also EXACTLY match their covered index sub-expression
// are tracked in exactMatchCoveringExprs so that they
// can be eliminated from the post-filter expressions.
List<AbstractExpression> exactMatchCoveringExprs = null;
boolean hasCoveredPredicate = false;
String predicatejson = index.getPredicatejson();
if (path == null) {
if (predicatejson.isEmpty()) {
// Skip the uselessly irrelevant whole-table index.
continue;
}
exactMatchCoveringExprs = new ArrayList<>();
hasCoveredPredicate = isPartialIndexPredicateCovered(tableScan, allExprs, predicatejson, exactMatchCoveringExprs);
if (!hasCoveredPredicate) {
// Skip the index with the inapplicable predicate.
continue;
}
// The partial index with a covered predicate can be used
// solely to eliminate a post-filter or even just to reduce the
// number of post-filtered tuples,
// even though its indexed columns are irrelevant -- so
// getRelevantAccessPathForIndex did not return a valid path.
// Override the path for a forward scan of the entire partial
// index. The irrelevant keys of the index are ignored.
path = getRelevantNaivePath(allJoinExprs, filterExprs);
path.index = index;
path.lookupType = IndexLookupType.GTE;
} else {
assert (path.index != null);
assert (path.index == index);
// its predicate is not applicable.
if (!predicatejson.isEmpty()) {
exactMatchCoveringExprs = new ArrayList<>();
hasCoveredPredicate = isPartialIndexPredicateCovered(tableScan, allExprs, predicatejson, exactMatchCoveringExprs);
if (!hasCoveredPredicate) {
// Skip the index with the inapplicable predicate.
continue;
}
}
}
assert (path != null);
if (hasCoveredPredicate) {
assert (exactMatchCoveringExprs != null);
filterPostPredicateForPartialIndex(path, exactMatchCoveringExprs);
}
if (postExprs != null) {
path.joinExprs.addAll(postExprs);
}
paths.add(path);
}
return paths;
}
use of org.voltdb.expressions.AbstractExpression in project voltdb by VoltDB.
the class SubPlanAssembler method getIndexableExpressionFromFilters.
/**
* For a given filter expression, return a normalized version of it that is always a comparison operator whose
* left-hand-side references the table specified and whose right-hand-side does not.
* Returns null if no such formulation of the filter expression is possible.
* For example, "WHERE F_ID = 2" would return it input intact if F_ID is in the table passed in.
* For join expressions like, "WHERE F_ID = Q_ID", it would also return the input expression if F_ID is in the table
* but Q_ID is not. If only Q_ID were defined for the table, it would return an expression for (Q_ID = F_ID).
* If both Q_ID and F_ID were defined on the table, null would be returned.
* Ideally, the left-hand-side expression is intended to be an indexed expression on the table using the current
* index. To help reduce false positives, the (base) columns and/or indexed expressions of the index are also
* provided to help further reduce non-null returns in uninteresting cases.
*
* @param targetComparator An allowed comparison operator
* -- its reverse is allowed in reversed expressions
* @param altTargetComparator An alternatively allowed comparison operator
* -- its reverse is allowed in reversed expressions
* @param coveringExpr The indexed expression on the table's column
* that might match a query filter, possibly null.
* @param coveringColId When coveringExpr is null,
* the id of the indexed column might match a query filter.
* @param tableScan The table scan on which the indexed expression is based
* @param filtersToCover the query conditions that may contain the desired filter
* @param allowIndexedJoinFilters Whether filters referencing other tables' columns are acceptable
* @param filterAction the desired disposition of the matched filter,
either EXCLUDE_FROM_POST_FILTERS or KEEP_IN_POST_FILTERS
* @return An IndexableExpression -- really just a pairing of a normalized form of expr with the
* potentially indexed expression on the left-hand-side and the potential index key expression on
* the right of a comparison operator, and a list of parameter bindings that are required for the
* index scan to be applicable.
* -- or null if there is no filter that matches the indexed expression
*/
private static IndexableExpression getIndexableExpressionFromFilters(ExpressionType targetComparator, ExpressionType altTargetComparator, AbstractExpression coveringExpr, int coveringColId, StmtTableScan tableScan, List<AbstractExpression> filtersToCover, boolean allowIndexedJoinFilters, boolean filterAction) {
List<AbstractExpression> binding = null;
AbstractExpression indexableExpr = null;
AbstractExpression otherExpr = null;
ComparisonExpression normalizedExpr = null;
AbstractExpression originalFilter = null;
for (AbstractExpression filter : filtersToCover) {
// ENG-8203: Not going to try to use index with sub-query expression
if (filter.hasSubquerySubexpression()) {
// SelectSubqueryExpression also can be scalar sub-query
continue;
}
// Expression type must be resolvable by an index scan
if ((filter.getExpressionType() == targetComparator) || (filter.getExpressionType() == altTargetComparator)) {
normalizedExpr = (ComparisonExpression) filter;
indexableExpr = filter.getLeft();
otherExpr = filter.getRight();
binding = bindingIfValidIndexedFilterOperand(tableScan, indexableExpr, otherExpr, coveringExpr, coveringColId);
if (binding != null) {
if (!allowIndexedJoinFilters) {
if (otherExpr.hasTupleValueSubexpression()) {
// This filter can not be used with the index, possibly due to interactions
// wih IN LIST processing that would require a three-way NLIJ.
binding = null;
continue;
}
}
// Additional restrictions apply to LIKE pattern arguments
if (targetComparator == ExpressionType.COMPARE_LIKE) {
if (otherExpr instanceof ParameterValueExpression) {
ParameterValueExpression pve = (ParameterValueExpression) otherExpr;
// Can't use an index for parameterized LIKE filters,
// e.g. "T1.column LIKE ?"
// UNLESS the parameter was artificially substituted
// for a user-specified constant AND that constant was a prefix pattern.
// In that case, the parameter has to be added to the bound list
// for this index/statement.
ConstantValueExpression cve = pve.getOriginalValue();
if (cve == null || !cve.isPrefixPatternString()) {
// the filter is not usable, so the binding is invalid
binding = null;
continue;
}
// Remember that the binding list returned by
// bindingIfValidIndexedFilterOperand above
// is often a "shared object" and is intended to be treated as immutable.
// To add a parameter to it, first copy the List.
List<AbstractExpression> moreBinding = new ArrayList<>(binding);
moreBinding.add(pve);
binding = moreBinding;
} else if (otherExpr instanceof ConstantValueExpression) {
// Can't use an index for non-prefix LIKE filters,
// e.g. " T1.column LIKE '%ish' "
ConstantValueExpression cve = (ConstantValueExpression) otherExpr;
if (!cve.isPrefixPatternString()) {
// The constant is not an index-friendly prefix pattern.
// the filter is not usable, so the binding is invalid
binding = null;
continue;
}
} else {
// Other cases are not indexable, e.g. " T1.column LIKE T2.column "
// the filter is not usable, so the binding is invalid
binding = null;
continue;
}
}
if (targetComparator == ExpressionType.COMPARE_IN) {
if (otherExpr.hasTupleValueSubexpression()) {
// This is a fancy edge case where the expression could only be indexed
// if it:
// A) does not reference the indexed table and
// B) has ee support for a three-way NLIJ where the table referenced in
// the list element expression feeds values from its current row to the
// Materialized scan which then re-evaluates its expressions to
// re-populate the temp table that drives the injected NLIJ with
// this index scan.
// This is a slightly more twisted variant of the three-way NLIJ that
// would be needed to support compound key indexing on a combination
// of (fixed) IN LIST elements and join key values from other tables.
// Punt for now on indexing this IN LIST filter.
// the filter is not usable, so the binding is invalid
binding = null;
continue;
}
if (otherExpr instanceof ParameterValueExpression) {
// It's OK to use an index for a parameterized IN filter,
// e.g. "T1.column IN ?"
// EVEN if the parameter was -- someday -- artificially substituted
// for an entire user-specified list of constants.
// As of now, that is beyond the capabilities of the ad hoc statement
// parameterizer, so "T1.column IN (3, 4)" can use the plan for
// "T1.column IN (?, ?)" that might have been originally cached for
// "T1.column IN (1, 2)" but "T1.column IN (1, 2, 3)" would need its own
// "T1.column IN (?, ?, ?)" plan, etc. per list element count.
} else //TODO: Some day, there may be an optimization here that allows an entire
// IN LIST of constants to be serialized as a single value instead of a
// VectorValue composed of ConstantValue arguments.
// What's TBD is whether that would get its own AbstractExpression class or
// just be a special case of ConstantValueExpression.
{
assert (otherExpr instanceof VectorValueExpression);
}
}
originalFilter = filter;
if (filterAction == EXCLUDE_FROM_POST_FILTERS) {
filtersToCover.remove(filter);
}
break;
}
}
if ((filter.getExpressionType() == ComparisonExpression.reverses.get(targetComparator)) || (filter.getExpressionType() == ComparisonExpression.reverses.get(altTargetComparator))) {
normalizedExpr = (ComparisonExpression) filter;
normalizedExpr = normalizedExpr.reverseOperator();
indexableExpr = filter.getRight();
otherExpr = filter.getLeft();
binding = bindingIfValidIndexedFilterOperand(tableScan, indexableExpr, otherExpr, coveringExpr, coveringColId);
if (binding != null) {
if (!allowIndexedJoinFilters) {
if (otherExpr.hasTupleValueSubexpression()) {
// This filter can not be used with the index, probably due to interactions
// with IN LIST processing of another key component that would require a
// three-way NLIJ to be injected.
binding = null;
continue;
}
}
originalFilter = filter;
if (filterAction == EXCLUDE_FROM_POST_FILTERS) {
filtersToCover.remove(filter);
}
break;
}
}
}
if (binding == null) {
// ran out of candidate filters.
return null;
}
return new IndexableExpression(originalFilter, normalizedExpr, binding);
}
use of org.voltdb.expressions.AbstractExpression in project voltdb by VoltDB.
the class SubPlanAssembler method determineIndexOrdering.
/**
* Determine if an index, which is in the AccessPath argument
* retval, can satisfy a parsed select statement's order by or
* window function ordering requirements.
*
* @param table only used here to validate base table names of ORDER BY columns' .
* @param bindingsForOrder restrictions on parameter settings that are prerequisite to the
* any ordering optimization determined here
* @param keyComponentCount the length of indexedExprs or indexedColRefs,
* ONE of which must be valid
* @param indexedExprs expressions for key components in the general case
* @param indexedColRefs column references for key components in the simpler case
* @param retval the eventual result of getRelevantAccessPathForIndex,
* the bearer of a (tentative) sortDirection determined here
* @param orderSpoilers positions of key components which MAY invalidate the tentative
* sortDirection
* @param bindingsForOrder restrictions on parameter settings that are prerequisite to the
* any ordering optimization determined here.
* @return the number of discovered orderSpoilers that will need to be recovered from,
* to maintain the established sortDirection - always 0 if no sort order was determined.
*/
private int determineIndexOrdering(StmtTableScan tableScan, int keyComponentCount, List<AbstractExpression> indexedExprs, List<ColumnRef> indexedColRefs, AccessPath retval, int[] orderSpoilers, List<AbstractExpression> bindingsForOrder) {
// Organize a little bit.
ParsedSelectStmt pss = (m_parsedStmt instanceof ParsedSelectStmt) ? ((ParsedSelectStmt) m_parsedStmt) : null;
boolean hasOrderBy = (m_parsedStmt.hasOrderByColumns() && (!m_parsedStmt.orderByColumns().isEmpty()));
boolean hasWindowFunctions = (pss != null && pss.hasWindowFunctionExpression());
//
if (!hasOrderBy && !hasWindowFunctions) {
return 0;
}
//
// We make a definition. Let S1 and S2 be sequences of expressions,
// and OS be an increasing sequence of indices into S2. Let erase(S2, OS) be
// the sequence of expressions which results from erasing all S2
// expressions whose indices are in OS. We say *S1 is a prefix of
// S2 with OS-singular values* if S1 is a prefix of erase(S2, OS).
// That is to say, if we erase the expressions in S2 whose indices are
// in OS, S1 is a prefix of the result.
//
// What expressions must we match?
// 1.) We have the parameters indexedExpressions and indexedColRefs.
// These are the expressions or column references in an index.
// Exactly one of them is non-null. Since these are both an
// indexed sequence of expression-like things, denote the
// non-null one IS.
// What expressions do we care about? We have two kinds.
// 1.) The expressions from the statement level order by clause, OO.
// This sequence of expressions must be a prefix of IS with OS
// singular values for some sequence OS. The sequence OS will be
// called the order spoilers. Later on we will test that the
// index expressions at the positions in OS can have only a
// single value.
// 2.) The expressions in a window function's partition by list, WP,
// followed by the expressions in the window function's
// order by list, WO. The partition by functions are a set not
// a sequence. We need to find a sequence of expressions, S,
// such that S is a permutation of P and S+WO is a singular prefix
// of IS.
//
// So, in both cases, statement level order by and window function, we are looking for
// a sequence of expressions, S1, and a list of IS indices, OS, such
// that S1 is a prefix of IS with OS-singular values.
//
// If the statement level order by expression list is not a prefix of
// the index, we still care to know about window functions. The reverse
// is not true. If there are window functions but they all fail to match the index,
// we must give up on this index, even if the statement level order
// by list is still matching. This is because the window functions will
// require sorting before the statement level order by clause's
// sort, and this window function sort will invalidate the statement level
// order by sort.
//
// Note also that it's possible that the statement level order by and
// the window function order by are compatible. So this index may provide
// all the sorting needed.
//
// There need to be enough indexed expressions to provide full sort coverage.
// More indexed expressions are ok.
//
// We keep a scoreboard which keeps track of everything. All the window
// functions and statement level order by functions are kept in the scoreboard.
//
WindowFunctionScoreboard windowFunctionScores = new WindowFunctionScoreboard(m_parsedStmt, tableScan);
// indexCtr is an index into the index expressions or columns.
for (int indexCtr = 0; !windowFunctionScores.isDone() && indexCtr < keyComponentCount; indexCtr += 1) {
// Figure out what to do with index expression or column at indexCtr.
// First, fetch it out.
AbstractExpression indexExpr = (indexedExprs == null) ? null : indexedExprs.get(indexCtr);
ColumnRef indexColRef = (indexedColRefs == null) ? null : indexedColRefs.get(indexCtr);
// Then see if it matches something. If
// this doesn't match one thing it may match
// another. If it doesn't match anything, it may
// be an order spoiler, which we will maintain in
// the scoreboard.
windowFunctionScores.matchIndexEntry(new ExpressionOrColumn(indexCtr, tableScan, indexExpr, SortDirectionType.INVALID, indexColRef));
}
//
return windowFunctionScores.getResult(retval, orderSpoilers, bindingsForOrder);
}
use of org.voltdb.expressions.AbstractExpression in project voltdb by VoltDB.
the class StatementPartitioning method analyzeForMultiPartitionAccess.
/**
* Given the query's list of tables and its collection(s) of equality-filtered columns and their equivalents,
* determine whether all joins involving partitioned tables can be executed locally on a single partition.
* This is only the case when they include equality comparisons between partition key columns.
* VoltDB will reject joins of multiple partitioned tables unless all their partition keys are
* constrained to be equal to each other.
* Example: select * from T1, T2 where T1.ID = T2.ID
* Additionally, in this case, there may be a constant equality filter on any of the columns,
* which we want to extract as our SP partitioning parameter.
*
* @param tableAliasList The tables.
* @param valueEquivalence Their column equality filters
* @return the number of independently partitioned tables
* -- partitioned tables that aren't joined or filtered by the same value.
* The caller can raise an alarm if there is more than one.
*/
public void analyzeForMultiPartitionAccess(Collection<StmtTableScan> scans, HashMap<AbstractExpression, Set<AbstractExpression>> valueEquivalence) {
//* enable to debug */ System.out.println("DEBUG: analyze4MPAccess w/ scans:" + scans.size() + " filters:" + valueEquivalence.size());
TupleValueExpression tokenPartitionKey = null;
Set<Set<AbstractExpression>> eqSets = new HashSet<Set<AbstractExpression>>();
int unfilteredPartitionKeyCount = 0;
// reset this flag to forget the last result of the multiple partition access path.
// AdHoc with parameters will call this function at least two times
// By default this flag should be true.
setJoinValid(true);
setJoinInvalidReason(null);
boolean subqueryHasReceiveNode = false;
boolean hasPartitionedTableJoin = false;
// Iterate over the tables to collect partition columns.
for (StmtTableScan tableScan : scans) {
// Replicated tables don't need filter coverage.
if (tableScan.getIsReplicated()) {
continue;
}
// The partition column can be null in an obscure edge case.
// The table is declared non-replicated yet specifies no partitioning column.
// This can occur legitimately when views based on partitioned tables neglect to group by the partition column.
// The interpretation of this edge case is that the table has "randomly distributed data".
// In such a case, the table is valid for use by MP queries only and can only be joined with replicated tables
// because it has no recognized partitioning join key.
List<SchemaColumn> columnsNeedingCoverage = tableScan.getPartitioningColumns();
if (tableScan instanceof StmtSubqueryScan) {
StmtSubqueryScan subScan = (StmtSubqueryScan) tableScan;
subScan.promoteSinglePartitionInfo(valueEquivalence, eqSets);
CompiledPlan subqueryPlan = subScan.getBestCostPlan();
if ((!subScan.canRunInOneFragment()) || ((subqueryPlan != null) && subqueryPlan.rootPlanGraph.hasAnyNodeOfClass(AbstractReceivePlanNode.class))) {
if (subqueryHasReceiveNode) {
// Has found another subquery with receive node on the same level
// Not going to support this kind of subquery join with 2 fragment plan.
setJoinValid(false);
setJoinInvalidReason("This multipartition query is not plannable. " + "It has a subquery which cannot be single partition.");
// Still needs to count the independent partition tables
break;
}
subqueryHasReceiveNode = true;
if (subScan.isTableAggregate()) {
// Any process based on this subquery should require 1 fragment only.
continue;
}
} else {
// this subquery partition table without receive node
hasPartitionedTableJoin = true;
}
} else {
// This table is a partition table
hasPartitionedTableJoin = true;
}
boolean unfiltered = true;
for (AbstractExpression candidateColumn : valueEquivalence.keySet()) {
if (!(candidateColumn instanceof TupleValueExpression)) {
continue;
}
TupleValueExpression candidatePartitionKey = (TupleValueExpression) candidateColumn;
if (!canCoverPartitioningColumn(candidatePartitionKey, columnsNeedingCoverage)) {
continue;
}
unfiltered = false;
if (tokenPartitionKey == null) {
tokenPartitionKey = candidatePartitionKey;
}
eqSets.add(valueEquivalence.get(candidatePartitionKey));
}
if (unfiltered) {
++unfilteredPartitionKeyCount;
}
}
// end for each table StmtTableScan in the collection
m_countOfIndependentlyPartitionedTables = eqSets.size() + unfilteredPartitionKeyCount;
//* enable to debug */ System.out.println("DEBUG: analyze4MPAccess found: " + m_countOfIndependentlyPartitionedTables + " = " + eqSets.size() + " + " + unfilteredPartitionKeyCount);
if (m_countOfIndependentlyPartitionedTables > 1) {
setJoinValid(false);
setJoinInvalidReason("This query is not plannable. " + "The planner cannot guarantee that all rows would be in a single partition.");
}
// on outer level. Not going to support this kind of join.
if (subqueryHasReceiveNode && hasPartitionedTableJoin) {
setJoinValid(false);
setJoinInvalidReason("This query is not plannable. It has a subquery which needs cross-partition access.");
}
if ((unfilteredPartitionKeyCount == 0) && (eqSets.size() == 1)) {
for (Set<AbstractExpression> partitioningValues : eqSets) {
for (AbstractExpression constExpr : partitioningValues) {
if (constExpr instanceof TupleValueExpression) {
continue;
}
VoltType valueType = tokenPartitionKey.getValueType();
addPartitioningExpression(tokenPartitionKey.getTableName() + '.' + tokenPartitionKey.getColumnName(), constExpr, valueType);
// Only need one constant value.
break;
}
}
}
}
Aggregations