use of org.apache.calcite.rex.RexTableInputRef in project calcite by apache.
the class AbstractMaterializedViewRule method generateEquivalenceClasses.
/**
* Given the equi-column predicates of the source and the target and the
* computed equivalence classes, it extracts possible mappings between
* the equivalence classes.
*
* <p>If there is no mapping, it returns null. If there is a exact match,
* it will return a compensation predicate that evaluates to true.
* Finally, if a compensation predicate needs to be enforced on top of
* the target to make the equivalences classes match, it returns that
* compensation predicate.
*/
private static RexNode generateEquivalenceClasses(RexBuilder rexBuilder, EquivalenceClasses sourceEC, EquivalenceClasses targetEC) {
if (sourceEC.getEquivalenceClasses().isEmpty() && targetEC.getEquivalenceClasses().isEmpty()) {
// Empty mapping and compensation predicate
return rexBuilder.makeLiteral(true);
}
if (sourceEC.getEquivalenceClasses().isEmpty() && !targetEC.getEquivalenceClasses().isEmpty()) {
// No column equality predicates in source, but column equality predicates in target
return null;
}
final List<Set<RexTableInputRef>> sourceEquivalenceClasses = sourceEC.getEquivalenceClasses();
final List<Set<RexTableInputRef>> targetEquivalenceClasses = targetEC.getEquivalenceClasses();
final Multimap<Integer, Integer> mapping = extractPossibleMapping(sourceEquivalenceClasses, targetEquivalenceClasses);
if (mapping == null) {
// bail out
return null;
}
// Create the compensation predicate
RexNode compensationPredicate = rexBuilder.makeLiteral(true);
for (int i = 0; i < sourceEquivalenceClasses.size(); i++) {
if (!mapping.containsKey(i)) {
// Add all predicates
Iterator<RexTableInputRef> it = sourceEquivalenceClasses.get(i).iterator();
RexTableInputRef e0 = it.next();
while (it.hasNext()) {
RexNode equals = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, e0, it.next());
compensationPredicate = rexBuilder.makeCall(SqlStdOperatorTable.AND, compensationPredicate, equals);
}
} else {
// Add only predicates that are not there
for (int j : mapping.get(i)) {
Set<RexTableInputRef> difference = new HashSet<>(sourceEquivalenceClasses.get(i));
difference.removeAll(targetEquivalenceClasses.get(j));
for (RexTableInputRef e : difference) {
RexNode equals = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, e, targetEquivalenceClasses.get(j).iterator().next());
compensationPredicate = rexBuilder.makeCall(SqlStdOperatorTable.AND, compensationPredicate, equals);
}
}
}
}
return compensationPredicate;
}
use of org.apache.calcite.rex.RexTableInputRef in project calcite by apache.
the class AbstractMaterializedViewRule method replaceWithOriginalReferences.
/**
* Given the input expression, it will replace (sub)expressions when possible
* using the content of the mapping. In particular, the mapping contains the
* digest of the expression and the index that the replacement input ref should
* point to.
*/
private static RexNode replaceWithOriginalReferences(final RexBuilder rexBuilder, final RelNode node, final NodeLineage nodeLineage, final RexNode exprToRewrite) {
// Currently we allow the following:
// 1) compensation pred can be directly map to expression
// 2) all references in compensation pred can be map to expressions
// We support bypassing lossless casts.
RexShuttle visitor = new RexShuttle() {
@Override
public RexNode visitCall(RexCall call) {
RexNode rw = replace(call);
return rw != null ? rw : super.visitCall(call);
}
@Override
public RexNode visitTableInputRef(RexTableInputRef inputRef) {
RexNode rw = replace(inputRef);
return rw != null ? rw : super.visitTableInputRef(inputRef);
}
private RexNode replace(RexNode e) {
Integer pos = nodeLineage.exprsLineage.get(e.toString());
if (pos != null) {
// Found it
return rexBuilder.makeInputRef(node, pos);
}
pos = nodeLineage.exprsLineageLosslessCasts.get(e.toString());
if (pos != null) {
// Found it
return rexBuilder.makeCast(e.getType(), rexBuilder.makeInputRef(node, pos));
}
return null;
}
};
return visitor.apply(exprToRewrite);
}
use of org.apache.calcite.rex.RexTableInputRef in project calcite by apache.
the class AbstractMaterializedViewRule method perform.
/**
* Rewriting logic is based on "Optimizing Queries Using Materialized Views:
* A Practical, Scalable Solution" by Goldstein and Larson.
*
* <p>On the query side, rules matches a Project-node chain or node, where node
* is either an Aggregate or a Join. Subplan rooted at the node operator must
* be composed of one or more of the following operators: TableScan, Project,
* Filter, and Join.
*
* <p>For each join MV, we need to check the following:
* <ol>
* <li> The plan rooted at the Join operator in the view produces all rows
* needed by the plan rooted at the Join operator in the query.</li>
* <li> All columns required by compensating predicates, i.e., predicates that
* need to be enforced over the view, are available at the view output.</li>
* <li> All output expressions can be computed from the output of the view.</li>
* <li> All output rows occur with the correct duplication factor. We might
* rely on existing Unique-Key - Foreign-Key relationships to extract that
* information.</li>
* </ol>
*
* <p>In turn, for each aggregate MV, we need to check the following:
* <ol>
* <li> The plan rooted at the Aggregate operator in the view produces all rows
* needed by the plan rooted at the Aggregate operator in the query.</li>
* <li> All columns required by compensating predicates, i.e., predicates that
* need to be enforced over the view, are available at the view output.</li>
* <li> The grouping columns in the query are a subset of the grouping columns
* in the view.</li>
* <li> All columns required to perform further grouping are available in the
* view output.</li>
* <li> All columns required to compute output expressions are available in the
* view output.</li>
* </ol>
*
* <p>The rule contains multiple extensions compared to the original paper. One of
* them is the possibility of creating rewritings using Union operators, e.g., if
* the result of a query is partially contained in the materialized view.
*/
protected void perform(RelOptRuleCall call, Project topProject, RelNode node) {
final RexBuilder rexBuilder = node.getCluster().getRexBuilder();
final RelMetadataQuery mq = RelMetadataQuery.instance();
final RelOptPlanner planner = call.getPlanner();
final RexExecutor executor = Util.first(planner.getExecutor(), RexUtil.EXECUTOR);
final RelOptPredicateList predicates = RelOptPredicateList.EMPTY;
final RexSimplify simplify = new RexSimplify(rexBuilder, predicates, true, executor);
final List<RelOptMaterialization> materializations = (planner instanceof VolcanoPlanner) ? ((VolcanoPlanner) planner).getMaterializations() : ImmutableList.<RelOptMaterialization>of();
if (!materializations.isEmpty()) {
// try to generate a rewriting are met
if (!isValidPlan(topProject, node, mq)) {
return;
}
// Obtain applicable (filtered) materializations
// TODO: Filtering of relevant materializations needs to be
// improved so we gather only materializations that might
// actually generate a valid rewriting.
final List<RelOptMaterialization> applicableMaterializations = RelOptMaterializations.getApplicableMaterializations(node, materializations);
if (!applicableMaterializations.isEmpty()) {
// 2. Initialize all query related auxiliary data structures
// that will be used throughout query rewriting process
// Generate query table references
final Set<RelTableRef> queryTableRefs = mq.getTableReferences(node);
if (queryTableRefs == null) {
// Bail out
return;
}
// Extract query predicates
final RelOptPredicateList queryPredicateList = mq.getAllPredicates(node);
if (queryPredicateList == null) {
// Bail out
return;
}
final RexNode pred = simplify.simplify(RexUtil.composeConjunction(rexBuilder, queryPredicateList.pulledUpPredicates, false));
final Triple<RexNode, RexNode, RexNode> queryPreds = splitPredicates(rexBuilder, pred);
// Extract query equivalence classes. An equivalence class is a set
// of columns in the query output that are known to be equal.
final EquivalenceClasses qEC = new EquivalenceClasses();
for (RexNode conj : RelOptUtil.conjunctions(queryPreds.getLeft())) {
assert conj.isA(SqlKind.EQUALS);
RexCall equiCond = (RexCall) conj;
qEC.addEquivalenceClass((RexTableInputRef) equiCond.getOperands().get(0), (RexTableInputRef) equiCond.getOperands().get(1));
}
// rewrite the given query
for (RelOptMaterialization materialization : applicableMaterializations) {
RelNode view = materialization.tableRel;
Project topViewProject;
RelNode viewNode;
if (materialization.queryRel instanceof Project) {
topViewProject = (Project) materialization.queryRel;
viewNode = topViewProject.getInput();
} else {
topViewProject = null;
viewNode = materialization.queryRel;
}
// 3.1. View checks before proceeding
if (!isValidPlan(topViewProject, viewNode, mq)) {
// Skip it
continue;
}
// 3.2. Initialize all query related auxiliary data structures
// that will be used throughout query rewriting process
// Extract view predicates
final RelOptPredicateList viewPredicateList = mq.getAllPredicates(viewNode);
if (viewPredicateList == null) {
// Skip it
continue;
}
final RexNode viewPred = simplify.simplify(RexUtil.composeConjunction(rexBuilder, viewPredicateList.pulledUpPredicates, false));
final Triple<RexNode, RexNode, RexNode> viewPreds = splitPredicates(rexBuilder, viewPred);
// Extract view table references
final Set<RelTableRef> viewTableRefs = mq.getTableReferences(viewNode);
if (viewTableRefs == null) {
// Bail out
return;
}
// Extract view tables
MatchModality matchModality;
Multimap<RexTableInputRef, RexTableInputRef> compensationEquiColumns = ArrayListMultimap.create();
if (!queryTableRefs.equals(viewTableRefs)) {
// subset of query tables (add additional tables through joins if possible)
if (viewTableRefs.containsAll(queryTableRefs)) {
matchModality = MatchModality.QUERY_PARTIAL;
final EquivalenceClasses vEC = new EquivalenceClasses();
for (RexNode conj : RelOptUtil.conjunctions(viewPreds.getLeft())) {
assert conj.isA(SqlKind.EQUALS);
RexCall equiCond = (RexCall) conj;
vEC.addEquivalenceClass((RexTableInputRef) equiCond.getOperands().get(0), (RexTableInputRef) equiCond.getOperands().get(1));
}
if (!compensatePartial(viewTableRefs, vEC, queryTableRefs, compensationEquiColumns)) {
// Cannot rewrite, skip it
continue;
}
} else if (queryTableRefs.containsAll(viewTableRefs)) {
matchModality = MatchModality.VIEW_PARTIAL;
ViewPartialRewriting partialRewritingResult = compensateViewPartial(call.builder(), rexBuilder, mq, view, topProject, node, queryTableRefs, qEC, topViewProject, viewNode, viewTableRefs);
if (partialRewritingResult == null) {
// Cannot rewrite, skip it
continue;
}
// Rewrite succeeded
view = partialRewritingResult.newView;
topViewProject = partialRewritingResult.newTopViewProject;
viewNode = partialRewritingResult.newViewNode;
} else {
// Skip it
continue;
}
} else {
matchModality = MatchModality.COMPLETE;
}
// 4. We map every table in the query to a table with the same qualified
// name (all query tables are contained in the view, thus this is equivalent
// to mapping every table in the query to a view table).
final Multimap<RelTableRef, RelTableRef> multiMapTables = ArrayListMultimap.create();
for (RelTableRef queryTableRef1 : queryTableRefs) {
for (RelTableRef queryTableRef2 : queryTableRefs) {
if (queryTableRef1.getQualifiedName().equals(queryTableRef2.getQualifiedName())) {
multiMapTables.put(queryTableRef1, queryTableRef2);
}
}
}
// If a table is used multiple times, we will create multiple mappings,
// and we will try to rewrite the query using each of the mappings.
// Then, we will try to map every source table (query) to a target
// table (view), and if we are successful, we will try to create
// compensation predicates to filter the view results further
// (if needed).
final List<BiMap<RelTableRef, RelTableRef>> flatListMappings = generateTableMappings(multiMapTables);
for (BiMap<RelTableRef, RelTableRef> queryToViewTableMapping : flatListMappings) {
// TableMapping : mapping query tables -> view tables
// 4.0. If compensation equivalence classes exist, we need to add
// the mapping to the query mapping
final EquivalenceClasses currQEC = EquivalenceClasses.copy(qEC);
if (matchModality == MatchModality.QUERY_PARTIAL) {
for (Entry<RexTableInputRef, RexTableInputRef> e : compensationEquiColumns.entries()) {
// Copy origin
RelTableRef queryTableRef = queryToViewTableMapping.inverse().get(e.getKey().getTableRef());
RexTableInputRef queryColumnRef = RexTableInputRef.of(queryTableRef, e.getKey().getIndex(), e.getKey().getType());
// Add to query equivalence classes and table mapping
currQEC.addEquivalenceClass(queryColumnRef, e.getValue());
queryToViewTableMapping.put(e.getValue().getTableRef(), // identity
e.getValue().getTableRef());
}
}
// 4.1. Compute compensation predicates, i.e., predicates that need to be
// enforced over the view to retain query semantics. The resulting predicates
// are expressed using {@link RexTableInputRef} over the query.
// First, to establish relationship, we swap column references of the view
// predicates to point to query tables and compute equivalence classes.
final RexNode viewColumnsEquiPred = RexUtil.swapTableReferences(rexBuilder, viewPreds.getLeft(), queryToViewTableMapping.inverse());
final EquivalenceClasses queryBasedVEC = new EquivalenceClasses();
for (RexNode conj : RelOptUtil.conjunctions(viewColumnsEquiPred)) {
assert conj.isA(SqlKind.EQUALS);
RexCall equiCond = (RexCall) conj;
queryBasedVEC.addEquivalenceClass((RexTableInputRef) equiCond.getOperands().get(0), (RexTableInputRef) equiCond.getOperands().get(1));
}
Triple<RexNode, RexNode, RexNode> compensationPreds = computeCompensationPredicates(rexBuilder, simplify, currQEC, queryPreds, queryBasedVEC, viewPreds, queryToViewTableMapping);
if (compensationPreds == null && generateUnionRewriting) {
// Attempt partial rewriting using union operator. This rewriting
// will read some data from the view and the rest of the data from
// the query computation. The resulting predicates are expressed
// using {@link RexTableInputRef} over the view.
compensationPreds = computeCompensationPredicates(rexBuilder, simplify, queryBasedVEC, viewPreds, currQEC, queryPreds, queryToViewTableMapping.inverse());
if (compensationPreds == null) {
// This was our last chance to use the view, skip it
continue;
}
RexNode compensationColumnsEquiPred = compensationPreds.getLeft();
RexNode otherCompensationPred = RexUtil.composeConjunction(rexBuilder, ImmutableList.of(compensationPreds.getMiddle(), compensationPreds.getRight()), false);
assert !compensationColumnsEquiPred.isAlwaysTrue() || !otherCompensationPred.isAlwaysTrue();
// b. Generate union branch (query).
final RelNode unionInputQuery = rewriteQuery(call.builder(), rexBuilder, simplify, mq, compensationColumnsEquiPred, otherCompensationPred, topProject, node, queryToViewTableMapping, queryBasedVEC, currQEC);
if (unionInputQuery == null) {
// Skip it
continue;
}
// c. Generate union branch (view).
// We trigger the unifying method. This method will either create a Project
// or an Aggregate operator on top of the view. It will also compute the
// output expressions for the query.
final RelNode unionInputView = rewriteView(call.builder(), rexBuilder, simplify, mq, matchModality, true, view, topProject, node, topViewProject, viewNode, queryToViewTableMapping, currQEC);
if (unionInputView == null) {
// Skip it
continue;
}
// d. Generate final rewriting (union).
final RelNode result = createUnion(call.builder(), rexBuilder, topProject, unionInputQuery, unionInputView);
if (result == null) {
// Skip it
continue;
}
call.transformTo(result);
} else if (compensationPreds != null) {
RexNode compensationColumnsEquiPred = compensationPreds.getLeft();
RexNode otherCompensationPred = RexUtil.composeConjunction(rexBuilder, ImmutableList.of(compensationPreds.getMiddle(), compensationPreds.getRight()), false);
// a. Compute final compensation predicate.
if (!compensationColumnsEquiPred.isAlwaysTrue() || !otherCompensationPred.isAlwaysTrue()) {
// All columns required by compensating predicates must be contained
// in the view output (condition 2).
List<RexNode> viewExprs = topViewProject == null ? extractReferences(rexBuilder, view) : topViewProject.getChildExps();
// since we want to enforce the rest
if (!compensationColumnsEquiPred.isAlwaysTrue()) {
compensationColumnsEquiPred = rewriteExpression(rexBuilder, mq, view, viewNode, viewExprs, queryToViewTableMapping.inverse(), queryBasedVEC, false, compensationColumnsEquiPred);
if (compensationColumnsEquiPred == null) {
// Skip it
continue;
}
}
// For the rest, we use the query equivalence classes
if (!otherCompensationPred.isAlwaysTrue()) {
otherCompensationPred = rewriteExpression(rexBuilder, mq, view, viewNode, viewExprs, queryToViewTableMapping.inverse(), currQEC, true, otherCompensationPred);
if (otherCompensationPred == null) {
// Skip it
continue;
}
}
}
final RexNode viewCompensationPred = RexUtil.composeConjunction(rexBuilder, ImmutableList.of(compensationColumnsEquiPred, otherCompensationPred), false);
// b. Generate final rewriting if possible.
// First, we add the compensation predicate (if any) on top of the view.
// Then, we trigger the unifying method. This method will either create a
// Project or an Aggregate operator on top of the view. It will also compute
// the output expressions for the query.
RelBuilder builder = call.builder();
RelNode viewWithFilter;
if (!viewCompensationPred.isAlwaysTrue()) {
RexNode newPred = simplify.simplify(viewCompensationPred);
viewWithFilter = builder.push(view).filter(newPred).build();
// We add (and push) the filter to the view plan before triggering the rewriting.
// This is useful in case some of the columns can be folded to same value after
// filter is added.
Pair<RelNode, RelNode> pushedNodes = pushFilterToOriginalViewPlan(builder, topViewProject, viewNode, newPred);
topViewProject = (Project) pushedNodes.left;
viewNode = pushedNodes.right;
} else {
viewWithFilter = builder.push(view).build();
}
final RelNode result = rewriteView(builder, rexBuilder, simplify, mq, matchModality, false, viewWithFilter, topProject, node, topViewProject, viewNode, queryToViewTableMapping, currQEC);
if (result == null) {
// Skip it
continue;
}
call.transformTo(result);
}
// end else
}
}
}
}
}
use of org.apache.calcite.rex.RexTableInputRef in project calcite by apache.
the class AbstractMaterializedViewRule method compensatePartial.
/**
* It checks whether the target can be rewritten using the source even though the
* source uses additional tables. In order to do that, we need to double-check
* that every join that exists in the source and is not in the target is a
* cardinality-preserving join, i.e., it only appends columns to the row
* without changing its multiplicity. Thus, the join needs to be:
* <ul>
* <li> Equi-join </li>
* <li> Between all columns in the keys </li>
* <li> Foreign-key columns do not allow NULL values </li>
* <li> Foreign-key </li>
* <li> Unique-key </li>
* </ul>
*
* <p>If it can be rewritten, it returns true. Further, it inserts the missing equi-join
* predicates in the input {@code compensationEquiColumns} multimap if it is provided.
* If it cannot be rewritten, it returns false.
*/
private static boolean compensatePartial(Set<RelTableRef> sourceTableRefs, EquivalenceClasses sourceEC, Set<RelTableRef> targetTableRefs, Multimap<RexTableInputRef, RexTableInputRef> compensationEquiColumns) {
// Create UK-FK graph with view tables
final DirectedGraph<RelTableRef, Edge> graph = DefaultDirectedGraph.create(Edge.FACTORY);
final Multimap<List<String>, RelTableRef> tableVNameToTableRefs = ArrayListMultimap.create();
final Set<RelTableRef> extraTableRefs = new HashSet<>();
for (RelTableRef tRef : sourceTableRefs) {
// Add tables in view as vertices
graph.addVertex(tRef);
tableVNameToTableRefs.put(tRef.getQualifiedName(), tRef);
if (!targetTableRefs.contains(tRef)) {
// Add to extra tables if table is not part of the query
extraTableRefs.add(tRef);
}
}
for (RelTableRef tRef : graph.vertexSet()) {
// Add edges between tables
List<RelReferentialConstraint> constraints = tRef.getTable().getReferentialConstraints();
for (RelReferentialConstraint constraint : constraints) {
Collection<RelTableRef> parentTableRefs = tableVNameToTableRefs.get(constraint.getTargetQualifiedName());
for (RelTableRef parentTRef : parentTableRefs) {
boolean canBeRewritten = true;
Multimap<RexTableInputRef, RexTableInputRef> equiColumns = ArrayListMultimap.create();
for (int pos = 0; pos < constraint.getNumColumns(); pos++) {
int foreignKeyPos = constraint.getColumnPairs().get(pos).source;
RelDataType foreignKeyColumnType = tRef.getTable().getRowType().getFieldList().get(foreignKeyPos).getType();
RexTableInputRef foreignKeyColumnRef = RexTableInputRef.of(tRef, foreignKeyPos, foreignKeyColumnType);
int uniqueKeyPos = constraint.getColumnPairs().get(pos).target;
RexTableInputRef uniqueKeyColumnRef = RexTableInputRef.of(parentTRef, uniqueKeyPos, parentTRef.getTable().getRowType().getFieldList().get(uniqueKeyPos).getType());
if (!foreignKeyColumnType.isNullable() && sourceEC.getEquivalenceClassesMap().containsKey(uniqueKeyColumnRef) && sourceEC.getEquivalenceClassesMap().get(uniqueKeyColumnRef).contains(foreignKeyColumnRef)) {
equiColumns.put(foreignKeyColumnRef, uniqueKeyColumnRef);
} else {
canBeRewritten = false;
break;
}
}
if (canBeRewritten) {
// Add edge FK -> UK
Edge edge = graph.getEdge(tRef, parentTRef);
if (edge == null) {
edge = graph.addEdge(tRef, parentTRef);
}
edge.equiColumns.putAll(equiColumns);
}
}
}
}
// Try to eliminate tables from graph: if we can do it, it means extra tables in
// view are cardinality-preserving joins
boolean done = false;
do {
List<RelTableRef> nodesToRemove = new ArrayList<>();
for (RelTableRef tRef : graph.vertexSet()) {
if (graph.getInwardEdges(tRef).size() == 1 && graph.getOutwardEdges(tRef).isEmpty()) {
// UK-FK join
nodesToRemove.add(tRef);
if (compensationEquiColumns != null && extraTableRefs.contains(tRef)) {
// We need to add to compensation columns as the table is not present in the query
compensationEquiColumns.putAll(graph.getInwardEdges(tRef).get(0).equiColumns);
}
}
}
if (!nodesToRemove.isEmpty()) {
graph.removeAllVertices(nodesToRemove);
} else {
done = true;
}
} while (!done);
// are tables present in the query: if they are, we can try to rewrite
if (!Collections.disjoint(graph.vertexSet(), extraTableRefs)) {
return false;
}
return true;
}
use of org.apache.calcite.rex.RexTableInputRef in project calcite by apache.
the class AbstractMaterializedViewRule method extractPossibleMapping.
/**
* Given the source and target equivalence classes, it extracts the possible mappings
* from each source equivalence class to each target equivalence class.
*
* <p>If any of the source equivalence classes cannot be mapped to a target equivalence
* class, it returns null.
*/
private static Multimap<Integer, Integer> extractPossibleMapping(List<Set<RexTableInputRef>> sourceEquivalenceClasses, List<Set<RexTableInputRef>> targetEquivalenceClasses) {
Multimap<Integer, Integer> mapping = ArrayListMultimap.create();
for (int i = 0; i < targetEquivalenceClasses.size(); i++) {
boolean foundQueryEquivalenceClass = false;
final Set<RexTableInputRef> viewEquivalenceClass = targetEquivalenceClasses.get(i);
for (int j = 0; j < sourceEquivalenceClasses.size(); j++) {
final Set<RexTableInputRef> queryEquivalenceClass = sourceEquivalenceClasses.get(j);
if (queryEquivalenceClass.containsAll(viewEquivalenceClass)) {
mapping.put(j, i);
foundQueryEquivalenceClass = true;
break;
}
}
if (!foundQueryEquivalenceClass) {
// Target equivalence class not found in source equivalence class
return null;
}
}
return mapping;
}
Aggregations