use of org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.LogicalOperatorDeepCopyWithNewVariablesVisitor in project asterixdb by apache.
the class FuzzyJoinRule method rewritePost.
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
// current opperator is join
if (op.getOperatorTag() != LogicalOperatorTag.INNERJOIN && op.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
return false;
}
// Find GET_ITEM function.
AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) op;
Mutable<ILogicalExpression> expRef = joinOp.getCondition();
Mutable<ILogicalExpression> getItemExprRef = getSimilarityExpression(expRef);
if (getItemExprRef == null) {
return false;
}
// Check if the GET_ITEM function is on one of the supported similarity-check functions.
AbstractFunctionCallExpression getItemFuncExpr = (AbstractFunctionCallExpression) getItemExprRef.getValue();
Mutable<ILogicalExpression> argRef = getItemFuncExpr.getArguments().get(0);
AbstractFunctionCallExpression simFuncExpr = (AbstractFunctionCallExpression) argRef.getValue();
if (!simFuncs.contains(simFuncExpr.getFunctionIdentifier())) {
return false;
}
// Skip this rule based on annotations.
if (simFuncExpr.getAnnotations().containsKey(IndexedNLJoinExpressionAnnotation.INSTANCE)) {
return false;
}
List<Mutable<ILogicalOperator>> inputOps = joinOp.getInputs();
ILogicalOperator leftInputOp = inputOps.get(0).getValue();
ILogicalOperator rightInputOp = inputOps.get(1).getValue();
List<Mutable<ILogicalExpression>> inputExps = simFuncExpr.getArguments();
ILogicalExpression inputExp0 = inputExps.get(0).getValue();
ILogicalExpression inputExp1 = inputExps.get(1).getValue();
// left and right expressions are variables
if (inputExp0.getExpressionTag() != LogicalExpressionTag.VARIABLE || inputExp1.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
return false;
}
LogicalVariable inputVar0 = ((VariableReferenceExpression) inputExp0).getVariableReference();
LogicalVariable inputVar1 = ((VariableReferenceExpression) inputExp1).getVariableReference();
LogicalVariable leftInputVar;
LogicalVariable rightInputVar;
liveVars.clear();
VariableUtilities.getLiveVariables(leftInputOp, liveVars);
if (liveVars.contains(inputVar0)) {
leftInputVar = inputVar0;
rightInputVar = inputVar1;
} else {
leftInputVar = inputVar1;
rightInputVar = inputVar0;
}
List<LogicalVariable> leftInputPKs = context.findPrimaryKey(leftInputVar);
List<LogicalVariable> rightInputPKs = context.findPrimaryKey(rightInputVar);
// Bail if primary keys could not be inferred.
if (leftInputPKs == null || rightInputPKs == null) {
return false;
}
// primary key has only one variable
if (leftInputPKs.size() != 1 || rightInputPKs.size() != 1) {
return false;
}
IAType leftType = (IAType) context.getOutputTypeEnvironment(leftInputOp).getVarType(leftInputVar);
IAType rightType = (IAType) context.getOutputTypeEnvironment(rightInputOp).getVarType(rightInputVar);
// left-hand side and right-hand side of "~=" has the same type
IAType left2 = TypeComputeUtils.getActualType(leftType);
IAType right2 = TypeComputeUtils.getActualType(rightType);
if (!left2.deepEqual(right2)) {
return false;
}
//
// -- - FIRE - --
//
MetadataProvider metadataProvider = ((MetadataProvider) context.getMetadataProvider());
FunctionIdentifier funcId = FuzzyUtils.getTokenizer(leftType.getTypeTag());
String tokenizer;
if (funcId == null) {
tokenizer = "";
} else {
tokenizer = funcId.getName();
}
float simThreshold = FuzzyUtils.getSimThreshold(metadataProvider);
String simFunction = FuzzyUtils.getSimFunction(metadataProvider);
// finalize AQL+ query
String prepareJoin;
switch(joinOp.getJoinKind()) {
case INNER:
{
prepareJoin = "join" + AQLPLUS;
break;
}
case LEFT_OUTER:
{
// other sort of bug.
return false;
// prepareJoin = "loj" + AQLPLUS;
// break;
}
default:
{
throw new IllegalStateException();
}
}
String aqlPlus = String.format(Locale.US, prepareJoin, tokenizer, tokenizer, simFunction, simThreshold, tokenizer, tokenizer, simFunction, simThreshold, simFunction, simThreshold, simThreshold);
LogicalVariable leftPKVar = leftInputPKs.get(0);
LogicalVariable rightPKVar = rightInputPKs.get(0);
Counter counter = new Counter(context.getVarCounter());
AQLPlusParser parser = new AQLPlusParser(new StringReader(aqlPlus));
parser.initScope();
parser.setVarCounter(counter);
List<Clause> clauses;
try {
clauses = parser.Clauses();
} catch (ParseException e) {
throw new AlgebricksException(e);
}
// The translator will compile metadata internally. Run this compilation
// under the same transaction id as the "outer" compilation.
AqlPlusExpressionToPlanTranslator translator = new AqlPlusExpressionToPlanTranslator(metadataProvider, counter);
context.setVarCounter(counter.get());
LogicalOperatorDeepCopyWithNewVariablesVisitor deepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor(context, context);
translator.addOperatorToMetaScope(new Identifier("#LEFT"), leftInputOp);
translator.addVariableToMetaScope(new Identifier("$$LEFT"), leftInputVar);
translator.addVariableToMetaScope(new Identifier("$$LEFTPK"), leftPKVar);
translator.addOperatorToMetaScope(new Identifier("#RIGHT"), rightInputOp);
translator.addVariableToMetaScope(new Identifier("$$RIGHT"), rightInputVar);
translator.addVariableToMetaScope(new Identifier("$$RIGHTPK"), rightPKVar);
translator.addOperatorToMetaScope(new Identifier("#LEFT_1"), deepCopyVisitor.deepCopy(leftInputOp));
translator.addVariableToMetaScope(new Identifier("$$LEFT_1"), deepCopyVisitor.varCopy(leftInputVar));
translator.addVariableToMetaScope(new Identifier("$$LEFTPK_1"), deepCopyVisitor.varCopy(leftPKVar));
deepCopyVisitor.updatePrimaryKeys(context);
deepCopyVisitor.reset();
// translator.addOperatorToMetaScope(new Identifier("#LEFT_2"),
// deepCopyVisitor.deepCopy(leftInputOp, null));
// translator.addVariableToMetaScope(new Identifier("$$LEFT_2"),
// deepCopyVisitor.varCopy(leftInputVar));
// translator.addVariableToMetaScope(new Identifier("$$LEFTPK_2"),
// deepCopyVisitor.varCopy(leftPKVar));
// deepCopyVisitor.updatePrimaryKeys(context);
// deepCopyVisitor.reset();
//
// translator.addOperatorToMetaScope(new Identifier("#LEFT_3"),
// deepCopyVisitor.deepCopy(leftInputOp, null));
// translator.addVariableToMetaScope(new Identifier("$$LEFT_3"),
// deepCopyVisitor.varCopy(leftInputVar));
// translator.addVariableToMetaScope(new Identifier("$$LEFTPK_3"),
// deepCopyVisitor.varCopy(leftPKVar));
// deepCopyVisitor.updatePrimaryKeys(context);
// deepCopyVisitor.reset();
translator.addOperatorToMetaScope(new Identifier("#RIGHT_1"), deepCopyVisitor.deepCopy(rightInputOp));
translator.addVariableToMetaScope(new Identifier("$$RIGHT_1"), deepCopyVisitor.varCopy(rightInputVar));
translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_1"), deepCopyVisitor.varCopy(rightPKVar));
deepCopyVisitor.updatePrimaryKeys(context);
deepCopyVisitor.reset();
// TODO pick side to run Stage 1, currently always picks RIGHT side
translator.addOperatorToMetaScope(new Identifier("#RIGHT_2"), deepCopyVisitor.deepCopy(rightInputOp));
translator.addVariableToMetaScope(new Identifier("$$RIGHT_2"), deepCopyVisitor.varCopy(rightInputVar));
translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_2"), deepCopyVisitor.varCopy(rightPKVar));
deepCopyVisitor.updatePrimaryKeys(context);
deepCopyVisitor.reset();
translator.addOperatorToMetaScope(new Identifier("#RIGHT_3"), deepCopyVisitor.deepCopy(rightInputOp));
translator.addVariableToMetaScope(new Identifier("$$RIGHT_3"), deepCopyVisitor.varCopy(rightInputVar));
translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_3"), deepCopyVisitor.varCopy(rightPKVar));
deepCopyVisitor.updatePrimaryKeys(context);
deepCopyVisitor.reset();
ILogicalPlan plan;
try {
plan = translator.translate(clauses);
} catch (AsterixException e) {
throw new AlgebricksException(e);
}
context.setVarCounter(counter.get());
ILogicalOperator outputOp = plan.getRoots().get(0).getValue();
SelectOperator extraSelect = null;
if (getItemExprRef != expRef) {
// more than one join condition
getItemExprRef.setValue(ConstantExpression.TRUE);
switch(joinOp.getJoinKind()) {
case INNER:
{
extraSelect = new SelectOperator(expRef, false, null);
extraSelect.getInputs().add(new MutableObject<ILogicalOperator>(outputOp));
outputOp = extraSelect;
break;
}
case LEFT_OUTER:
{
if (((AbstractLogicalOperator) outputOp).getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
throw new IllegalStateException();
}
LeftOuterJoinOperator topJoin = (LeftOuterJoinOperator) outputOp;
topJoin.getCondition().setValue(expRef.getValue());
break;
}
default:
{
throw new IllegalStateException();
}
}
}
opRef.setValue(outputOp);
OperatorPropertiesUtil.typeOpRec(opRef, context);
return true;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.LogicalOperatorDeepCopyWithNewVariablesVisitor in project asterixdb by apache.
the class InvertedIndexAccessMethod method copyAndReinitProbeSubTree.
/**
* Copies the probeSubTree (using new variables), and reinitializes the probeSubTree to it.
* Accordingly replaces the variables in the given joinCond, and the optFuncExpr.
* Returns a reference to the original plan root.
*/
private Mutable<ILogicalOperator> copyAndReinitProbeSubTree(OptimizableOperatorSubTree probeSubTree, ILogicalExpression joinCond, IOptimizableFuncExpr optFuncExpr, List<LogicalVariable> originalSubTreePKs, List<LogicalVariable> surrogateSubTreePKs, IOptimizationContext context) throws AlgebricksException {
probeSubTree.getPrimaryKeyVars(null, originalSubTreePKs);
// Create two copies of the original probe subtree.
// The first copy, which becomes the new probe subtree, will retain the primary-key and secondary-search key variables,
// but have all other variables replaced with new ones.
// The second copy, which will become an input to the top-level equi-join to resolve the surrogates,
// will have all primary-key and secondary-search keys replaced, but retains all other original variables.
// Variable replacement map for the first copy.
LinkedHashMap<LogicalVariable, LogicalVariable> newProbeSubTreeVarMap = new LinkedHashMap<>();
// Variable replacement map for the second copy.
LinkedHashMap<LogicalVariable, LogicalVariable> joinInputSubTreeVarMap = new LinkedHashMap<>();
// Init with all live vars.
List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
VariableUtilities.getLiveVariables(probeSubTree.getRoot(), liveVars);
for (LogicalVariable var : liveVars) {
joinInputSubTreeVarMap.put(var, var);
}
// Fill variable replacement maps.
for (int i = 0; i < optFuncExpr.getNumLogicalVars(); i++) {
joinInputSubTreeVarMap.put(optFuncExpr.getLogicalVar(i), context.newVar());
newProbeSubTreeVarMap.put(optFuncExpr.getLogicalVar(i), optFuncExpr.getLogicalVar(i));
}
for (int i = 0; i < originalSubTreePKs.size(); i++) {
LogicalVariable newPKVar = context.newVar();
surrogateSubTreePKs.add(newPKVar);
joinInputSubTreeVarMap.put(originalSubTreePKs.get(i), newPKVar);
newProbeSubTreeVarMap.put(originalSubTreePKs.get(i), originalSubTreePKs.get(i));
}
// Create first copy.
LogicalOperatorDeepCopyWithNewVariablesVisitor firstDeepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor(context, context, newProbeSubTreeVarMap);
ILogicalOperator newProbeSubTree = firstDeepCopyVisitor.deepCopy(probeSubTree.getRoot());
inferTypes(newProbeSubTree, context);
Mutable<ILogicalOperator> newProbeSubTreeRootRef = new MutableObject<ILogicalOperator>(newProbeSubTree);
// Create second copy.
LogicalOperatorDeepCopyWithNewVariablesVisitor secondDeepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor(context, context, joinInputSubTreeVarMap);
ILogicalOperator joinInputSubTree = secondDeepCopyVisitor.deepCopy(probeSubTree.getRoot());
inferTypes(joinInputSubTree, context);
probeSubTree.getRootRef().setValue(joinInputSubTree);
// Remember the original probe subtree reference so we can return it.
Mutable<ILogicalOperator> originalProbeSubTreeRootRef = probeSubTree.getRootRef();
// Replace the original probe subtree with its copy.
Dataset origDataset = probeSubTree.getDataset();
ARecordType origRecordType = probeSubTree.getRecordType();
probeSubTree.initFromSubTree(newProbeSubTreeRootRef);
probeSubTree.setDataset(origDataset);
probeSubTree.setRecordType(origRecordType);
// Replace the variables in the join condition based on the mapping of variables
// in the new probe subtree.
Map<LogicalVariable, LogicalVariable> varMapping = firstDeepCopyVisitor.getInputToOutputVariableMapping();
for (Map.Entry<LogicalVariable, LogicalVariable> varMapEntry : varMapping.entrySet()) {
if (varMapEntry.getKey() != varMapEntry.getValue()) {
joinCond.substituteVar(varMapEntry.getKey(), varMapEntry.getValue());
}
}
return originalProbeSubTreeRootRef;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.LogicalOperatorDeepCopyWithNewVariablesVisitor in project asterixdb by apache.
the class InvertedIndexAccessMethod method createPanicNestedLoopJoinPlan.
private Mutable<ILogicalOperator> createPanicNestedLoopJoinPlan(Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, IOptimizableFuncExpr optFuncExpr, Index chosenIndex, Map<LogicalVariable, LogicalVariable> panicVarMap, IOptimizationContext context) throws AlgebricksException {
LogicalVariable inputSearchVar = getInputSearchVar(optFuncExpr, indexSubTree);
// We split the plan into two "branches", and add selections on each side.
AbstractLogicalOperator replicateOp = new ReplicateOperator(2);
replicateOp.getInputs().add(new MutableObject<ILogicalOperator>(probeSubTree.getRoot()));
replicateOp.setExecutionMode(ExecutionMode.PARTITIONED);
context.computeAndSetTypeEnvironmentForOperator(replicateOp);
// Create select ops for removing tuples that are filterable and not filterable, respectively.
IVariableTypeEnvironment probeTypeEnv = context.getOutputTypeEnvironment(probeSubTree.getRoot());
IAType inputSearchVarType;
if (chosenIndex.isEnforcingKeyFileds()) {
inputSearchVarType = optFuncExpr.getFieldType(optFuncExpr.findLogicalVar(inputSearchVar));
} else {
inputSearchVarType = (IAType) probeTypeEnv.getVarType(inputSearchVar);
}
Mutable<ILogicalOperator> isFilterableSelectOpRef = new MutableObject<ILogicalOperator>();
Mutable<ILogicalOperator> isNotFilterableSelectOpRef = new MutableObject<ILogicalOperator>();
createIsFilterableSelectOps(replicateOp, inputSearchVar, inputSearchVarType, optFuncExpr, chosenIndex, context, isFilterableSelectOpRef, isNotFilterableSelectOpRef);
List<LogicalVariable> originalLiveVars = new ArrayList<LogicalVariable>();
VariableUtilities.getLiveVariables(indexSubTree.getRoot(), originalLiveVars);
// Copy the scan subtree in indexSubTree.
LogicalOperatorDeepCopyWithNewVariablesVisitor deepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor(context, context);
ILogicalOperator scanSubTree = deepCopyVisitor.deepCopy(indexSubTree.getRoot());
Map<LogicalVariable, LogicalVariable> copyVarMap = deepCopyVisitor.getInputToOutputVariableMapping();
panicVarMap.putAll(copyVarMap);
List<LogicalVariable> copyLiveVars = new ArrayList<LogicalVariable>();
VariableUtilities.getLiveVariables(scanSubTree, copyLiveVars);
// Replace the inputs of the given join op, and replace variables in its
// condition since we deep-copied one of the scanner subtrees which
// changed variables.
AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue();
for (Map.Entry<LogicalVariable, LogicalVariable> entry : copyVarMap.entrySet()) {
joinOp.getCondition().getValue().substituteVar(entry.getKey(), entry.getValue());
}
joinOp.getInputs().clear();
joinOp.getInputs().add(new MutableObject<ILogicalOperator>(scanSubTree));
// Make sure that the build input (which may be materialized causing blocking) comes from
// the split+select, otherwise the plan will have a deadlock.
joinOp.getInputs().add(isNotFilterableSelectOpRef);
context.computeAndSetTypeEnvironmentForOperator(joinOp);
// Return the new root of the probeSubTree.
return isFilterableSelectOpRef;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.LogicalOperatorDeepCopyWithNewVariablesVisitor in project asterixdb by apache.
the class InlineAllNtsInSubplanVisitor method visitNestedTupleSourceOperator.
@Override
public ILogicalOperator visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) throws AlgebricksException {
if (op.getDataSourceReference().getValue() != subplanOperator) {
return op;
}
LogicalOperatorDeepCopyWithNewVariablesVisitor deepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor(context, context);
ILogicalOperator copiedInputOperator = deepCopyVisitor.deepCopy(subplanInputOperator);
// Updates the primary key info in the copied plan segment.
Map<LogicalVariable, LogicalVariable> varMap = deepCopyVisitor.getInputToOutputVariableMapping();
addPrimaryKeys(varMap);
Pair<ILogicalOperator, Set<LogicalVariable>> primaryOpAndVars = EquivalenceClassUtils.findOrCreatePrimaryKeyOpAndVariables(copiedInputOperator, true, context);
correlatedKeyVars.clear();
correlatedKeyVars.addAll(primaryOpAndVars.second);
// Update key variables and input-output-var mapping.
for (Map.Entry<LogicalVariable, LogicalVariable> entry : varMap.entrySet()) {
LogicalVariable oldVar = entry.getKey();
LogicalVariable newVar = entry.getValue();
if (correlatedKeyVars.contains(oldVar)) {
correlatedKeyVars.remove(oldVar);
correlatedKeyVars.add(newVar);
}
updateInputToOutputVarMapping(oldVar, newVar, true);
}
return primaryOpAndVars.first;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.LogicalOperatorDeepCopyWithNewVariablesVisitor in project asterixdb by apache.
the class LangExpressionToPlanTranslator method eliminateSharedOperatorReference.
/**
* Eliminate shared operator references in a query plan rooted at <code>currentOpRef.getValue()</code>.
* Deep copy a new query plan subtree whenever there is a shared operator reference.
*
* @param currentOpRef,
* the operator reference to consider
* @param opRefSet,
* the set storing seen operator references so far.
* @return a mapping that maps old variables to new variables, for the ancestors of
* <code>currentOpRef</code> to replace variables properly.
* @throws CompilationException
*/
private LinkedHashMap<LogicalVariable, LogicalVariable> eliminateSharedOperatorReference(Mutable<ILogicalOperator> currentOpRef, Set<Mutable<ILogicalOperator>> opRefSet) throws CompilationException {
try {
opRefSet.add(currentOpRef);
AbstractLogicalOperator currentOperator = (AbstractLogicalOperator) currentOpRef.getValue();
// Recursively eliminates shared references in nested plans.
if (currentOperator.hasNestedPlans()) {
// Since a nested plan tree itself can never be shared with another nested plan tree in
// another operator, the operation called in the if block does not need to replace
// any variables further for <code>currentOpRef.getValue()</code> nor its ancestor.
AbstractOperatorWithNestedPlans opWithNestedPlan = (AbstractOperatorWithNestedPlans) currentOperator;
for (ILogicalPlan plan : opWithNestedPlan.getNestedPlans()) {
for (Mutable<ILogicalOperator> rootRef : plan.getRoots()) {
Set<Mutable<ILogicalOperator>> nestedOpRefSet = new HashSet<>();
eliminateSharedOperatorReference(rootRef, nestedOpRefSet);
}
}
}
int childIndex = 0;
LinkedHashMap<LogicalVariable, LogicalVariable> varMap = new LinkedHashMap<>();
for (Mutable<ILogicalOperator> childRef : currentOperator.getInputs()) {
if (opRefSet.contains(childRef)) {
// There is a shared operator reference in the query plan.
// Deep copies the child plan.
LogicalOperatorDeepCopyWithNewVariablesVisitor visitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor(context, null);
ILogicalOperator newChild = childRef.getValue().accept(visitor, null);
LinkedHashMap<LogicalVariable, LogicalVariable> cloneVarMap = visitor.getInputToOutputVariableMapping();
// Substitute variables according to the deep copy which generates new variables.
VariableUtilities.substituteVariables(currentOperator, cloneVarMap, null);
varMap.putAll(cloneVarMap);
// Sets the new child.
childRef = new MutableObject<>(newChild);
currentOperator.getInputs().set(childIndex, childRef);
}
// Recursively eliminate shared operator reference for the operator subtree,
// even if it is a deep copy of some other one.
LinkedHashMap<LogicalVariable, LogicalVariable> childVarMap = eliminateSharedOperatorReference(childRef, opRefSet);
// Substitute variables according to the new subtree.
VariableUtilities.substituteVariables(currentOperator, childVarMap, null);
// in childVarMap.
for (Map.Entry<LogicalVariable, LogicalVariable> entry : varMap.entrySet()) {
LogicalVariable newVar = childVarMap.get(entry.getValue());
if (newVar != null) {
entry.setValue(newVar);
}
}
varMap.putAll(childVarMap);
++childIndex;
}
// Only retain live variables for parent operators to substitute variables.
Set<LogicalVariable> liveVars = new HashSet<>();
VariableUtilities.getLiveVariables(currentOperator, liveVars);
varMap.values().retainAll(liveVars);
return varMap;
} catch (AlgebricksException e) {
throw new CompilationException(e);
}
}
Aggregations