use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator in project asterixdb by apache.
the class SubplanOutOfGroupRule method rewritePost.
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
if (op0.getOperatorTag() != LogicalOperatorTag.GROUP) {
return false;
}
GroupByOperator gby = (GroupByOperator) op0;
Iterator<ILogicalPlan> plansIter = gby.getNestedPlans().iterator();
ILogicalPlan p = null;
while (plansIter.hasNext()) {
p = plansIter.next();
}
if (p == null) {
return false;
}
if (p.getRoots().size() != 1) {
return false;
}
Mutable<ILogicalOperator> op1Ref = p.getRoots().get(0);
AbstractLogicalOperator op1 = (AbstractLogicalOperator) op1Ref.getValue();
boolean found = false;
while (op1.getInputs().size() == 1) {
if (op1.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
SubplanOperator subplan = (SubplanOperator) op1;
AbstractLogicalOperator op2 = (AbstractLogicalOperator) subplan.getInputs().get(0).getValue();
if (OperatorPropertiesUtil.isMissingTest(op2)) {
if (subplan.getNestedPlans().size() == 1) {
ILogicalPlan p1 = subplan.getNestedPlans().get(0);
if (p1.getRoots().size() == 1) {
AbstractLogicalOperator r1 = (AbstractLogicalOperator) p1.getRoots().get(0).getValue();
if (r1.getOperatorTag() == LogicalOperatorTag.INNERJOIN || r1.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
// now, check that it propagates all variables,
// so it can be pushed
List<LogicalVariable> op2Vars = new ArrayList<LogicalVariable>();
VariableUtilities.getLiveVariables(op2, op2Vars);
List<LogicalVariable> op1Vars = new ArrayList<LogicalVariable>();
VariableUtilities.getLiveVariables(subplan, op1Vars);
if (op1Vars.containsAll(op2Vars)) {
found = true;
break;
}
}
}
}
}
}
op1Ref = op1.getInputs().get(0);
op1 = (AbstractLogicalOperator) op1Ref.getValue();
}
if (!found) {
return false;
}
ILogicalOperator subplan = op1;
ILogicalOperator op2 = op1.getInputs().get(0).getValue();
op1Ref.setValue(op2);
Mutable<ILogicalOperator> opUnderRef = gby.getInputs().get(0);
ILogicalOperator opUnder = opUnderRef.getValue();
subplan.getInputs().clear();
subplan.getInputs().add(new MutableObject<ILogicalOperator>(opUnder));
opUnderRef.setValue(subplan);
return true;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator in project asterixdb by apache.
the class ListifyUnnestingFunctionRule method listifyUnnestingFunction.
// Performs the actual logical transformation.
private boolean listifyUnnestingFunction(ILogicalOperator op, Mutable<ILogicalExpression> exprRef, AbstractFunctionCallExpression func, IOptimizationContext context) throws AlgebricksException {
IFunctionInfo functionInfo = func.getFunctionInfo();
// Checks if the function is an unnesting function.
if (!BuiltinFunctions.isBuiltinUnnestingFunction(functionInfo.getFunctionIdentifier())) {
return false;
}
// Generates the listified collection in a subplan.
SubplanOperator subplanOperator = new SubplanOperator();
// Creates a nested tuple source operator.
NestedTupleSourceOperator ntsOperator = new NestedTupleSourceOperator(new MutableObject<>(subplanOperator));
// Unnests the dataset.
LogicalVariable unnestVar = context.newVar();
ILogicalExpression unnestExpr = new UnnestingFunctionCallExpression(functionInfo, func.getArguments());
UnnestOperator unnestOperator = new UnnestOperator(unnestVar, new MutableObject<>(unnestExpr));
unnestOperator.getInputs().add(new MutableObject<>(ntsOperator));
// Listify the dataset into one collection.
LogicalVariable aggVar = context.newVar();
Mutable<ILogicalExpression> aggArgExprRef = new MutableObject<>(new VariableReferenceExpression(unnestVar));
ILogicalExpression aggExpr = new AggregateFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.LISTIFY), false, new ArrayList<>(Collections.singletonList(aggArgExprRef)));
AggregateOperator aggregateOperator = new AggregateOperator(new ArrayList<>(Collections.singletonList(aggVar)), new ArrayList<>(Collections.singletonList(new MutableObject<>(aggExpr))));
aggregateOperator.getInputs().add(new MutableObject<>(unnestOperator));
// Adds the aggregate operator as the root of the subplan.
subplanOperator.setRootOp(new MutableObject<>(aggregateOperator));
// Sticks a subplan operator into the query plan.
// Note: given the way we compile JOINs, the unnesting function expression cannot appear in
// any binary operators.
// Example test queries:
// asterixdb/asterix-app/src/test/resources/runtimets/results/list/query-ASTERIXDB-159-2
// asterixdb/asterix-app/src/test/resources/runtimets/results/list/query-ASTERIXDB-159-3
subplanOperator.getInputs().add(op.getInputs().get(0));
op.getInputs().set(0, new MutableObject<>(subplanOperator));
exprRef.setValue(new VariableReferenceExpression(aggVar));
// Computes type environments for new operators.
context.computeAndSetTypeEnvironmentForOperator(ntsOperator);
context.computeAndSetTypeEnvironmentForOperator(unnestOperator);
context.computeAndSetTypeEnvironmentForOperator(aggregateOperator);
context.computeAndSetTypeEnvironmentForOperator(subplanOperator);
return true;
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator in project asterixdb by apache.
the class InlineSubplanInputForNestedTupleSourceRule method applySpecialFlattening.
private Pair<Boolean, LinkedHashMap<LogicalVariable, LogicalVariable>> applySpecialFlattening(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
SubplanOperator subplanOp = (SubplanOperator) opRef.getValue();
Mutable<ILogicalOperator> inputOpRef = subplanOp.getInputs().get(0);
LinkedHashMap<LogicalVariable, LogicalVariable> replacedVarMap = new LinkedHashMap<>();
// Recursively applies this rule to the nested plan of the subplan operator,
// for the case where there are nested subplan operators within {@code subplanOp}.
Pair<Boolean, LinkedHashMap<LogicalVariable, LogicalVariable>> result = rewriteSubplanOperator(subplanOp.getNestedPlans().get(0).getRoots().get(0), context);
ILogicalOperator inputOpBackup = inputOpRef.getValue();
// Gets live variables and covering variables from the subplan's input operator.
Pair<ILogicalOperator, Set<LogicalVariable>> primaryOpAndVars = EquivalenceClassUtils.findOrCreatePrimaryKeyOpAndVariables(inputOpBackup, false, context);
ILogicalOperator inputOp = primaryOpAndVars.first;
Set<LogicalVariable> primaryKeyVars = primaryOpAndVars.second;
inputOpRef.setValue(inputOp);
Set<LogicalVariable> liveVars = new HashSet<>();
VariableUtilities.getLiveVariables(inputOp, liveVars);
Pair<Set<LogicalVariable>, Mutable<ILogicalOperator>> notNullVarsAndTopJoinRef = SubplanFlatteningUtil.inlineLeftNtsInSubplanJoin(subplanOp, context);
if (notNullVarsAndTopJoinRef.first == null) {
inputOpRef.setValue(inputOpBackup);
return new Pair<>(false, replacedVarMap);
}
Set<LogicalVariable> notNullVars = notNullVarsAndTopJoinRef.first;
Mutable<ILogicalOperator> topJoinRef = notNullVarsAndTopJoinRef.second;
// Creates a group-by operator.
List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByDecorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
GroupByOperator groupbyOp = new GroupByOperator(groupByList, groupByDecorList, subplanOp.getNestedPlans());
for (LogicalVariable coverVar : primaryKeyVars) {
LogicalVariable newVar = context.newVar();
groupByList.add(new Pair<>(newVar, new MutableObject<>(new VariableReferenceExpression(coverVar))));
// Adds variables for replacements in ancestors.
replacedVarMap.put(coverVar, newVar);
}
for (LogicalVariable liveVar : liveVars) {
if (primaryKeyVars.contains(liveVar)) {
continue;
}
groupByDecorList.add(new Pair<>(null, new MutableObject<>(new VariableReferenceExpression(liveVar))));
}
groupbyOp.getInputs().add(new MutableObject<>(topJoinRef.getValue()));
if (!notNullVars.isEmpty()) {
// Adds a select operator into the nested plan for group-by to remove tuples with NULL on {@code assignVar}, i.e.,
// subplan input tuples that are filtered out within a subplan.
List<Mutable<ILogicalExpression>> nullCheckExprRefs = new ArrayList<>();
for (LogicalVariable notNullVar : notNullVars) {
Mutable<ILogicalExpression> filterVarExpr = new MutableObject<>(new VariableReferenceExpression(notNullVar));
List<Mutable<ILogicalExpression>> args = new ArrayList<>();
args.add(filterVarExpr);
List<Mutable<ILogicalExpression>> argsForNotFunction = new ArrayList<>();
argsForNotFunction.add(new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.IS_MISSING), args)));
nullCheckExprRefs.add(new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.NOT), argsForNotFunction)));
}
Mutable<ILogicalExpression> selectExprRef = nullCheckExprRefs.size() > 1 ? new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.AND), nullCheckExprRefs)) : nullCheckExprRefs.get(0);
SelectOperator selectOp = new SelectOperator(selectExprRef, false, null);
topJoinRef.setValue(selectOp);
selectOp.getInputs().add(new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(groupbyOp))));
} else {
// The original join operator in the Subplan is a left-outer join.
// Therefore, no null-check variable is injected and no SelectOperator needs to be added.
topJoinRef.setValue(new NestedTupleSourceOperator(new MutableObject<>(groupbyOp)));
}
opRef.setValue(groupbyOp);
OperatorManipulationUtil.computeTypeEnvironmentBottomUp(groupbyOp, context);
VariableUtilities.substituteVariables(groupbyOp, result.second, context);
replacedVarMap.putAll(result.second);
return new Pair<>(true, replacedVarMap);
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator in project asterixdb by apache.
the class InlineSubplanInputForNestedTupleSourceRule method applyGeneralFlattening.
private Pair<Boolean, LinkedHashMap<LogicalVariable, LogicalVariable>> applyGeneralFlattening(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
SubplanOperator subplanOp = (SubplanOperator) opRef.getValue();
if (!SubplanFlatteningUtil.containsOperators(subplanOp, ImmutableSet.of(LogicalOperatorTag.DATASOURCESCAN, LogicalOperatorTag.INNERJOIN, // We don't have nested runtime for union-all and distinct hence we have to include them here.
LogicalOperatorTag.LEFTOUTERJOIN, LogicalOperatorTag.UNIONALL, LogicalOperatorTag.DISTINCT))) {
return new Pair<>(false, new LinkedHashMap<>());
}
Mutable<ILogicalOperator> inputOpRef = subplanOp.getInputs().get(0);
ILogicalOperator inputOpBackup = inputOpRef.getValue();
// Creates parameters for the left outer join operator.
Pair<ILogicalOperator, Set<LogicalVariable>> primaryOpAndVars = EquivalenceClassUtils.findOrCreatePrimaryKeyOpAndVariables(inputOpBackup, true, context);
ILogicalOperator inputOp = primaryOpAndVars.first;
Set<LogicalVariable> primaryKeyVars = primaryOpAndVars.second;
inputOpRef.setValue(inputOp);
Set<LogicalVariable> inputLiveVars = new HashSet<>();
VariableUtilities.getLiveVariables(inputOp, inputLiveVars);
Pair<Map<LogicalVariable, LogicalVariable>, List<Pair<IOrder, Mutable<ILogicalExpression>>>> varMapAndOrderExprs = SubplanFlatteningUtil.inlineAllNestedTupleSource(subplanOp, context);
Map<LogicalVariable, LogicalVariable> varMap = varMapAndOrderExprs.first;
if (varMap == null) {
inputOpRef.setValue(inputOpBackup);
return new Pair<>(false, new LinkedHashMap<>());
}
Mutable<ILogicalOperator> lowestAggregateRefInSubplan = SubplanFlatteningUtil.findLowestAggregate(subplanOp.getNestedPlans().get(0).getRoots().get(0));
Mutable<ILogicalOperator> rightInputOpRef = lowestAggregateRefInSubplan.getValue().getInputs().get(0);
ILogicalOperator rightInputOp = rightInputOpRef.getValue();
// Creates a variable to indicate whether a left input tuple is killed in the plan rooted at rightInputOp.
LogicalVariable assignVar = context.newVar();
ILogicalOperator assignOp = new AssignOperator(assignVar, new MutableObject<>(ConstantExpression.TRUE));
assignOp.getInputs().add(rightInputOpRef);
context.computeAndSetTypeEnvironmentForOperator(assignOp);
rightInputOpRef = new MutableObject<>(assignOp);
// Constructs the join predicate for the leftOuter join.
List<Mutable<ILogicalExpression>> joinPredicates = new ArrayList<>();
for (LogicalVariable liveVar : primaryKeyVars) {
List<Mutable<ILogicalExpression>> arguments = new ArrayList<>();
arguments.add(new MutableObject<>(new VariableReferenceExpression(liveVar)));
LogicalVariable rightVar = varMap.get(liveVar);
arguments.add(new MutableObject<>(new VariableReferenceExpression(rightVar)));
ILogicalExpression expr = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AlgebricksBuiltinFunctions.EQ), arguments);
joinPredicates.add(new MutableObject<>(expr));
}
ILogicalExpression joinExpr = joinPredicates.size() > 1 ? new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AlgebricksBuiltinFunctions.AND), joinPredicates) : joinPredicates.size() > 0 ? joinPredicates.get(0).getValue() : ConstantExpression.TRUE;
LeftOuterJoinOperator leftOuterJoinOp = new LeftOuterJoinOperator(new MutableObject<>(joinExpr), inputOpRef, rightInputOpRef);
OperatorManipulationUtil.computeTypeEnvironmentBottomUp(rightInputOp, context);
context.computeAndSetTypeEnvironmentForOperator(leftOuterJoinOp);
// Creates group-by operator.
List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByDecorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
List<ILogicalPlan> nestedPlans = new ArrayList<>();
GroupByOperator groupbyOp = new GroupByOperator(groupByList, groupByDecorList, nestedPlans);
LinkedHashMap<LogicalVariable, LogicalVariable> replacedVarMap = new LinkedHashMap<>();
for (LogicalVariable liveVar : primaryKeyVars) {
LogicalVariable newVar = context.newVar();
groupByList.add(new Pair<>(newVar, new MutableObject<>(new VariableReferenceExpression(liveVar))));
// Adds variables for replacements in ancestors.
replacedVarMap.put(liveVar, newVar);
}
for (LogicalVariable liveVar : inputLiveVars) {
if (primaryKeyVars.contains(liveVar)) {
continue;
}
groupByDecorList.add(new Pair<>(null, new MutableObject<>(new VariableReferenceExpression(liveVar))));
}
// Sets up the nested plan for the groupby operator.
Mutable<ILogicalOperator> aggOpRef = subplanOp.getNestedPlans().get(0).getRoots().get(0);
// Clears the input of the lowest aggregate.
lowestAggregateRefInSubplan.getValue().getInputs().clear();
Mutable<ILogicalOperator> currentOpRef = lowestAggregateRefInSubplan;
// Adds an optional order operator.
List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExprs = varMapAndOrderExprs.second;
if (!orderExprs.isEmpty()) {
OrderOperator orderOp = new OrderOperator(orderExprs);
currentOpRef = new MutableObject<>(orderOp);
lowestAggregateRefInSubplan.getValue().getInputs().add(currentOpRef);
}
// Adds a select operator into the nested plan for group-by to remove tuples with NULL on {@code assignVar}, i.e.,
// subplan input tuples that are filtered out within a subplan.
Mutable<ILogicalExpression> filterVarExpr = new MutableObject<>(new VariableReferenceExpression(assignVar));
List<Mutable<ILogicalExpression>> args = new ArrayList<>();
args.add(filterVarExpr);
List<Mutable<ILogicalExpression>> argsForNotFunction = new ArrayList<>();
argsForNotFunction.add(new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.IS_MISSING), args)));
SelectOperator selectOp = new SelectOperator(new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.NOT), argsForNotFunction)), false, null);
currentOpRef.getValue().getInputs().add(new MutableObject<>(selectOp));
selectOp.getInputs().add(new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(groupbyOp))));
List<Mutable<ILogicalOperator>> nestedRoots = new ArrayList<>();
nestedRoots.add(aggOpRef);
nestedPlans.add(new ALogicalPlanImpl(nestedRoots));
groupbyOp.getInputs().add(new MutableObject<>(leftOuterJoinOp));
// Replaces subplan with the group-by operator.
opRef.setValue(groupbyOp);
OperatorManipulationUtil.computeTypeEnvironmentBottomUp(groupbyOp, context);
// Recursively applys this rule to the nested plan of the subplan operator,
// for the case where there are nested subplan operators within {@code subplanOp}.
Pair<Boolean, LinkedHashMap<LogicalVariable, LogicalVariable>> result = rewriteSubplanOperator(rightInputOpRef, context);
VariableUtilities.substituteVariables(leftOuterJoinOp, result.second, context);
VariableUtilities.substituteVariables(groupbyOp, result.second, context);
// No var mapping from the right input operator should be populated up.
return new Pair<>(true, replacedVarMap);
}
use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator in project asterixdb by apache.
the class LangExpressionToPlanTranslator method langExprToAlgExpression.
protected Pair<ILogicalExpression, Mutable<ILogicalOperator>> langExprToAlgExpression(Expression expr, Mutable<ILogicalOperator> topOpRef) throws CompilationException {
switch(expr.getKind()) {
case VARIABLE_EXPRESSION:
VariableReferenceExpression ve = new VariableReferenceExpression(context.getVar(((VariableExpr) expr).getVar().getId()));
return new Pair<>(ve, topOpRef);
case LITERAL_EXPRESSION:
LiteralExpr val = (LiteralExpr) expr;
return new Pair<>(new ConstantExpression(new AsterixConstantValue(ConstantHelper.objectFromLiteral(val.getValue()))), topOpRef);
default:
if (expressionNeedsNoNesting(expr)) {
Pair<ILogicalOperator, LogicalVariable> p = expr.accept(this, topOpRef);
ILogicalExpression exp = ((AssignOperator) p.first).getExpressions().get(0).getValue();
return new Pair<>(exp, p.first.getInputs().get(0));
} else {
Mutable<ILogicalOperator> srcRef = new MutableObject<>();
Pair<ILogicalOperator, LogicalVariable> p = expr.accept(this, srcRef);
if (p.first.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
if (topOpRef.getValue() != null) {
srcRef.setValue(topOpRef.getValue());
} else {
// Re-binds the bottom operator reference to {@code topOpRef}.
rebindBottomOpRef(p.first, srcRef, topOpRef);
}
Mutable<ILogicalOperator> top2 = new MutableObject<>(p.first);
return new Pair<>(new VariableReferenceExpression(p.second), top2);
} else {
SubplanOperator s = new SubplanOperator();
s.getInputs().add(topOpRef);
srcRef.setValue(new NestedTupleSourceOperator(new MutableObject<>(s)));
Mutable<ILogicalOperator> planRoot = new MutableObject<>(p.first);
s.setRootOp(planRoot);
return new Pair<>(new VariableReferenceExpression(p.second), new MutableObject<>(s));
}
}
}
}
Aggregations