use of org.apache.commons.lang3.tuple.Pair in project asterixdb by apache.
the class AbstractIntroduceCombinerRule method tryToPushAgg.
protected Pair<Boolean, Mutable<ILogicalOperator>> tryToPushAgg(AggregateOperator initAgg, GroupByOperator newGbyOp, Set<SimilarAggregatesInfo> toReplaceSet, IOptimizationContext context) throws AlgebricksException {
ArrayList<LogicalVariable> pushedVars = new ArrayList<LogicalVariable>();
ArrayList<Mutable<ILogicalExpression>> pushedExprs = new ArrayList<Mutable<ILogicalExpression>>();
List<LogicalVariable> initVars = initAgg.getVariables();
List<Mutable<ILogicalExpression>> initExprs = initAgg.getExpressions();
int numExprs = initVars.size();
// First make sure that all agg funcs are two step, otherwise we cannot use local aggs.
for (int i = 0; i < numExprs; i++) {
AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) initExprs.get(i).getValue();
if (!aggFun.isTwoStep()) {
return new Pair<Boolean, Mutable<ILogicalOperator>>(false, null);
}
}
boolean haveAggToReplace = false;
for (int i = 0; i < numExprs; i++) {
Mutable<ILogicalExpression> expRef = initExprs.get(i);
AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) expRef.getValue();
IFunctionInfo fi1 = aggFun.getStepOneAggregate();
// Clone the aggregate's args.
List<Mutable<ILogicalExpression>> newArgs = new ArrayList<Mutable<ILogicalExpression>>(aggFun.getArguments().size());
for (Mutable<ILogicalExpression> er : aggFun.getArguments()) {
newArgs.add(new MutableObject<ILogicalExpression>(er.getValue().cloneExpression()));
}
IFunctionInfo fi2 = aggFun.getStepTwoAggregate();
SimilarAggregatesInfo inf = new SimilarAggregatesInfo();
LogicalVariable newAggVar = context.newVar();
pushedVars.add(newAggVar);
inf.stepOneResult = new VariableReferenceExpression(newAggVar);
inf.simAggs = new ArrayList<AggregateExprInfo>();
toReplaceSet.add(inf);
AggregateFunctionCallExpression aggLocal = new AggregateFunctionCallExpression(fi1, false, newArgs);
pushedExprs.add(new MutableObject<ILogicalExpression>(aggLocal));
AggregateExprInfo aei = new AggregateExprInfo();
aei.aggExprRef = expRef;
aei.newFunInfo = fi2;
inf.simAggs.add(aei);
haveAggToReplace = true;
}
if (!pushedVars.isEmpty()) {
AggregateOperator pushedAgg = new AggregateOperator(pushedVars, pushedExprs);
pushedAgg.setExecutionMode(ExecutionMode.LOCAL);
// If newGbyOp is null, then we optimizing an aggregate without group by.
if (newGbyOp != null) {
// Cut and paste nested input pipelines of initAgg to pushedAgg's input
Mutable<ILogicalOperator> inputRef = initAgg.getInputs().get(0);
Mutable<ILogicalOperator> bottomRef = inputRef;
while (bottomRef.getValue().getInputs().size() > 0) {
bottomRef = bottomRef.getValue().getInputs().get(0);
}
ILogicalOperator oldNts = bottomRef.getValue();
initAgg.getInputs().clear();
initAgg.getInputs().add(new MutableObject<ILogicalOperator>(oldNts));
// Hook up the nested aggregate op with the outer group by.
NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(newGbyOp));
nts.setExecutionMode(ExecutionMode.LOCAL);
bottomRef.setValue(nts);
pushedAgg.getInputs().add(inputRef);
} else {
// The local aggregate operator is fed by the input of the original aggregate operator.
pushedAgg.getInputs().add(new MutableObject<ILogicalOperator>(initAgg.getInputs().get(0).getValue()));
// Reintroduce assign op for the global agg partitioning var.
initAgg.getInputs().get(0).setValue(pushedAgg);
pushedAgg.setGlobal(false);
context.computeAndSetTypeEnvironmentForOperator(pushedAgg);
}
return new Pair<Boolean, Mutable<ILogicalOperator>>(true, new MutableObject<ILogicalOperator>(pushedAgg));
} else {
return new Pair<Boolean, Mutable<ILogicalOperator>>(haveAggToReplace, null);
}
}
use of org.apache.commons.lang3.tuple.Pair in project asterixdb by apache.
the class ExtractGbyExpressionsRule method gbyExprWasRewritten.
private boolean gbyExprWasRewritten(GroupByOperator g, IOptimizationContext context) throws AlgebricksException {
if (!gbyHasComplexExpr(g)) {
return false;
}
Mutable<ILogicalOperator> opRef2 = g.getInputs().get(0);
for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gbyPair : g.getGroupByList()) {
ILogicalExpression expr = gbyPair.second.getValue();
if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
LogicalVariable v = extractExprIntoAssignOpRef(expr, opRef2, context);
gbyPair.second.setValue(new VariableReferenceExpression(v));
}
}
return true;
}
use of org.apache.commons.lang3.tuple.Pair in project asterixdb by apache.
the class ExtractGroupByDecorVariablesRule method rewritePost.
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
ILogicalOperator op = opRef.getValue();
if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
return false;
}
GroupByOperator groupByOperator = (GroupByOperator) op;
List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList = groupByOperator.getDecorList();
// Returns immediately if there is no decoration entry.
if (groupByOperator.getDecorList() == null || groupByOperator.getDecorList().isEmpty()) {
return false;
}
// Goes over the decoration list and performs the rewrite.
boolean changed = false;
List<LogicalVariable> vars = new ArrayList<>();
List<Mutable<ILogicalExpression>> exprs = new ArrayList<>();
for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorVarExpr : decorList) {
Mutable<ILogicalExpression> exprRef = decorVarExpr.second;
ILogicalExpression expr = exprRef.getValue();
if (expr == null || expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
continue;
}
// Rewrites the decoration entry if the decoration expression is not a variable reference expression.
changed = true;
LogicalVariable newVar = context.newVar();
vars.add(newVar);
exprs.add(exprRef);
// Normalizes the decor entry -- expression be a variable reference
decorVarExpr.second = new MutableObject<>(new VariableReferenceExpression(newVar));
}
if (!changed) {
return false;
}
// Injects an assign operator to evaluate the decoration expression.
AssignOperator assignOperator = new AssignOperator(vars, exprs);
assignOperator.getInputs().addAll(op.getInputs());
op.getInputs().set(0, new MutableObject<>(assignOperator));
return changed;
}
use of org.apache.commons.lang3.tuple.Pair in project asterixdb by apache.
the class IntroduceAggregateCombinerRule method rewritePost.
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
if (context.checkIfInDontApplySet(this, op)) {
return false;
}
context.addToDontApplySet(this, op);
if (op.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
return false;
}
AggregateOperator aggOp = (AggregateOperator) op;
if (!aggOp.isGlobal() || aggOp.getExecutionMode() == ExecutionMode.LOCAL) {
return false;
}
Set<SimilarAggregatesInfo> toReplaceSet = new HashSet<SimilarAggregatesInfo>();
Pair<Boolean, Mutable<ILogicalOperator>> result = tryToPushAgg(aggOp, null, toReplaceSet, context);
if (!result.first || result.second == null) {
return false;
}
replaceOriginalAggFuncs(toReplaceSet);
context.computeAndSetTypeEnvironmentForOperator(aggOp);
return true;
}
use of org.apache.commons.lang3.tuple.Pair in project asterixdb by apache.
the class AbstractIntroduceGroupByCombinerRule method tryToPushRoot.
private boolean tryToPushRoot(Mutable<ILogicalOperator> root, GroupByOperator oldGbyOp, GroupByOperator newGbyOp, BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context, List<Mutable<ILogicalOperator>> toPushAccumulate, Set<SimilarAggregatesInfo> toReplaceSet) throws AlgebricksException {
AbstractLogicalOperator op1 = (AbstractLogicalOperator) root.getValue();
if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
return false;
}
AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
// Finds nested group-by if any.
AbstractLogicalOperator op3 = op2;
while (op3.getOperatorTag() != LogicalOperatorTag.GROUP && op3.getInputs().size() == 1) {
op3 = (AbstractLogicalOperator) op3.getInputs().get(0).getValue();
}
if (op3.getOperatorTag() != LogicalOperatorTag.GROUP) {
AggregateOperator initAgg = (AggregateOperator) op1;
Pair<Boolean, Mutable<ILogicalOperator>> pOpRef = tryToPushAgg(initAgg, newGbyOp, toReplaceSet, context);
if (!pOpRef.first) {
return false;
}
Mutable<ILogicalOperator> opRef = pOpRef.second;
if (opRef != null) {
toPushAccumulate.add(opRef);
}
bi.modifyGbyMap.put(oldGbyOp, gbyVars);
return true;
} else {
GroupByOperator nestedGby = (GroupByOperator) op3;
List<LogicalVariable> gbyVars2 = nestedGby.getGbyVarList();
Set<LogicalVariable> freeVars = new HashSet<>();
// Removes non-free variables defined in the nested plan.
OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(nestedGby, freeVars);
gbyVars2.retainAll(freeVars);
List<LogicalVariable> concatGbyVars = new ArrayList<LogicalVariable>(gbyVars);
concatGbyVars.addAll(gbyVars2);
for (ILogicalPlan p : nestedGby.getNestedPlans()) {
for (Mutable<ILogicalOperator> r2 : p.getRoots()) {
if (!tryToPushRoot(r2, nestedGby, newGbyOp, bi, concatGbyVars, context, toPushAccumulate, toReplaceSet)) {
return false;
}
}
}
/***
* Push the nested pipeline which provides the input to the nested group operator into newGbyOp (the combined gby op).
* The change is to fix asterixdb issue 782.
*/
// Finds the reference of the bottom-most operator in the pipeline that
// should not be pushed to the combiner group-by.
Mutable<ILogicalOperator> currentOpRef = new MutableObject<ILogicalOperator>(nestedGby);
Mutable<ILogicalOperator> bottomOpRef = findBottomOpRefStayInOldGby(currentOpRef);
// Adds the used variables in the pipeline from <code>currentOpRef</code> to <code>bottomOpRef</code>
// into the group-by keys for the introduced combiner group-by operator.
Set<LogicalVariable> usedVars = collectUsedFreeVariables(currentOpRef, bottomOpRef);
for (LogicalVariable usedVar : usedVars) {
if (!concatGbyVars.contains(usedVar)) {
concatGbyVars.add(usedVar);
}
}
// Retains the nested pipeline above the identified operator in the old group-by operator.
// Pushes the nested pipeline under the select operator into the new group-by operator.
Mutable<ILogicalOperator> oldNtsRef = findNtsRef(currentOpRef);
ILogicalOperator opToCombiner = bottomOpRef.getValue().getInputs().get(0).getValue();
if (opToCombiner.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
// No pipeline other than the aggregate operator needs to push to combiner.
return true;
}
bottomOpRef.getValue().getInputs().set(0, new MutableObject<ILogicalOperator>(oldNtsRef.getValue()));
Mutable<ILogicalOperator> newGbyNestedOpRef = findNtsRef(toPushAccumulate.get(0));
NestedTupleSourceOperator newNts = (NestedTupleSourceOperator) newGbyNestedOpRef.getValue();
newGbyNestedOpRef.setValue(opToCombiner);
oldNtsRef.setValue(newNts);
return true;
}
}
Aggregations