Search in sources :

Example 26 with MutableObject

use of org.apache.commons.lang3.mutable.MutableObject in project asterixdb by apache.

the class AbstractDecorrelationRule method buildVarExprList.

protected void buildVarExprList(Collection<LogicalVariable> vars, IOptimizationContext context, GroupByOperator g, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException {
    for (LogicalVariable ov : vars) {
        LogicalVariable newVar = context.newVar();
        ILogicalExpression varExpr = new VariableReferenceExpression(newVar);
        outVeList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(ov, new MutableObject<ILogicalExpression>(varExpr)));
        for (ILogicalPlan p : g.getNestedPlans()) {
            for (Mutable<ILogicalOperator> r : p.getRoots()) {
                OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true, context);
            }
        }
    }
}
Also used : LogicalVariable(org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable) Mutable(org.apache.commons.lang3.mutable.Mutable) ILogicalExpression(org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression) VariableReferenceExpression(org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) ILogicalPlan(org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan) MutableObject(org.apache.commons.lang3.mutable.MutableObject)

Example 27 with MutableObject

use of org.apache.commons.lang3.mutable.MutableObject in project asterixdb by apache.

the class AbstractIntroduceCombinerRule method tryToPushAgg.

protected Pair<Boolean, Mutable<ILogicalOperator>> tryToPushAgg(AggregateOperator initAgg, GroupByOperator newGbyOp, Set<SimilarAggregatesInfo> toReplaceSet, IOptimizationContext context) throws AlgebricksException {
    ArrayList<LogicalVariable> pushedVars = new ArrayList<LogicalVariable>();
    ArrayList<Mutable<ILogicalExpression>> pushedExprs = new ArrayList<Mutable<ILogicalExpression>>();
    List<LogicalVariable> initVars = initAgg.getVariables();
    List<Mutable<ILogicalExpression>> initExprs = initAgg.getExpressions();
    int numExprs = initVars.size();
    // First make sure that all agg funcs are two step, otherwise we cannot use local aggs.
    for (int i = 0; i < numExprs; i++) {
        AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) initExprs.get(i).getValue();
        if (!aggFun.isTwoStep()) {
            return new Pair<Boolean, Mutable<ILogicalOperator>>(false, null);
        }
    }
    boolean haveAggToReplace = false;
    for (int i = 0; i < numExprs; i++) {
        Mutable<ILogicalExpression> expRef = initExprs.get(i);
        AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) expRef.getValue();
        IFunctionInfo fi1 = aggFun.getStepOneAggregate();
        // Clone the aggregate's args.
        List<Mutable<ILogicalExpression>> newArgs = new ArrayList<Mutable<ILogicalExpression>>(aggFun.getArguments().size());
        for (Mutable<ILogicalExpression> er : aggFun.getArguments()) {
            newArgs.add(new MutableObject<ILogicalExpression>(er.getValue().cloneExpression()));
        }
        IFunctionInfo fi2 = aggFun.getStepTwoAggregate();
        SimilarAggregatesInfo inf = new SimilarAggregatesInfo();
        LogicalVariable newAggVar = context.newVar();
        pushedVars.add(newAggVar);
        inf.stepOneResult = new VariableReferenceExpression(newAggVar);
        inf.simAggs = new ArrayList<AggregateExprInfo>();
        toReplaceSet.add(inf);
        AggregateFunctionCallExpression aggLocal = new AggregateFunctionCallExpression(fi1, false, newArgs);
        pushedExprs.add(new MutableObject<ILogicalExpression>(aggLocal));
        AggregateExprInfo aei = new AggregateExprInfo();
        aei.aggExprRef = expRef;
        aei.newFunInfo = fi2;
        inf.simAggs.add(aei);
        haveAggToReplace = true;
    }
    if (!pushedVars.isEmpty()) {
        AggregateOperator pushedAgg = new AggregateOperator(pushedVars, pushedExprs);
        pushedAgg.setExecutionMode(ExecutionMode.LOCAL);
        // If newGbyOp is null, then we optimizing an aggregate without group by.
        if (newGbyOp != null) {
            // Cut and paste nested input pipelines of initAgg to pushedAgg's input
            Mutable<ILogicalOperator> inputRef = initAgg.getInputs().get(0);
            Mutable<ILogicalOperator> bottomRef = inputRef;
            while (bottomRef.getValue().getInputs().size() > 0) {
                bottomRef = bottomRef.getValue().getInputs().get(0);
            }
            ILogicalOperator oldNts = bottomRef.getValue();
            initAgg.getInputs().clear();
            initAgg.getInputs().add(new MutableObject<ILogicalOperator>(oldNts));
            // Hook up the nested aggregate op with the outer group by.
            NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(newGbyOp));
            nts.setExecutionMode(ExecutionMode.LOCAL);
            bottomRef.setValue(nts);
            pushedAgg.getInputs().add(inputRef);
        } else {
            // The local aggregate operator is fed by the input of the original aggregate operator.
            pushedAgg.getInputs().add(new MutableObject<ILogicalOperator>(initAgg.getInputs().get(0).getValue()));
            // Reintroduce assign op for the global agg partitioning var.
            initAgg.getInputs().get(0).setValue(pushedAgg);
            pushedAgg.setGlobal(false);
            context.computeAndSetTypeEnvironmentForOperator(pushedAgg);
        }
        return new Pair<Boolean, Mutable<ILogicalOperator>>(true, new MutableObject<ILogicalOperator>(pushedAgg));
    } else {
        return new Pair<Boolean, Mutable<ILogicalOperator>>(haveAggToReplace, null);
    }
}
Also used : LogicalVariable(org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable) AggregateFunctionCallExpression(org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression) NestedTupleSourceOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator) IFunctionInfo(org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) ArrayList(java.util.ArrayList) Mutable(org.apache.commons.lang3.mutable.Mutable) ILogicalExpression(org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression) VariableReferenceExpression(org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression) AggregateOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator) Pair(org.apache.hyracks.algebricks.common.utils.Pair)

Example 28 with MutableObject

use of org.apache.commons.lang3.mutable.MutableObject in project asterixdb by apache.

the class ExtractFunctionsFromJoinConditionRule method assignFunctionExpressions.

private boolean assignFunctionExpressions(AbstractLogicalOperator joinOp, ILogicalExpression expr, IOptimizationContext context) throws AlgebricksException {
    if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
        return false;
    }
    AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) expr;
    FunctionIdentifier fi = fexp.getFunctionIdentifier();
    boolean modified = false;
    if (fi.equals(AlgebricksBuiltinFunctions.AND) || fi.equals(AlgebricksBuiltinFunctions.OR) || processArgumentsToFunction(fi)) {
        for (Mutable<ILogicalExpression> a : fexp.getArguments()) {
            if (assignFunctionExpressions(joinOp, a.getValue(), context)) {
                modified = true;
            }
        }
        return modified;
    } else if (AlgebricksBuiltinFunctions.isComparisonFunction(fi) || isComparisonFunction(fi)) {
        for (Mutable<ILogicalExpression> exprRef : fexp.getArguments()) {
            if (exprRef.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
                LogicalVariable newVar = context.newVar();
                AssignOperator newAssign = new AssignOperator(newVar, new MutableObject<ILogicalExpression>(exprRef.getValue().cloneExpression()));
                newAssign.setExecutionMode(joinOp.getExecutionMode());
                // Place assign below joinOp.
                List<LogicalVariable> used = new ArrayList<LogicalVariable>();
                VariableUtilities.getUsedVariables(newAssign, used);
                Mutable<ILogicalOperator> leftBranchRef = joinOp.getInputs().get(0);
                ILogicalOperator leftBranch = leftBranchRef.getValue();
                List<LogicalVariable> leftBranchVariables = new ArrayList<LogicalVariable>();
                VariableUtilities.getLiveVariables(leftBranch, leftBranchVariables);
                if (leftBranchVariables.containsAll(used)) {
                    // place assign on left branch
                    newAssign.getInputs().add(new MutableObject<ILogicalOperator>(leftBranch));
                    leftBranchRef.setValue(newAssign);
                    modified = true;
                } else {
                    Mutable<ILogicalOperator> rightBranchRef = joinOp.getInputs().get(1);
                    ILogicalOperator rightBranch = rightBranchRef.getValue();
                    List<LogicalVariable> rightBranchVariables = new ArrayList<LogicalVariable>();
                    VariableUtilities.getLiveVariables(rightBranch, rightBranchVariables);
                    if (rightBranchVariables.containsAll(used)) {
                        // place assign on right branch
                        newAssign.getInputs().add(new MutableObject<ILogicalOperator>(rightBranch));
                        rightBranchRef.setValue(newAssign);
                        modified = true;
                    }
                }
                if (modified) {
                    // Replace original expr with variable reference.
                    exprRef.setValue(new VariableReferenceExpression(newVar));
                    context.computeAndSetTypeEnvironmentForOperator(newAssign);
                    context.computeAndSetTypeEnvironmentForOperator(joinOp);
                }
            }
        }
        return modified;
    } else {
        return false;
    }
}
Also used : LogicalVariable(org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable) AbstractFunctionCallExpression(org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) AssignOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator) FunctionIdentifier(org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier) Mutable(org.apache.commons.lang3.mutable.Mutable) ILogicalExpression(org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression) VariableReferenceExpression(org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression) ArrayList(java.util.ArrayList) List(java.util.List) MutableObject(org.apache.commons.lang3.mutable.MutableObject)

Example 29 with MutableObject

use of org.apache.commons.lang3.mutable.MutableObject in project asterixdb by apache.

the class ExtractGroupByDecorVariablesRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    ILogicalOperator op = opRef.getValue();
    if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
        return false;
    }
    GroupByOperator groupByOperator = (GroupByOperator) op;
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList = groupByOperator.getDecorList();
    // Returns immediately if there is no decoration entry.
    if (groupByOperator.getDecorList() == null || groupByOperator.getDecorList().isEmpty()) {
        return false;
    }
    // Goes over the decoration list and performs the rewrite.
    boolean changed = false;
    List<LogicalVariable> vars = new ArrayList<>();
    List<Mutable<ILogicalExpression>> exprs = new ArrayList<>();
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorVarExpr : decorList) {
        Mutable<ILogicalExpression> exprRef = decorVarExpr.second;
        ILogicalExpression expr = exprRef.getValue();
        if (expr == null || expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
            continue;
        }
        // Rewrites the decoration entry if the decoration expression is not a variable reference expression.
        changed = true;
        LogicalVariable newVar = context.newVar();
        vars.add(newVar);
        exprs.add(exprRef);
        // Normalizes the decor entry -- expression be a variable reference
        decorVarExpr.second = new MutableObject<>(new VariableReferenceExpression(newVar));
    }
    if (!changed) {
        return false;
    }
    // Injects an assign operator to evaluate the decoration expression.
    AssignOperator assignOperator = new AssignOperator(vars, exprs);
    assignOperator.getInputs().addAll(op.getInputs());
    op.getInputs().set(0, new MutableObject<>(assignOperator));
    return changed;
}
Also used : LogicalVariable(org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable) GroupByOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) ArrayList(java.util.ArrayList) AssignOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator) Mutable(org.apache.commons.lang3.mutable.Mutable) ILogicalExpression(org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression) VariableReferenceExpression(org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression) Pair(org.apache.hyracks.algebricks.common.utils.Pair)

Example 30 with MutableObject

use of org.apache.commons.lang3.mutable.MutableObject in project asterixdb by apache.

the class AbstractIntroduceGroupByCombinerRule method tryToPushRoot.

private boolean tryToPushRoot(Mutable<ILogicalOperator> root, GroupByOperator oldGbyOp, GroupByOperator newGbyOp, BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context, List<Mutable<ILogicalOperator>> toPushAccumulate, Set<SimilarAggregatesInfo> toReplaceSet) throws AlgebricksException {
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) root.getValue();
    if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
        return false;
    }
    AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
    // Finds nested group-by if any.
    AbstractLogicalOperator op3 = op2;
    while (op3.getOperatorTag() != LogicalOperatorTag.GROUP && op3.getInputs().size() == 1) {
        op3 = (AbstractLogicalOperator) op3.getInputs().get(0).getValue();
    }
    if (op3.getOperatorTag() != LogicalOperatorTag.GROUP) {
        AggregateOperator initAgg = (AggregateOperator) op1;
        Pair<Boolean, Mutable<ILogicalOperator>> pOpRef = tryToPushAgg(initAgg, newGbyOp, toReplaceSet, context);
        if (!pOpRef.first) {
            return false;
        }
        Mutable<ILogicalOperator> opRef = pOpRef.second;
        if (opRef != null) {
            toPushAccumulate.add(opRef);
        }
        bi.modifyGbyMap.put(oldGbyOp, gbyVars);
        return true;
    } else {
        GroupByOperator nestedGby = (GroupByOperator) op3;
        List<LogicalVariable> gbyVars2 = nestedGby.getGbyVarList();
        Set<LogicalVariable> freeVars = new HashSet<>();
        // Removes non-free variables defined in the nested plan.
        OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(nestedGby, freeVars);
        gbyVars2.retainAll(freeVars);
        List<LogicalVariable> concatGbyVars = new ArrayList<LogicalVariable>(gbyVars);
        concatGbyVars.addAll(gbyVars2);
        for (ILogicalPlan p : nestedGby.getNestedPlans()) {
            for (Mutable<ILogicalOperator> r2 : p.getRoots()) {
                if (!tryToPushRoot(r2, nestedGby, newGbyOp, bi, concatGbyVars, context, toPushAccumulate, toReplaceSet)) {
                    return false;
                }
            }
        }
        /***
             * Push the nested pipeline which provides the input to the nested group operator into newGbyOp (the combined gby op).
             * The change is to fix asterixdb issue 782.
             */
        // Finds the reference of the bottom-most operator in the pipeline that
        // should not be pushed to the combiner group-by.
        Mutable<ILogicalOperator> currentOpRef = new MutableObject<ILogicalOperator>(nestedGby);
        Mutable<ILogicalOperator> bottomOpRef = findBottomOpRefStayInOldGby(currentOpRef);
        // Adds the used variables in the pipeline from <code>currentOpRef</code> to <code>bottomOpRef</code>
        // into the group-by keys for the introduced combiner group-by operator.
        Set<LogicalVariable> usedVars = collectUsedFreeVariables(currentOpRef, bottomOpRef);
        for (LogicalVariable usedVar : usedVars) {
            if (!concatGbyVars.contains(usedVar)) {
                concatGbyVars.add(usedVar);
            }
        }
        // Retains the nested pipeline above the identified operator in the old group-by operator.
        // Pushes the nested pipeline under the select operator into the new group-by operator.
        Mutable<ILogicalOperator> oldNtsRef = findNtsRef(currentOpRef);
        ILogicalOperator opToCombiner = bottomOpRef.getValue().getInputs().get(0).getValue();
        if (opToCombiner.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
            // No pipeline other than the aggregate operator needs to push to combiner.
            return true;
        }
        bottomOpRef.getValue().getInputs().set(0, new MutableObject<ILogicalOperator>(oldNtsRef.getValue()));
        Mutable<ILogicalOperator> newGbyNestedOpRef = findNtsRef(toPushAccumulate.get(0));
        NestedTupleSourceOperator newNts = (NestedTupleSourceOperator) newGbyNestedOpRef.getValue();
        newGbyNestedOpRef.setValue(opToCombiner);
        oldNtsRef.setValue(newNts);
        return true;
    }
}
Also used : LogicalVariable(org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable) NestedTupleSourceOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator) GroupByOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator) AbstractLogicalOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) ArrayList(java.util.ArrayList) Mutable(org.apache.commons.lang3.mutable.Mutable) AggregateOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator) ILogicalPlan(org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan) HashSet(java.util.HashSet) MutableObject(org.apache.commons.lang3.mutable.MutableObject)

Aggregations

Mutable (org.apache.commons.lang3.mutable.Mutable)119 ILogicalExpression (org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression)113 LogicalVariable (org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable)101 ILogicalOperator (org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator)91 ArrayList (java.util.ArrayList)87 VariableReferenceExpression (org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression)78 MutableObject (org.apache.commons.lang3.mutable.MutableObject)76 ScalarFunctionCallExpression (org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression)54 AssignOperator (org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator)49 Pair (org.apache.hyracks.algebricks.common.utils.Pair)47 AbstractFunctionCallExpression (org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression)46 ConstantExpression (org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression)35 AbstractLogicalOperator (org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator)29 GbyVariableExpressionPair (org.apache.asterix.lang.common.expression.GbyVariableExpressionPair)26 ILogicalPlan (org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan)25 AsterixConstantValue (org.apache.asterix.om.constants.AsterixConstantValue)23 AggregateFunctionCallExpression (org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression)23 UnnestingFunctionCallExpression (org.apache.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression)23 AlgebricksException (org.apache.hyracks.algebricks.common.exceptions.AlgebricksException)22 ILangExpression (org.apache.asterix.lang.common.base.ILangExpression)18