Examples with AggregateOperator org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator used on opensource projects

Example 1 with AggregateOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator in project asterixdb by apache.

the class CountVarToCountOneRule method rewritePost.

// It is only for a group-by having just one aggregate which is a count.
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
    if (op1.getOperatorTag() != LogicalOperatorTag.GROUP) {
        return false;
    }
    GroupByOperator g = (GroupByOperator) op1;
    if (g.getNestedPlans().size() != 1) {
        return false;
    }
    ILogicalPlan p = g.getNestedPlans().get(0);
    if (p.getRoots().size() != 1) {
        return false;
    }
    AbstractLogicalOperator op2 = (AbstractLogicalOperator) p.getRoots().get(0).getValue();
    if (op2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
        return false;
    }
    AggregateOperator agg = (AggregateOperator) op2;
    if (agg.getExpressions().size() != 1) {
        return false;
    }
    ILogicalExpression exp2 = agg.getExpressions().get(0).getValue();
    if (exp2.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
        return false;
    }
    AbstractFunctionCallExpression fun = (AbstractFunctionCallExpression) exp2;
    if (fun.getFunctionIdentifier() != BuiltinFunctions.COUNT) {
        return false;
    }
    ILogicalExpression exp3 = fun.getArguments().get(0).getValue();
    if (exp3.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
        return false;
    }
    if (((AbstractLogicalOperator) agg.getInputs().get(0).getValue()).getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
        return false;
    }
    fun.getArguments().get(0).setValue(new ConstantExpression(new AsterixConstantValue(new AInt64(1L))));
    return true;
}

Example 2 with AggregateOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator in project asterixdb by apache.

the class PushAggregateIntoNestedSubplanRule method removeRedundantListifies.

private void removeRedundantListifies(Map<LogicalVariable, Integer> nspAggVars, Map<LogicalVariable, AbstractOperatorWithNestedPlans> nspWithAgg, Map<LogicalVariable, Integer> nspAggVarToPlanIndex) throws AlgebricksException {
    List<Pair<AbstractOperatorWithNestedPlans, Integer>> removeList = new ArrayList<>();
    for (Map.Entry<LogicalVariable, Integer> aggVarEntry : nspAggVars.entrySet()) {
        LogicalVariable aggVar = aggVarEntry.getKey();
        int occurs = aggVarEntry.getValue();
        if (occurs == 0) {
            AbstractOperatorWithNestedPlans nspOp = nspWithAgg.get(aggVar);
            AggregateOperator aggOp = (AggregateOperator) nspOp.getNestedPlans().get(nspAggVarToPlanIndex.get(aggVar)).getRoots().get(0).getValue();
            int pos = aggOp.getVariables().indexOf(aggVar);
            if (pos >= 0) {
                aggOp.getVariables().remove(pos);
                aggOp.getExpressions().remove(pos);
                List<LogicalVariable> producedVarsAtAgg = new ArrayList<>();
                VariableUtilities.getProducedVariablesInDescendantsAndSelf(aggOp, producedVarsAtAgg);
                if (producedVarsAtAgg.isEmpty()) {
                    removeList.add(new Pair<>(nspOp, nspAggVarToPlanIndex.get(aggVar)));
                }
            }
        }
    }
    // Collects subplans that is to be removed.
    Map<AbstractOperatorWithNestedPlans, List<ILogicalPlan>> nspToSubplanListMap = new HashMap<>();
    for (Pair<AbstractOperatorWithNestedPlans, Integer> remove : removeList) {
        AbstractOperatorWithNestedPlans groupByOperator = remove.first;
        ILogicalPlan subplan = remove.first.getNestedPlans().get(remove.second);
        if (nspToSubplanListMap.containsKey(groupByOperator)) {
            List<ILogicalPlan> subplans = nspToSubplanListMap.get(groupByOperator);
            subplans.add(subplan);
        } else {
            List<ILogicalPlan> subplans = new ArrayList<>();
            subplans.add(subplan);
            nspToSubplanListMap.put(groupByOperator, subplans);
        }
    }
    // Removes subplans.
    for (Map.Entry<AbstractOperatorWithNestedPlans, List<ILogicalPlan>> entry : nspToSubplanListMap.entrySet()) {
        entry.getKey().getNestedPlans().removeAll(entry.getValue());
    }
}

Example 3 with AggregateOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator in project asterixdb by apache.

the class PushAggregateIntoNestedSubplanRule method rewriteAggregateInNestedSubplan.

private void rewriteAggregateInNestedSubplan(LogicalVariable oldAggVar, AbstractOperatorWithNestedPlans nspOp, AggregateFunctionCallExpression aggFun, LogicalVariable newAggVar, IOptimizationContext context) throws AlgebricksException {
    for (int j = 0; j < nspOp.getNestedPlans().size(); j++) {
        AggregateOperator aggOp = (AggregateOperator) nspOp.getNestedPlans().get(j).getRoots().get(0).getValue();
        int n = aggOp.getVariables().size();
        for (int i = 0; i < n; i++) {
            LogicalVariable v = aggOp.getVariables().get(i);
            if (v.equals(oldAggVar)) {
                AbstractFunctionCallExpression oldAggExpr = (AbstractFunctionCallExpression) aggOp.getExpressions().get(i).getValue();
                AggregateFunctionCallExpression newAggFun = BuiltinFunctions.makeAggregateFunctionExpression(aggFun.getFunctionIdentifier(), new ArrayList<>());
                for (Mutable<ILogicalExpression> arg : oldAggExpr.getArguments()) {
                    ILogicalExpression cloned = arg.getValue().cloneExpression();
                    newAggFun.getArguments().add(new MutableObject<>(cloned));
                }
                aggOp.getVariables().add(newAggVar);
                aggOp.getExpressions().add(new MutableObject<>(newAggFun));
                context.computeAndSetTypeEnvironmentForOperator(aggOp);
                break;
            }
        }
    }
}

Example 4 with AggregateOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator in project asterixdb by apache.

the class PushAggregateIntoNestedSubplanRule method pushSubplanAsAggIntoNestedSubplan.

private boolean pushSubplanAsAggIntoNestedSubplan(Mutable<ILogicalOperator> subplanOpRef, AbstractOperatorWithNestedPlans nspOp, LogicalVariable varFromNestedAgg, Map<LogicalVariable, Integer> nspAggVars, Map<LogicalVariable, AbstractOperatorWithNestedPlans> nspWithAgg, Map<LogicalVariable, Integer> nspAggVarToPlanIndex, IOptimizationContext context) throws AlgebricksException {
    SubplanOperator subplan = (SubplanOperator) subplanOpRef.getValue();
    // only free var can be varFromNestedAgg
    HashSet<LogicalVariable> freeVars = new HashSet<>();
    OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, freeVars);
    for (LogicalVariable vFree : freeVars) {
        if (!vFree.equals(varFromNestedAgg)) {
            return false;
        }
    }
    List<ILogicalPlan> plans = subplan.getNestedPlans();
    if (plans.size() > 1) {
        return false;
    }
    ILogicalPlan p = plans.get(0);
    if (p.getRoots().size() > 1) {
        return false;
    }
    Mutable<ILogicalOperator> opRef = p.getRoots().get(0);
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    if (op.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
        return false;
    }
    AggregateOperator aggInSubplanOp = (AggregateOperator) op;
    LogicalVariable unnestVar = null;
    boolean pushableNestedSubplan = false;
    while (op.getInputs().size() == 1) {
        opRef = op.getInputs().get(0);
        op = (AbstractLogicalOperator) opRef.getValue();
        switch(op.getOperatorTag()) {
            case ASSIGN:
                break;
            case UNNEST:
                UnnestOperator unnest = (UnnestOperator) op;
                if (unnest.getPositionalVariable() != null) {
                    // TODO currently subplan with both accumulating and running aggregate is not supported.
                    return false;
                }
                ILogicalExpression expr = unnest.getExpressionRef().getValue();
                if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
                    return false;
                }
                AbstractFunctionCallExpression fun = (AbstractFunctionCallExpression) expr;
                if (fun.getFunctionIdentifier() != BuiltinFunctions.SCAN_COLLECTION) {
                    return false;
                }
                ILogicalExpression arg0 = fun.getArguments().get(0).getValue();
                if (arg0.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                    return false;
                }
                VariableReferenceExpression varExpr = (VariableReferenceExpression) arg0;
                if (!varExpr.getVariableReference().equals(varFromNestedAgg)) {
                    return false;
                }
                opRef = op.getInputs().get(0);
                op = (AbstractLogicalOperator) opRef.getValue();
                if (op.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
                    return false;
                }
                pushableNestedSubplan = true;
                unnestVar = unnest.getVariable();
                break;
            default:
                return false;
        }
    }
    if (!pushableNestedSubplan) {
        return false;
    }
    for (int i = 0; i < nspOp.getNestedPlans().size(); i++) {
        Mutable<ILogicalOperator> nspAggRef = nspOp.getNestedPlans().get(i).getRoots().get(0);
        AggregateOperator nspAgg = (AggregateOperator) nspAggRef.getValue();
        Mutable<ILogicalOperator> nspAggChildRef = nspAgg.getInputs().get(0);
        LogicalVariable listifyVar = findListifiedVariable(nspAgg, varFromNestedAgg);
        if (listifyVar == null) {
            continue;
        }
        OperatorManipulationUtil.substituteVarRec(aggInSubplanOp, unnestVar, listifyVar, true, context);
        nspAgg.getVariables().addAll(aggInSubplanOp.getVariables());
        nspAgg.getExpressions().addAll(aggInSubplanOp.getExpressions());
        for (LogicalVariable v : aggInSubplanOp.getVariables()) {
            nspWithAgg.put(v, nspOp);
            nspAggVars.put(v, 0);
            nspAggVarToPlanIndex.put(v, i);
        }
        Mutable<ILogicalOperator> opRef1InSubplan = aggInSubplanOp.getInputs().get(0);
        if (!opRef1InSubplan.getValue().getInputs().isEmpty()) {
            Mutable<ILogicalOperator> opRef2InSubplan = opRef1InSubplan.getValue().getInputs().get(0);
            AbstractLogicalOperator op2InSubplan = (AbstractLogicalOperator) opRef2InSubplan.getValue();
            if (op2InSubplan.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
                List<Mutable<ILogicalOperator>> nspInpList = nspAgg.getInputs();
                nspInpList.clear();
                nspInpList.add(opRef1InSubplan);
                while (true) {
                    opRef2InSubplan = opRef1InSubplan.getValue().getInputs().get(0);
                    op2InSubplan = (AbstractLogicalOperator) opRef2InSubplan.getValue();
                    if (op2InSubplan.getOperatorTag() == LogicalOperatorTag.UNNEST) {
                        List<Mutable<ILogicalOperator>> opInpList = opRef1InSubplan.getValue().getInputs();
                        opInpList.clear();
                        opInpList.add(nspAggChildRef);
                        break;
                    }
                    opRef1InSubplan = opRef2InSubplan;
                    if (opRef1InSubplan.getValue().getInputs().isEmpty()) {
                        throw new IllegalStateException("PushAggregateIntoNestedSubplanRule: could not find UNNEST.");
                    }
                }
            }
        }
        subplanOpRef.setValue(subplan.getInputs().get(0).getValue());
        OperatorPropertiesUtil.typeOpRec(nspAggRef, context);
    }
    return true;
}

Example 5 with AggregateOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator in project asterixdb by apache.

the class RemoveRedundantListifyRule method applies.

private boolean applies(Mutable<ILogicalOperator> opRef, Set<LogicalVariable> varUsedAbove, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
    if (op1.getOperatorTag() != LogicalOperatorTag.UNNEST) {
        return false;
    }
    UnnestOperator unnest1 = (UnnestOperator) op1;
    ILogicalExpression expr = unnest1.getExpressionRef().getValue();
    if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
        return false;
    }
    if (((AbstractFunctionCallExpression) expr).getFunctionIdentifier() != BuiltinFunctions.SCAN_COLLECTION) {
        return false;
    }
    AbstractFunctionCallExpression functionCall = (AbstractFunctionCallExpression) expr;
    ILogicalExpression functionCallArgExpr = functionCall.getArguments().get(0).getValue();
    if (functionCallArgExpr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
        return false;
    }
    LogicalVariable unnestedVar = ((VariableReferenceExpression) functionCallArgExpr).getVariableReference();
    if (varUsedAbove.contains(unnestedVar)) {
        return false;
    }
    Mutable<ILogicalOperator> aggregateParentRef = opRef;
    AbstractLogicalOperator r = op1;
    boolean metAggregate = false;
    while (r.getInputs().size() == 1) {
        aggregateParentRef = r.getInputs().get(0);
        r = (AbstractLogicalOperator) aggregateParentRef.getValue();
        if (r.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
            AssignOperator assign = (AssignOperator) r;
            List<LogicalVariable> variables = assign.getVariables();
            // The assign operator doesn't produce any variable that is used by the unnest.
            if (variables.contains(unnestedVar)) {
                return false;
            }
        } else {
            if (r.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
                metAggregate = true;
            }
            break;
        }
    }
    if (!metAggregate) {
        return false;
    }
    AggregateOperator agg = (AggregateOperator) r;
    if (agg.getVariables().size() > 1) {
        return false;
    }
    LogicalVariable aggVar = agg.getVariables().get(0);
    ILogicalExpression aggFun = agg.getExpressions().get(0).getValue();
    if (!aggVar.equals(unnestedVar) || ((AbstractLogicalExpression) aggFun).getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
        return false;
    }
    AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) aggFun;
    if (!BuiltinFunctions.LISTIFY.equals(f.getFunctionIdentifier())) {
        return false;
    }
    if (f.getArguments().size() != 1) {
        return false;
    }
    ILogicalExpression arg0 = f.getArguments().get(0).getValue();
    if (((AbstractLogicalExpression) arg0).getExpressionTag() != LogicalExpressionTag.VARIABLE) {
        return false;
    }
    LogicalVariable paramVar = ((VariableReferenceExpression) arg0).getVariableReference();
    List<LogicalVariable> assgnVars = new ArrayList<>(1);
    assgnVars.add(unnest1.getVariable());
    List<Mutable<ILogicalExpression>> assgnExprs = new ArrayList<>(1);
    assgnExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(paramVar)));
    AssignOperator assign = new AssignOperator(assgnVars, assgnExprs);
    assign.getInputs().add(agg.getInputs().get(0));
    context.computeAndSetTypeEnvironmentForOperator(assign);
    LogicalVariable posVar = unnest1.getPositionalVariable();
    if (posVar == null) {
        // Removes the aggregate operator.
        aggregateParentRef.setValue(assign);
    } else {
        List<LogicalVariable> raggVars = new ArrayList<>(1);
        raggVars.add(posVar);
        List<Mutable<ILogicalExpression>> rAggExprs = new ArrayList<>(1);
        StatefulFunctionCallExpression tidFun = new StatefulFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.TID), UnpartitionedPropertyComputer.INSTANCE);
        rAggExprs.add(new MutableObject<ILogicalExpression>(tidFun));
        RunningAggregateOperator rAgg = new RunningAggregateOperator(raggVars, rAggExprs);
        rAgg.getInputs().add(new MutableObject<ILogicalOperator>(assign));
        aggregateParentRef.setValue(rAgg);
        context.computeAndSetTypeEnvironmentForOperator(rAgg);
    }
    // Removes the unnest operator.
    opRef.setValue(unnest1.getInputs().get(0).getValue());
    return true;
}