Examples with GroupByOperator org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator used on opensource projects

Example 21 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class IsomorphismOperatorVisitor method visitGroupByOperator.

@Override
public Boolean visitGroupByOperator(GroupByOperator op, ILogicalOperator arg) throws AlgebricksException {
    AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
    // properties
    if (aop.getOperatorTag() != LogicalOperatorTag.GROUP || aop.getPhysicalOperator().getOperatorTag() != op.getPhysicalOperator().getOperatorTag()) {
        return Boolean.FALSE;
    }
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> keyLists = op.getGroupByList();
    GroupByOperator gbyOpArg = (GroupByOperator) copyAndSubstituteVar(op, arg);
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> keyListsArg = gbyOpArg.getGroupByList();
    List<Pair<LogicalVariable, ILogicalExpression>> listLeft = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>();
    List<Pair<LogicalVariable, ILogicalExpression>> listRight = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>();
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : keyLists) {
        listLeft.add(new Pair<LogicalVariable, ILogicalExpression>(pair.first, pair.second.getValue()));
    }
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : keyListsArg) {
        listRight.add(new Pair<LogicalVariable, ILogicalExpression>(pair.first, pair.second.getValue()));
    }
    boolean isomorphic = VariableUtilities.varListEqualUnordered(listLeft, listRight);
    if (!isomorphic) {
        return Boolean.FALSE;
    }
    int sizeOp = op.getNestedPlans().size();
    int sizeArg = gbyOpArg.getNestedPlans().size();
    if (sizeOp != sizeArg) {
        return Boolean.FALSE;
    }
    GroupByOperator argOp = (GroupByOperator) arg;
    List<ILogicalPlan> plans = op.getNestedPlans();
    List<ILogicalPlan> plansArg = argOp.getNestedPlans();
    for (int i = 0; i < plans.size(); i++) {
        List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots();
        List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots();
        if (roots.size() != rootsArg.size()) {
            return Boolean.FALSE;
        }
        for (int j = 0; j < roots.size(); j++) {
            ILogicalOperator topOp1 = roots.get(j).getValue();
            ILogicalOperator topOp2 = rootsArg.get(j).getValue();
            isomorphic = IsomorphismUtilities.isOperatorIsomorphicPlanSegment(topOp1, topOp2);
            if (!isomorphic) {
                return Boolean.FALSE;
            }
        }
    }
    return isomorphic;
}

Example 22 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class IntroduceGroupByForSubplanRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
    if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
        return false;
    }
    SubplanOperator subplan = (SubplanOperator) op0;
    Iterator<ILogicalPlan> plansIter = subplan.getNestedPlans().iterator();
    ILogicalPlan p = null;
    while (plansIter.hasNext()) {
        p = plansIter.next();
    }
    if (p == null) {
        return false;
    }
    if (p.getRoots().size() != 1) {
        return false;
    }
    Mutable<ILogicalOperator> subplanRoot = p.getRoots().get(0);
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getValue();
    Mutable<ILogicalOperator> botRef = subplanRoot;
    AbstractLogicalOperator op2;
    // Project is optional
    if (op1.getOperatorTag() != LogicalOperatorTag.PROJECT) {
        op2 = op1;
    } else {
        ProjectOperator project = (ProjectOperator) op1;
        botRef = project.getInputs().get(0);
        op2 = (AbstractLogicalOperator) botRef.getValue();
    }
    if (op2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
        return false;
    }
    AggregateOperator aggregate = (AggregateOperator) op2;
    Set<LogicalVariable> free = new HashSet<LogicalVariable>();
    VariableUtilities.getUsedVariables(aggregate, free);
    Mutable<ILogicalOperator> op3Ref = aggregate.getInputs().get(0);
    AbstractLogicalOperator op3 = (AbstractLogicalOperator) op3Ref.getValue();
    while (op3.getInputs().size() == 1) {
        Set<LogicalVariable> prod = new HashSet<LogicalVariable>();
        VariableUtilities.getProducedVariables(op3, prod);
        free.removeAll(prod);
        VariableUtilities.getUsedVariables(op3, free);
        botRef = op3Ref;
        op3Ref = op3.getInputs().get(0);
        op3 = (AbstractLogicalOperator) op3Ref.getValue();
    }
    if (op3.getOperatorTag() != LogicalOperatorTag.INNERJOIN && op3.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
        return false;
    }
    AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op3;
    if (join.getCondition().getValue() == ConstantExpression.TRUE) {
        return false;
    }
    VariableUtilities.getUsedVariables(join, free);
    AbstractLogicalOperator b0 = (AbstractLogicalOperator) join.getInputs().get(0).getValue();
    // see if there's an NTS at the end of the pipeline
    NestedTupleSourceOperator outerNts = getNts(b0);
    if (outerNts == null) {
        AbstractLogicalOperator b1 = (AbstractLogicalOperator) join.getInputs().get(1).getValue();
        outerNts = getNts(b1);
        if (outerNts == null) {
            return false;
        }
    }
    Set<LogicalVariable> pkVars = computeGbyVars(outerNts, free, context);
    if (pkVars == null || pkVars.size() < 1) {
        // there is no non-trivial primary key, group-by keys are all live variables
        // that were produced by descendant or self
        ILogicalOperator subplanInput = subplan.getInputs().get(0).getValue();
        pkVars = new HashSet<LogicalVariable>();
        //get live variables
        VariableUtilities.getLiveVariables(subplanInput, pkVars);
        //get produced variables
        Set<LogicalVariable> producedVars = new HashSet<LogicalVariable>();
        VariableUtilities.getProducedVariablesInDescendantsAndSelf(subplanInput, producedVars);
        //retain the intersection
        pkVars.retainAll(producedVars);
    }
    AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Found FD for introducing group-by: " + pkVars);
    Mutable<ILogicalOperator> rightRef = join.getInputs().get(1);
    LogicalVariable testForNull = null;
    AbstractLogicalOperator right = (AbstractLogicalOperator) rightRef.getValue();
    switch(right.getOperatorTag()) {
        case UNNEST:
            {
                UnnestOperator innerUnnest = (UnnestOperator) right;
                // Select [ $y != null ]
                testForNull = innerUnnest.getVariable();
                break;
            }
        case RUNNINGAGGREGATE:
            {
                ILogicalOperator inputToRunningAggregate = right.getInputs().get(0).getValue();
                Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
                VariableUtilities.getProducedVariables(inputToRunningAggregate, producedVars);
                if (!producedVars.isEmpty()) {
                    // Select [ $y != null ]
                    testForNull = producedVars.iterator().next();
                }
                break;
            }
        case DATASOURCESCAN:
            {
                DataSourceScanOperator innerScan = (DataSourceScanOperator) right;
                // Select [ $y != null ]
                if (innerScan.getVariables().size() == 1) {
                    testForNull = innerScan.getVariables().get(0);
                }
                break;
            }
        default:
            break;
    }
    if (testForNull == null) {
        testForNull = context.newVar();
        AssignOperator tmpAsgn = new AssignOperator(testForNull, new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
        tmpAsgn.getInputs().add(new MutableObject<ILogicalOperator>(rightRef.getValue()));
        rightRef.setValue(tmpAsgn);
        context.computeAndSetTypeEnvironmentForOperator(tmpAsgn);
    }
    IFunctionInfo finfoEq = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.IS_MISSING);
    ILogicalExpression isNullTest = new ScalarFunctionCallExpression(finfoEq, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(testForNull)));
    IFunctionInfo finfoNot = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.NOT);
    ScalarFunctionCallExpression nonNullTest = new ScalarFunctionCallExpression(finfoNot, new MutableObject<ILogicalExpression>(isNullTest));
    SelectOperator selectNonNull = new SelectOperator(new MutableObject<ILogicalExpression>(nonNullTest), false, null);
    GroupByOperator g = new GroupByOperator();
    Mutable<ILogicalOperator> newSubplanRef = new MutableObject<ILogicalOperator>(subplan);
    NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(g));
    opRef.setValue(g);
    selectNonNull.getInputs().add(new MutableObject<ILogicalOperator>(nts));
    List<Mutable<ILogicalOperator>> prodInpList = botRef.getValue().getInputs();
    prodInpList.clear();
    prodInpList.add(new MutableObject<ILogicalOperator>(selectNonNull));
    ILogicalPlan gPlan = new ALogicalPlanImpl(new MutableObject<ILogicalOperator>(subplanRoot.getValue()));
    g.getNestedPlans().add(gPlan);
    subplanRoot.setValue(op3Ref.getValue());
    g.getInputs().add(newSubplanRef);
    HashSet<LogicalVariable> underVars = new HashSet<LogicalVariable>();
    VariableUtilities.getLiveVariables(subplan.getInputs().get(0).getValue(), underVars);
    underVars.removeAll(pkVars);
    Map<LogicalVariable, LogicalVariable> mappedVars = buildVarExprList(pkVars, context, g, g.getGroupByList());
    context.updatePrimaryKeys(mappedVars);
    for (LogicalVariable uv : underVars) {
        g.getDecorList().add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(null, new MutableObject<ILogicalExpression>(new VariableReferenceExpression(uv))));
    }
    OperatorPropertiesUtil.typeOpRec(subplanRoot, context);
    OperatorPropertiesUtil.typeOpRec(gPlan.getRoots().get(0), context);
    context.computeAndSetTypeEnvironmentForOperator(g);
    return true;
}

Example 23 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class SubplanOutOfGroupRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
    if (op0.getOperatorTag() != LogicalOperatorTag.GROUP) {
        return false;
    }
    GroupByOperator gby = (GroupByOperator) op0;
    Iterator<ILogicalPlan> plansIter = gby.getNestedPlans().iterator();
    ILogicalPlan p = null;
    while (plansIter.hasNext()) {
        p = plansIter.next();
    }
    if (p == null) {
        return false;
    }
    if (p.getRoots().size() != 1) {
        return false;
    }
    Mutable<ILogicalOperator> op1Ref = p.getRoots().get(0);
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) op1Ref.getValue();
    boolean found = false;
    while (op1.getInputs().size() == 1) {
        if (op1.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
            SubplanOperator subplan = (SubplanOperator) op1;
            AbstractLogicalOperator op2 = (AbstractLogicalOperator) subplan.getInputs().get(0).getValue();
            if (OperatorPropertiesUtil.isMissingTest(op2)) {
                if (subplan.getNestedPlans().size() == 1) {
                    ILogicalPlan p1 = subplan.getNestedPlans().get(0);
                    if (p1.getRoots().size() == 1) {
                        AbstractLogicalOperator r1 = (AbstractLogicalOperator) p1.getRoots().get(0).getValue();
                        if (r1.getOperatorTag() == LogicalOperatorTag.INNERJOIN || r1.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
                            // now, check that it propagates all variables,
                            // so it can be pushed
                            List<LogicalVariable> op2Vars = new ArrayList<LogicalVariable>();
                            VariableUtilities.getLiveVariables(op2, op2Vars);
                            List<LogicalVariable> op1Vars = new ArrayList<LogicalVariable>();
                            VariableUtilities.getLiveVariables(subplan, op1Vars);
                            if (op1Vars.containsAll(op2Vars)) {
                                found = true;
                                break;
                            }
                        }
                    }
                }
            }
        }
        op1Ref = op1.getInputs().get(0);
        op1 = (AbstractLogicalOperator) op1Ref.getValue();
    }
    if (!found) {
        return false;
    }
    ILogicalOperator subplan = op1;
    ILogicalOperator op2 = op1.getInputs().get(0).getValue();
    op1Ref.setValue(op2);
    Mutable<ILogicalOperator> opUnderRef = gby.getInputs().get(0);
    ILogicalOperator opUnder = opUnderRef.getValue();
    subplan.getInputs().clear();
    subplan.getInputs().add(new MutableObject<ILogicalOperator>(opUnder));
    opUnderRef.setValue(subplan);
    return true;
}

Example 24 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class CancelUnnestWithNestedListifyRule method applies.

private boolean applies(Mutable<ILogicalOperator> opRef, Set<LogicalVariable> varUsedAbove, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
    if (op1.getOperatorTag() != LogicalOperatorTag.UNNEST) {
        return false;
    }
    UnnestOperator unnest1 = (UnnestOperator) op1;
    ILogicalExpression expr = unnest1.getExpressionRef().getValue();
    LogicalVariable unnestedVar;
    switch(expr.getExpressionTag()) {
        case VARIABLE:
            unnestedVar = ((VariableReferenceExpression) expr).getVariableReference();
            break;
        case FUNCTION_CALL:
            if (((AbstractFunctionCallExpression) expr).getFunctionIdentifier() != BuiltinFunctions.SCAN_COLLECTION) {
                return false;
            }
            AbstractFunctionCallExpression functionCall = (AbstractFunctionCallExpression) expr;
            ILogicalExpression functionCallArgExpr = functionCall.getArguments().get(0).getValue();
            if (functionCallArgExpr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                return false;
            }
            unnestedVar = ((VariableReferenceExpression) functionCallArgExpr).getVariableReference();
            break;
        default:
            return false;
    }
    if (varUsedAbove.contains(unnestedVar)) {
        return false;
    }
    Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
    AbstractLogicalOperator r = (AbstractLogicalOperator) opRef2.getValue();
    if (r.getOperatorTag() != LogicalOperatorTag.GROUP) {
        return false;
    }
    // go inside of a group-by plan
    GroupByOperator gby = (GroupByOperator) r;
    if (gby.getNestedPlans().size() != 1) {
        return false;
    }
    if (gby.getNestedPlans().get(0).getRoots().size() != 1) {
        return false;
    }
    AbstractLogicalOperator nestedPlanRoot = (AbstractLogicalOperator) gby.getNestedPlans().get(0).getRoots().get(0).getValue();
    if (nestedPlanRoot.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
        return false;
    }
    AggregateOperator agg = (AggregateOperator) nestedPlanRoot;
    Mutable<ILogicalOperator> aggInputOpRef = agg.getInputs().get(0);
    if (agg.getVariables().size() > 1) {
        return false;
    }
    if (OperatorManipulationUtil.ancestorOfOperators(agg, ImmutableSet.of(LogicalOperatorTag.LIMIT, LogicalOperatorTag.ORDER, LogicalOperatorTag.GROUP, LogicalOperatorTag.DISTINCT))) {
        return false;
    }
    LogicalVariable aggVar = agg.getVariables().get(0);
    ILogicalExpression aggFun = agg.getExpressions().get(0).getValue();
    if (!aggVar.equals(unnestedVar) || ((AbstractLogicalExpression) aggFun).getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
        return false;
    }
    AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) aggFun;
    if (!BuiltinFunctions.LISTIFY.equals(f.getFunctionIdentifier())) {
        return false;
    }
    if (f.getArguments().size() != 1) {
        return false;
    }
    ILogicalExpression arg0 = f.getArguments().get(0).getValue();
    if (((AbstractLogicalExpression) arg0).getExpressionTag() != LogicalExpressionTag.VARIABLE) {
        return false;
    }
    LogicalVariable paramVar = ((VariableReferenceExpression) arg0).getVariableReference();
    ArrayList<LogicalVariable> assgnVars = new ArrayList<LogicalVariable>(1);
    assgnVars.add(unnest1.getVariable());
    ArrayList<Mutable<ILogicalExpression>> assgnExprs = new ArrayList<Mutable<ILogicalExpression>>(1);
    assgnExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(paramVar)));
    AssignOperator assign = new AssignOperator(assgnVars, assgnExprs);
    LogicalVariable posVar = unnest1.getPositionalVariable();
    if (posVar == null) {
        // Creates assignment for group-by keys.
        ArrayList<LogicalVariable> gbyKeyAssgnVars = new ArrayList<LogicalVariable>();
        ArrayList<Mutable<ILogicalExpression>> gbyKeyAssgnExprs = new ArrayList<Mutable<ILogicalExpression>>();
        for (int i = 0; i < gby.getGroupByList().size(); i++) {
            if (gby.getGroupByList().get(i).first != null) {
                gbyKeyAssgnVars.add(gby.getGroupByList().get(i).first);
                gbyKeyAssgnExprs.add(gby.getGroupByList().get(i).second);
            }
        }
        // Moves the nested pipeline before aggregation out of the group-by op.
        Mutable<ILogicalOperator> bottomOpRef = aggInputOpRef;
        AbstractLogicalOperator bottomOp = (AbstractLogicalOperator) bottomOpRef.getValue();
        while (bottomOp.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
            bottomOpRef = bottomOp.getInputs().get(0);
            bottomOp = (AbstractLogicalOperator) bottomOpRef.getValue();
        }
        // Removes the group-by operator.
        opRef.setValue(assign);
        assign.getInputs().add(aggInputOpRef);
        AssignOperator gbyKeyAssign = new AssignOperator(gbyKeyAssgnVars, gbyKeyAssgnExprs);
        gbyKeyAssign.getInputs().add(gby.getInputs().get(0));
        bottomOpRef.setValue(gbyKeyAssign);
        context.computeAndSetTypeEnvironmentForOperator(gbyKeyAssign);
        context.computeAndSetTypeEnvironmentForOperator(assign);
    } else {
        // if positional variable is used in unnest, the unnest will be pushed into the group-by as a running-aggregate
        // First create assign for the unnest variable
        List<LogicalVariable> nestedAssignVars = new ArrayList<LogicalVariable>();
        List<Mutable<ILogicalExpression>> nestedAssignExprs = new ArrayList<Mutable<ILogicalExpression>>();
        nestedAssignVars.add(unnest1.getVariable());
        nestedAssignExprs.add(new MutableObject<ILogicalExpression>(arg0));
        AssignOperator nestedAssign = new AssignOperator(nestedAssignVars, nestedAssignExprs);
        nestedAssign.getInputs().add(opRef2);
        // Then create running aggregation for the positional variable
        List<LogicalVariable> raggVars = new ArrayList<LogicalVariable>();
        List<Mutable<ILogicalExpression>> raggExprs = new ArrayList<Mutable<ILogicalExpression>>();
        raggVars.add(posVar);
        StatefulFunctionCallExpression fce = new StatefulFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.TID), UnpartitionedPropertyComputer.INSTANCE);
        raggExprs.add(new MutableObject<ILogicalExpression>(fce));
        RunningAggregateOperator raggOp = new RunningAggregateOperator(raggVars, raggExprs);
        raggOp.setExecutionMode(unnest1.getExecutionMode());
        RunningAggregatePOperator raggPOp = new RunningAggregatePOperator();
        raggOp.setPhysicalOperator(raggPOp);
        raggOp.getInputs().add(nestedPlanRoot.getInputs().get(0));
        gby.getNestedPlans().get(0).getRoots().set(0, new MutableObject<ILogicalOperator>(raggOp));
        opRef.setValue(nestedAssign);
        context.computeAndSetTypeEnvironmentForOperator(nestedAssign);
        context.computeAndSetTypeEnvironmentForOperator(raggOp);
        context.computeAndSetTypeEnvironmentForOperator(gby);
    }
    return true;
}

Example 25 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class PushGroupByThroughProduct method push.

private void push(Mutable<ILogicalOperator> opRefGby, Mutable<ILogicalOperator> opRefJoin, int branch, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorToPush, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorNotToPush, IOptimizationContext context) throws AlgebricksException {
    GroupByOperator gby = (GroupByOperator) opRefGby.getValue();
    AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) opRefJoin.getValue();
    gby.getDecorList().clear();
    gby.getDecorList().addAll(decorToPush);
    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorNotToPush) {
        LogicalVariable v1 = p.first;
        if (v1 != null) {
            VariableReferenceExpression varRef = (VariableReferenceExpression) p.second.getValue();
            LogicalVariable v2 = varRef.getVariableReference();
            OperatorManipulationUtil.substituteVarRec(join, v2, v1, true, context);
        }
    }
    Mutable<ILogicalOperator> branchRef = join.getInputs().get(branch);
    ILogicalOperator opBranch = branchRef.getValue();
    opRefJoin.setValue(opBranch);
    branchRef.setValue(gby);
    opRefGby.setValue(join);
}