Examples with ListSet org.apache.hyracks.algebricks.common.utils.ListSet used on opensource projects

Example 21 with ListSet

use of org.apache.hyracks.algebricks.common.utils.ListSet in project asterixdb by apache.

the class RemoveUnusedAssignAndAggregateRule method removeUnusedAssigns.

private void removeUnusedAssigns(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    Set<LogicalVariable> assignVarsSetForThisOp = removeAssignVarFromConsideration(opRef);
    while (removeFromAssigns(op, assignVarsSetForThisOp, context) == 0) {
        if (op.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
            break;
        }
        op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
        opRef.setValue(op);
        assignVarsSetForThisOp = removeAssignVarFromConsideration(opRef);
    }
    Iterator<Mutable<ILogicalOperator>> childIter = op.getInputs().iterator();
    while (childIter.hasNext()) {
        Mutable<ILogicalOperator> cRef = childIter.next();
        removeUnusedAssigns(cRef, context);
    }
    if (op.hasNestedPlans()) {
        AbstractOperatorWithNestedPlans opWithNest = (AbstractOperatorWithNestedPlans) op;
        Iterator<ILogicalPlan> planIter = opWithNest.getNestedPlans().iterator();
        while (planIter.hasNext()) {
            ILogicalPlan p = planIter.next();
            for (Mutable<ILogicalOperator> r : p.getRoots()) {
                removeUnusedAssigns(r, context);
            }
        }
        // Removes redundant nested plans that produces nothing
        for (int i = opWithNest.getNestedPlans().size() - 1; i >= 0; i--) {
            ILogicalPlan nestedPlan = opWithNest.getNestedPlans().get(i);
            List<Mutable<ILogicalOperator>> rootsToBeRemoved = new ArrayList<Mutable<ILogicalOperator>>();
            for (Mutable<ILogicalOperator> r : nestedPlan.getRoots()) {
                ILogicalOperator topOp = r.getValue();
                Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
                VariableUtilities.getProducedVariablesInDescendantsAndSelf(topOp, producedVars);
                if (producedVars.size() == 0) {
                    rootsToBeRemoved.add(r);
                }
            }
            // (because a lot of places uses this assumption,  TODO(yingyib): clean them up).
            if (nestedPlan.getRoots().size() == rootsToBeRemoved.size() && opWithNest.getNestedPlans().size() > 1) {
                nestedPlan.getRoots().removeAll(rootsToBeRemoved);
                opWithNest.getNestedPlans().remove(nestedPlan);
            }
        }
    }
}

Example 22 with ListSet

use of org.apache.hyracks.algebricks.common.utils.ListSet in project asterixdb by apache.

the class PushSubplanIntoGroupByRule method rewritePre.

@Override
public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    ILogicalOperator parentOperator = opRef.getValue();
    if (context.checkIfInDontApplySet(this, parentOperator)) {
        return false;
    }
    context.addToDontApplySet(this, parentOperator);
    VariableUtilities.getUsedVariables(parentOperator, usedVarsSoFar);
    if (parentOperator.getInputs().size() <= 0) {
        return false;
    }
    boolean changed = false;
    GroupByOperator gby = null;
    for (Mutable<ILogicalOperator> ref : parentOperator.getInputs()) {
        AbstractLogicalOperator op = (AbstractLogicalOperator) ref.getValue();
        /** Only processes subplan operator. */
        List<SubplanOperator> subplans = new ArrayList<SubplanOperator>();
        if (op.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
            while (op.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
                SubplanOperator currentSubplan = (SubplanOperator) op;
                subplans.add(currentSubplan);
                op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
            }
            /** Only processes the case a group-by operator is the input of the subplan operators. */
            if (op.getOperatorTag() == LogicalOperatorTag.GROUP) {
                gby = (GroupByOperator) op;
                List<ILogicalPlan> newGbyNestedPlans = new ArrayList<ILogicalPlan>();
                for (SubplanOperator subplan : subplans) {
                    List<ILogicalPlan> subplanNestedPlans = subplan.getNestedPlans();
                    List<ILogicalPlan> gbyNestedPlans = gby.getNestedPlans();
                    List<ILogicalPlan> subplanNestedPlansToRemove = new ArrayList<ILogicalPlan>();
                    for (ILogicalPlan subplanNestedPlan : subplanNestedPlans) {
                        List<Mutable<ILogicalOperator>> rootOpRefs = subplanNestedPlan.getRoots();
                        List<Mutable<ILogicalOperator>> rootOpRefsToRemove = new ArrayList<Mutable<ILogicalOperator>>();
                        for (Mutable<ILogicalOperator> rootOpRef : rootOpRefs) {
                            /** Gets free variables in the root operator of a nested plan and its descent. */
                            Set<LogicalVariable> freeVars = new ListSet<LogicalVariable>();
                            VariableUtilities.getUsedVariablesInDescendantsAndSelf(rootOpRef.getValue(), freeVars);
                            Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
                            VariableUtilities.getProducedVariablesInDescendantsAndSelf(rootOpRef.getValue(), producedVars);
                            freeVars.removeAll(producedVars);
                            /** * Checks whether the above freeVars are all contained in live variables * of one nested plan inside the group-by operator. * If yes, then the subplan can be pushed into the nested plan of the group-by. */
                            for (ILogicalPlan gbyNestedPlanOriginal : gbyNestedPlans) {
                                // add a subplan in the original gby
                                if (!newGbyNestedPlans.contains(gbyNestedPlanOriginal)) {
                                    newGbyNestedPlans.add(gbyNestedPlanOriginal);
                                }
                                // add a pushed subplan
                                ILogicalPlan gbyNestedPlan = OperatorManipulationUtil.deepCopy(gbyNestedPlanOriginal, context);
                                List<Mutable<ILogicalOperator>> gbyRootOpRefs = gbyNestedPlan.getRoots();
                                for (int rootIndex = 0; rootIndex < gbyRootOpRefs.size(); rootIndex++) {
                                    //set the nts for a original subplan
                                    Mutable<ILogicalOperator> originalGbyRootOpRef = gbyNestedPlanOriginal.getRoots().get(rootIndex);
                                    Mutable<ILogicalOperator> originalGbyNtsRef = downToNts(originalGbyRootOpRef);
                                    NestedTupleSourceOperator originalNts = (NestedTupleSourceOperator) originalGbyNtsRef.getValue();
                                    originalNts.setDataSourceReference(new MutableObject<ILogicalOperator>(gby));
                                    //push a new subplan if possible
                                    Mutable<ILogicalOperator> gbyRootOpRef = gbyRootOpRefs.get(rootIndex);
                                    Set<LogicalVariable> liveVars = new ListSet<LogicalVariable>();
                                    VariableUtilities.getLiveVariables(gbyRootOpRef.getValue(), liveVars);
                                    if (liveVars.containsAll(freeVars)) {
                                        /** Does the actual push. */
                                        Mutable<ILogicalOperator> ntsRef = downToNts(rootOpRef);
                                        ntsRef.setValue(gbyRootOpRef.getValue());
                                        // Removes unused vars.
                                        AggregateOperator aggOp = (AggregateOperator) gbyRootOpRef.getValue();
                                        for (int varIndex = aggOp.getVariables().size() - 1; varIndex >= 0; varIndex--) {
                                            if (!freeVars.contains(aggOp.getVariables().get(varIndex))) {
                                                aggOp.getVariables().remove(varIndex);
                                                aggOp.getExpressions().remove(varIndex);
                                            }
                                        }
                                        gbyRootOpRef.setValue(rootOpRef.getValue());
                                        rootOpRefsToRemove.add(rootOpRef);
                                        // Sets the nts for a new pushed plan.
                                        Mutable<ILogicalOperator> oldGbyNtsRef = downToNts(gbyRootOpRef);
                                        NestedTupleSourceOperator nts = (NestedTupleSourceOperator) oldGbyNtsRef.getValue();
                                        nts.setDataSourceReference(new MutableObject<ILogicalOperator>(gby));
                                        newGbyNestedPlans.add(gbyNestedPlan);
                                        changed = true;
                                        continue;
                                    }
                                }
                            }
                        }
                        rootOpRefs.removeAll(rootOpRefsToRemove);
                        if (rootOpRefs.size() == 0) {
                            subplanNestedPlansToRemove.add(subplanNestedPlan);
                        }
                    }
                    subplanNestedPlans.removeAll(subplanNestedPlansToRemove);
                }
                if (changed) {
                    ref.setValue(gby);
                    gby.getNestedPlans().clear();
                    gby.getNestedPlans().addAll(newGbyNestedPlans);
                }
            }
        }
    }
    if (changed) {
        cleanup(gby);
        context.computeAndSetTypeEnvironmentForOperator(gby);
        context.computeAndSetTypeEnvironmentForOperator(parentOperator);
    }
    return changed;
}