Examples with ILogicalPlan org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan used on opensource projects

Example 46 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class PushSubplanIntoGroupByRule method rewritePre.

@Override
public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    ILogicalOperator parentOperator = opRef.getValue();
    if (context.checkIfInDontApplySet(this, parentOperator)) {
        return false;
    }
    context.addToDontApplySet(this, parentOperator);
    VariableUtilities.getUsedVariables(parentOperator, usedVarsSoFar);
    if (parentOperator.getInputs().size() <= 0) {
        return false;
    }
    boolean changed = false;
    GroupByOperator gby = null;
    for (Mutable<ILogicalOperator> ref : parentOperator.getInputs()) {
        AbstractLogicalOperator op = (AbstractLogicalOperator) ref.getValue();
        /** Only processes subplan operator. */
        List<SubplanOperator> subplans = new ArrayList<SubplanOperator>();
        if (op.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
            while (op.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
                SubplanOperator currentSubplan = (SubplanOperator) op;
                subplans.add(currentSubplan);
                op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
            }
            /** Only processes the case a group-by operator is the input of the subplan operators. */
            if (op.getOperatorTag() == LogicalOperatorTag.GROUP) {
                gby = (GroupByOperator) op;
                List<ILogicalPlan> newGbyNestedPlans = new ArrayList<ILogicalPlan>();
                for (SubplanOperator subplan : subplans) {
                    List<ILogicalPlan> subplanNestedPlans = subplan.getNestedPlans();
                    List<ILogicalPlan> gbyNestedPlans = gby.getNestedPlans();
                    List<ILogicalPlan> subplanNestedPlansToRemove = new ArrayList<ILogicalPlan>();
                    for (ILogicalPlan subplanNestedPlan : subplanNestedPlans) {
                        List<Mutable<ILogicalOperator>> rootOpRefs = subplanNestedPlan.getRoots();
                        List<Mutable<ILogicalOperator>> rootOpRefsToRemove = new ArrayList<Mutable<ILogicalOperator>>();
                        for (Mutable<ILogicalOperator> rootOpRef : rootOpRefs) {
                            /** Gets free variables in the root operator of a nested plan and its descent. */
                            Set<LogicalVariable> freeVars = new ListSet<LogicalVariable>();
                            VariableUtilities.getUsedVariablesInDescendantsAndSelf(rootOpRef.getValue(), freeVars);
                            Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
                            VariableUtilities.getProducedVariablesInDescendantsAndSelf(rootOpRef.getValue(), producedVars);
                            freeVars.removeAll(producedVars);
                            /** * Checks whether the above freeVars are all contained in live variables * of one nested plan inside the group-by operator. * If yes, then the subplan can be pushed into the nested plan of the group-by. */
                            for (ILogicalPlan gbyNestedPlanOriginal : gbyNestedPlans) {
                                // add a subplan in the original gby
                                if (!newGbyNestedPlans.contains(gbyNestedPlanOriginal)) {
                                    newGbyNestedPlans.add(gbyNestedPlanOriginal);
                                }
                                // add a pushed subplan
                                ILogicalPlan gbyNestedPlan = OperatorManipulationUtil.deepCopy(gbyNestedPlanOriginal, context);
                                List<Mutable<ILogicalOperator>> gbyRootOpRefs = gbyNestedPlan.getRoots();
                                for (int rootIndex = 0; rootIndex < gbyRootOpRefs.size(); rootIndex++) {
                                    //set the nts for a original subplan
                                    Mutable<ILogicalOperator> originalGbyRootOpRef = gbyNestedPlanOriginal.getRoots().get(rootIndex);
                                    Mutable<ILogicalOperator> originalGbyNtsRef = downToNts(originalGbyRootOpRef);
                                    NestedTupleSourceOperator originalNts = (NestedTupleSourceOperator) originalGbyNtsRef.getValue();
                                    originalNts.setDataSourceReference(new MutableObject<ILogicalOperator>(gby));
                                    //push a new subplan if possible
                                    Mutable<ILogicalOperator> gbyRootOpRef = gbyRootOpRefs.get(rootIndex);
                                    Set<LogicalVariable> liveVars = new ListSet<LogicalVariable>();
                                    VariableUtilities.getLiveVariables(gbyRootOpRef.getValue(), liveVars);
                                    if (liveVars.containsAll(freeVars)) {
                                        /** Does the actual push. */
                                        Mutable<ILogicalOperator> ntsRef = downToNts(rootOpRef);
                                        ntsRef.setValue(gbyRootOpRef.getValue());
                                        // Removes unused vars.
                                        AggregateOperator aggOp = (AggregateOperator) gbyRootOpRef.getValue();
                                        for (int varIndex = aggOp.getVariables().size() - 1; varIndex >= 0; varIndex--) {
                                            if (!freeVars.contains(aggOp.getVariables().get(varIndex))) {
                                                aggOp.getVariables().remove(varIndex);
                                                aggOp.getExpressions().remove(varIndex);
                                            }
                                        }
                                        gbyRootOpRef.setValue(rootOpRef.getValue());
                                        rootOpRefsToRemove.add(rootOpRef);
                                        // Sets the nts for a new pushed plan.
                                        Mutable<ILogicalOperator> oldGbyNtsRef = downToNts(gbyRootOpRef);
                                        NestedTupleSourceOperator nts = (NestedTupleSourceOperator) oldGbyNtsRef.getValue();
                                        nts.setDataSourceReference(new MutableObject<ILogicalOperator>(gby));
                                        newGbyNestedPlans.add(gbyNestedPlan);
                                        changed = true;
                                        continue;
                                    }
                                }
                            }
                        }
                        rootOpRefs.removeAll(rootOpRefsToRemove);
                        if (rootOpRefs.size() == 0) {
                            subplanNestedPlansToRemove.add(subplanNestedPlan);
                        }
                    }
                    subplanNestedPlans.removeAll(subplanNestedPlansToRemove);
                }
                if (changed) {
                    ref.setValue(gby);
                    gby.getNestedPlans().clear();
                    gby.getNestedPlans().addAll(newGbyNestedPlans);
                }
            }
        }
    }
    if (changed) {
        cleanup(gby);
        context.computeAndSetTypeEnvironmentForOperator(gby);
        context.computeAndSetTypeEnvironmentForOperator(parentOperator);
    }
    return changed;
}

Example 47 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class RemoveUnusedAssignAndAggregateRule method collectUnusedAssignedVars.

private void collectUnusedAssignedVars(Mutable<ILogicalOperator> opRef, Set<LogicalVariable> accumulatedUsedVarFromRootSet, boolean first, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    if (!first) {
        context.addToDontApplySet(this, op);
    }
    Set<LogicalVariable> assignVarsSetInThisOp = new HashSet<>();
    Set<LogicalVariable> usedVarsSetInThisOp = new HashSet<>();
    // Add used variables in this operator to the accumulated used variables set?
    boolean addUsedVarsInThisOp = true;
    // ASSIGN, AGGREGATE, UNNEST, UNIONALL, or GROUP operator found?
    boolean targetOpFound = false;
    switch(op.getOperatorTag()) {
        case ASSIGN:
            AssignOperator assign = (AssignOperator) op;
            assignVarsSetInThisOp.addAll(assign.getVariables());
            targetOpFound = true;
            break;
        case AGGREGATE:
            AggregateOperator agg = (AggregateOperator) op;
            assignVarsSetInThisOp.addAll(agg.getVariables());
            targetOpFound = true;
            break;
        case UNNEST:
            UnnestOperator uOp = (UnnestOperator) op;
            LogicalVariable pVar = uOp.getPositionalVariable();
            if (pVar != null) {
                assignVarsSetInThisOp.add(pVar);
                targetOpFound = true;
            }
            break;
        case UNIONALL:
            UnionAllOperator unionOp = (UnionAllOperator) op;
            for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> varMapping : unionOp.getVariableMappings()) {
                assignVarsSetInThisOp.add(varMapping.third);
            }
            targetOpFound = true;
            // Don't add used variables in UNIONALL.
            addUsedVarsInThisOp = false;
            break;
        case GROUP:
            GroupByOperator groupByOp = (GroupByOperator) op;
            for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorMapping : groupByOp.getDecorList()) {
                LogicalVariable decorVar = decorMapping.first;
                if (decorVar != null) {
                    assignVarsSetInThisOp.add(decorVar);
                    targetOpFound = true;
                } else {
                    // A decor var mapping can have a variable reference expression without a new variable
                    // definition, which is for rebinding the referred variable.
                    VariableReferenceExpression varExpr = (VariableReferenceExpression) decorMapping.second.getValue();
                    LogicalVariable reboundDecorVar = varExpr.getVariableReference();
                    assignVarsSetInThisOp.add(reboundDecorVar);
                }
            }
            break;
        default:
            break;
    }
    if (targetOpFound) {
        assignedVarMap.put(opRef, assignVarsSetInThisOp);
        assignedVarSet.addAll(assignVarsSetInThisOp);
    }
    if (addUsedVarsInThisOp) {
        VariableUtilities.getUsedVariables(op, usedVarsSetInThisOp);
        accumulatedUsedVarFromRootSet.addAll(usedVarsSetInThisOp);
        // paths in the plan.
        if (accumulatedUsedVarFromRootMap.containsKey(opRef)) {
            accumulatedUsedVarFromRootMap.get(opRef).addAll(accumulatedUsedVarFromRootSet);
        } else {
            accumulatedUsedVarFromRootMap.put(opRef, new HashSet<LogicalVariable>(accumulatedUsedVarFromRootSet));
        }
    } else {
        accumulatedUsedVarFromRootMap.put(opRef, new HashSet<LogicalVariable>(accumulatedUsedVarFromRootSet));
    }
    for (Mutable<ILogicalOperator> c : op.getInputs()) {
        collectUnusedAssignedVars(c, new HashSet<LogicalVariable>(accumulatedUsedVarFromRootSet), false, context);
    }
    if (op.hasNestedPlans()) {
        AbstractOperatorWithNestedPlans opWithNested = (AbstractOperatorWithNestedPlans) op;
        for (ILogicalPlan plan : opWithNested.getNestedPlans()) {
            for (Mutable<ILogicalOperator> r : plan.getRoots()) {
                collectUnusedAssignedVars(r, new HashSet<LogicalVariable>(accumulatedUsedVarFromRootSet), false, context);
            }
        }
    }
}

Example 48 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class SetAlgebricksPhysicalOperatorsRule method generateMergeAggregationExpressions.

private static boolean generateMergeAggregationExpressions(GroupByOperator gby, IOptimizationContext context) throws AlgebricksException {
    if (gby.getNestedPlans().size() != 1) {
        //an aggregate and a nested-tuple-source.
        throw new AlgebricksException("External group-by currently works only for one nested plan with one root containing" + "an aggregate and a nested-tuple-source.");
    }
    ILogicalPlan p0 = gby.getNestedPlans().get(0);
    if (p0.getRoots().size() != 1) {
        //an aggregate and a nested-tuple-source.
        throw new AlgebricksException("External group-by currently works only for one nested plan with one root containing" + "an aggregate and a nested-tuple-source.");
    }
    IMergeAggregationExpressionFactory mergeAggregationExpressionFactory = context.getMergeAggregationExpressionFactory();
    Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
    AbstractLogicalOperator r0Logical = (AbstractLogicalOperator) r0.getValue();
    if (r0Logical.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
        return false;
    }
    // Check whether there are multiple aggregates in the sub plan.
    ILogicalOperator r1Logical = r0Logical;
    while (r1Logical.hasInputs()) {
        r1Logical = r1Logical.getInputs().get(0).getValue();
        if (r1Logical.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
            return false;
        }
    }
    AggregateOperator aggOp = (AggregateOperator) r0.getValue();
    List<Mutable<ILogicalExpression>> aggFuncRefs = aggOp.getExpressions();
    List<LogicalVariable> originalAggVars = aggOp.getVariables();
    int n = aggOp.getExpressions().size();
    List<Mutable<ILogicalExpression>> mergeExpressionRefs = new ArrayList<Mutable<ILogicalExpression>>();
    for (int i = 0; i < n; i++) {
        ILogicalExpression mergeExpr = mergeAggregationExpressionFactory.createMergeAggregation(originalAggVars.get(i), aggFuncRefs.get(i).getValue(), context);
        if (mergeExpr == null) {
            return false;
        }
        mergeExpressionRefs.add(new MutableObject<ILogicalExpression>(mergeExpr));
    }
    aggOp.setMergeExpressions(mergeExpressionRefs);
    return true;
}

Example 49 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class IntroduceGroupByForSubplanRule method buildVarExprList.

private Map<LogicalVariable, LogicalVariable> buildVarExprList(Collection<LogicalVariable> vars, IOptimizationContext context, GroupByOperator g, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException {
    Map<LogicalVariable, LogicalVariable> m = new HashMap<LogicalVariable, LogicalVariable>();
    for (LogicalVariable ov : vars) {
        LogicalVariable newVar = context.newVar();
        ILogicalExpression varExpr = new VariableReferenceExpression(newVar);
        outVeList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(ov, new MutableObject<ILogicalExpression>(varExpr)));
        for (ILogicalPlan p : g.getNestedPlans()) {
            for (Mutable<ILogicalOperator> r : p.getRoots()) {
                OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true, context);
            }
        }
        AbstractLogicalOperator opUnder = (AbstractLogicalOperator) g.getInputs().get(0).getValue();
        OperatorManipulationUtil.substituteVarRec(opUnder, ov, newVar, true, context);
        m.put(ov, newVar);
    }
    return m;
}

Example 50 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class MoveFreeVariableOperatorOutOfSubplanRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    if (op.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
        return false;
    }
    SubplanOperator subplanOp = (SubplanOperator) op;
    ILogicalOperator inputOp = subplanOp.getInputs().get(0).getValue();
    Set<LogicalVariable> liveVarsBeforeSubplan = new HashSet<>();
    VariableUtilities.getLiveVariables(inputOp, liveVarsBeforeSubplan);
    boolean changed = false;
    for (ILogicalPlan plan : subplanOp.getNestedPlans()) {
        for (Mutable<ILogicalOperator> rootRef : plan.getRoots()) {
            //Make sure we are looking at subplan with a scan/join
            if (!descOrSelfIsScanOrJoin(rootRef.getValue())) {
                continue;
            }
            Mutable<ILogicalOperator> currentOpRef = rootRef;
            ILogicalOperator currentOp = rootRef.getValue();
            while (currentOp.getInputs().size() == 1) {
                Mutable<ILogicalOperator> childOpRef = currentOp.getInputs().get(0);
                ILogicalOperator childOp = childOpRef.getValue();
                // Try to move operators that only uses free variables out of the subplan.
                if (movableOperator(currentOp.getOperatorTag()) && independentOperator(currentOp, liveVarsBeforeSubplan) && producedVariablesCanbePropagated(currentOp)) {
                    extractOperator(subplanOp, inputOp, currentOpRef);
                    inputOp = currentOp;
                    changed = true;
                } else {
                    // in the case the operator is not moved, move currentOpRef to childOpRef.
                    currentOpRef = childOpRef;
                }
                currentOp = childOp;
            }
        }
    }
    return changed;
}