Examples with GroupByOperator org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator used on opensource projects

Example 46 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class RemoveUnusedAssignAndAggregateRule method collectUnusedAssignedVars.

private void collectUnusedAssignedVars(Mutable<ILogicalOperator> opRef, Set<LogicalVariable> accumulatedUsedVarFromRootSet, boolean first, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    if (!first) {
        context.addToDontApplySet(this, op);
    }
    Set<LogicalVariable> assignVarsSetInThisOp = new HashSet<>();
    Set<LogicalVariable> usedVarsSetInThisOp = new HashSet<>();
    // Add used variables in this operator to the accumulated used variables set?
    boolean addUsedVarsInThisOp = true;
    // ASSIGN, AGGREGATE, UNNEST, UNIONALL, or GROUP operator found?
    boolean targetOpFound = false;
    switch(op.getOperatorTag()) {
        case ASSIGN:
            AssignOperator assign = (AssignOperator) op;
            assignVarsSetInThisOp.addAll(assign.getVariables());
            targetOpFound = true;
            break;
        case AGGREGATE:
            AggregateOperator agg = (AggregateOperator) op;
            assignVarsSetInThisOp.addAll(agg.getVariables());
            targetOpFound = true;
            break;
        case UNNEST:
            UnnestOperator uOp = (UnnestOperator) op;
            LogicalVariable pVar = uOp.getPositionalVariable();
            if (pVar != null) {
                assignVarsSetInThisOp.add(pVar);
                targetOpFound = true;
            }
            break;
        case UNIONALL:
            UnionAllOperator unionOp = (UnionAllOperator) op;
            for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> varMapping : unionOp.getVariableMappings()) {
                assignVarsSetInThisOp.add(varMapping.third);
            }
            targetOpFound = true;
            // Don't add used variables in UNIONALL.
            addUsedVarsInThisOp = false;
            break;
        case GROUP:
            GroupByOperator groupByOp = (GroupByOperator) op;
            for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorMapping : groupByOp.getDecorList()) {
                LogicalVariable decorVar = decorMapping.first;
                if (decorVar != null) {
                    assignVarsSetInThisOp.add(decorVar);
                    targetOpFound = true;
                } else {
                    // A decor var mapping can have a variable reference expression without a new variable
                    // definition, which is for rebinding the referred variable.
                    VariableReferenceExpression varExpr = (VariableReferenceExpression) decorMapping.second.getValue();
                    LogicalVariable reboundDecorVar = varExpr.getVariableReference();
                    assignVarsSetInThisOp.add(reboundDecorVar);
                }
            }
            break;
        default:
            break;
    }
    if (targetOpFound) {
        assignedVarMap.put(opRef, assignVarsSetInThisOp);
        assignedVarSet.addAll(assignVarsSetInThisOp);
    }
    if (addUsedVarsInThisOp) {
        VariableUtilities.getUsedVariables(op, usedVarsSetInThisOp);
        accumulatedUsedVarFromRootSet.addAll(usedVarsSetInThisOp);
        // paths in the plan.
        if (accumulatedUsedVarFromRootMap.containsKey(opRef)) {
            accumulatedUsedVarFromRootMap.get(opRef).addAll(accumulatedUsedVarFromRootSet);
        } else {
            accumulatedUsedVarFromRootMap.put(opRef, new HashSet<LogicalVariable>(accumulatedUsedVarFromRootSet));
        }
    } else {
        accumulatedUsedVarFromRootMap.put(opRef, new HashSet<LogicalVariable>(accumulatedUsedVarFromRootSet));
    }
    for (Mutable<ILogicalOperator> c : op.getInputs()) {
        collectUnusedAssignedVars(c, new HashSet<LogicalVariable>(accumulatedUsedVarFromRootSet), false, context);
    }
    if (op.hasNestedPlans()) {
        AbstractOperatorWithNestedPlans opWithNested = (AbstractOperatorWithNestedPlans) op;
        for (ILogicalPlan plan : opWithNested.getNestedPlans()) {
            for (Mutable<ILogicalOperator> r : plan.getRoots()) {
                collectUnusedAssignedVars(r, new HashSet<LogicalVariable>(accumulatedUsedVarFromRootSet), false, context);
            }
        }
    }
}

Example 47 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class PushProjectDownRule method pushThroughOp.

private static Pair<Boolean, Boolean> pushThroughOp(HashSet<LogicalVariable> toPush, Mutable<ILogicalOperator> opRef2, ILogicalOperator initialOp, IOptimizationContext context) throws AlgebricksException {
    List<LogicalVariable> initProjectList = new ArrayList<LogicalVariable>(toPush);
    AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
    do {
        if (op2.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE || op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE || op2.getOperatorTag() == LogicalOperatorTag.PROJECT || op2.getOperatorTag() == LogicalOperatorTag.REPLICATE || op2.getOperatorTag() == LogicalOperatorTag.SPLIT || op2.getOperatorTag() == LogicalOperatorTag.UNIONALL) {
            return new Pair<Boolean, Boolean>(false, false);
        }
        if (!op2.isMap()) {
            break;
        }
        LinkedList<LogicalVariable> usedVars = new LinkedList<LogicalVariable>();
        VariableUtilities.getUsedVariables(op2, usedVars);
        toPush.addAll(usedVars);
        LinkedList<LogicalVariable> producedVars = new LinkedList<LogicalVariable>();
        VariableUtilities.getProducedVariables(op2, producedVars);
        toPush.removeAll(producedVars);
        // we assume pipelineable ops. have only one input
        opRef2 = op2.getInputs().get(0);
        op2 = (AbstractLogicalOperator) opRef2.getValue();
    } while (true);
    LinkedList<LogicalVariable> produced2 = new LinkedList<LogicalVariable>();
    VariableUtilities.getProducedVariables(op2, produced2);
    LinkedList<LogicalVariable> used2 = new LinkedList<LogicalVariable>();
    VariableUtilities.getUsedVariables(op2, used2);
    boolean canCommuteProjection = initProjectList.containsAll(toPush) && initProjectList.containsAll(produced2) && initProjectList.containsAll(used2);
    // get rid of useless decor vars.
    if (!canCommuteProjection && op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
        boolean gbyChanged = false;
        GroupByOperator gby = (GroupByOperator) op2;
        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> newDecorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
            LogicalVariable decorVar = GroupByOperator.getDecorVariable(p);
            if (!toPush.contains(decorVar)) {
                used2.remove(decorVar);
                gbyChanged = true;
            } else {
                newDecorList.add(p);
            }
        }
        gby.getDecorList().clear();
        gby.getDecorList().addAll(newDecorList);
        if (gbyChanged) {
            context.computeAndSetTypeEnvironmentForOperator(gby);
        }
    }
    used2.clear();
    VariableUtilities.getUsedVariables(op2, used2);
    // remember that toPush is a Set
    toPush.addAll(used2);
    toPush.removeAll(produced2);
    if (toPush.isEmpty()) {
        return new Pair<Boolean, Boolean>(false, false);
    }
    boolean smthWasPushed = false;
    for (Mutable<ILogicalOperator> c : op2.getInputs()) {
        if (pushNeededProjections(toPush, c, context, initialOp)) {
            smthWasPushed = true;
        }
    }
    if (op2.hasNestedPlans()) {
        AbstractOperatorWithNestedPlans n = (AbstractOperatorWithNestedPlans) op2;
        for (ILogicalPlan p : n.getNestedPlans()) {
            for (Mutable<ILogicalOperator> r : p.getRoots()) {
                if (pushNeededProjections(toPush, r, context, initialOp)) {
                    smthWasPushed = true;
                }
            }
        }
    }
    return new Pair<Boolean, Boolean>(smthWasPushed, canCommuteProjection);
}

Example 48 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class PushGroupByIntoSortRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    ILogicalOperator op1 = opRef.getValue();
    if (op1 == null) {
        return false;
    }
    boolean changed = false;
    for (Mutable<ILogicalOperator> childRef : op1.getInputs()) {
        AbstractLogicalOperator op = (AbstractLogicalOperator) childRef.getValue();
        if (op.getOperatorTag() == LogicalOperatorTag.GROUP) {
            PhysicalOperatorTag opTag = op.getPhysicalOperator().getOperatorTag();
            GroupByOperator groupByOperator = (GroupByOperator) op;
            if (opTag == PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY) {
                Mutable<ILogicalOperator> op2Ref = op.getInputs().get(0).getValue().getInputs().get(0);
                AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
                if (op2.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.STABLE_SORT) {
                    AbstractStableSortPOperator sortPhysicalOperator = (AbstractStableSortPOperator) op2.getPhysicalOperator();
                    if (groupByOperator.getNestedPlans().size() != 1) {
                        //an aggregate and a nested-tuple-source.
                        continue;
                    }
                    ILogicalPlan p0 = groupByOperator.getNestedPlans().get(0);
                    if (p0.getRoots().size() != 1) {
                        //an aggregate and a nested-tuple-source.
                        continue;
                    }
                    Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
                    AbstractLogicalOperator r0Logical = (AbstractLogicalOperator) r0.getValue();
                    if (r0Logical.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
                        //we only rewrite aggregation function; do nothing for running aggregates
                        continue;
                    }
                    AggregateOperator aggOp = (AggregateOperator) r0.getValue();
                    AbstractLogicalOperator aggInputOp = (AbstractLogicalOperator) aggOp.getInputs().get(0).getValue();
                    if (aggInputOp.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
                        continue;
                    }
                    boolean hasIntermediateAggregate = generateMergeAggregationExpressions(groupByOperator, context);
                    if (!hasIntermediateAggregate) {
                        continue;
                    }
                    //replace preclustered gby with sort gby
                    if (!groupByOperator.isGroupAll()) {
                        op.setPhysicalOperator(new SortGroupByPOperator(groupByOperator.getGroupByList(), context.getPhysicalOptimizationConfig().getMaxFramesExternalGroupBy(), sortPhysicalOperator.getSortColumns()));
                    }
                    // remove the stable sort operator
                    op.getInputs().clear();
                    op.getInputs().addAll(op2.getInputs());
                    changed = true;
                }
            }
            continue;
        } else {
            continue;
        }
    }
    return changed;
}

Example 49 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class RemoveRedundantGroupByDecorVarsRule method checkAndApplyTheRule.

/**
     * Collect used variables in each operator in the plan until the optimizer sees a GroupBy operator.
     * It first removes duplicated variables in the decor list.
     * Then, it eliminates useless variables in the decor list that are not going to be used
     * after the given groupBy operator.
     */
protected boolean checkAndApplyTheRule(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    Set<LogicalVariable> usedVarsFromThisOp = new HashSet<>();
    Set<LogicalVariable> collectedUsedVarsBeforeThisOpFromRoot = new HashSet<>();
    boolean redundantVarsRemoved = false;
    boolean uselessVarsRemoved = false;
    // Found Group-By operator?
    if (op.getOperatorTag() == LogicalOperatorTag.GROUP) {
        GroupByOperator groupByOp = (GroupByOperator) op;
        Set<LogicalVariable> decorVars = new HashSet<>();
        // First, get rid of duplicated variables from a group-by operator's decor list.
        Iterator<Pair<LogicalVariable, Mutable<ILogicalExpression>>> iter = groupByOp.getDecorList().iterator();
        while (iter.hasNext()) {
            Pair<LogicalVariable, Mutable<ILogicalExpression>> decor = iter.next();
            if (decor.first != null || decor.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                continue;
            }
            VariableReferenceExpression varRefExpr = (VariableReferenceExpression) decor.second.getValue();
            LogicalVariable var = varRefExpr.getVariableReference();
            if (decorVars.contains(var)) {
                iter.remove();
                redundantVarsRemoved = true;
            } else {
                decorVars.add(var);
            }
        }
        // Next, get rid of useless decor variables in the GROUP-BY operator.
        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> newDecorList = new ArrayList<>();
        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : groupByOp.getDecorList()) {
            LogicalVariable decorVar = GroupByOperator.getDecorVariable(p);
            // If a variable in the decor list will not be used after this operator, then it needs to be removed.
            if (!usedVars.contains(decorVar)) {
                uselessVarsRemoved = true;
            } else {
                // Maintain the variable since it will be used.
                newDecorList.add(p);
            }
        }
        // then the decor list needs to be reset without those variables.
        if (uselessVarsRemoved) {
            groupByOp.getDecorList().clear();
            groupByOp.getDecorList().addAll(newDecorList);
        }
        // since if there are more GROUP-BY operators, the next trigger on this plan will find them.
        if (redundantVarsRemoved || uselessVarsRemoved) {
            context.computeAndSetTypeEnvironmentForOperator(groupByOp);
            context.addToDontApplySet(this, op);
            return redundantVarsRemoved || uselessVarsRemoved;
        }
    }
    // Either we have found a GroupBy operator but no removal is happened or
    // there we haven't found a GroupBy operator yet. Thus, we add used variables for this operator
    // and keep traversing the plan.
    VariableUtilities.getUsedVariables(op, usedVarsFromThisOp);
    collectedUsedVarsBeforeThisOpFromRoot.addAll(usedVars);
    usedVars.addAll(usedVarsFromThisOp);
    // Recursively check the plan and try to optimize it.
    for (int i = 0; i < op.getInputs().size(); i++) {
        boolean groupByChanged = checkAndApplyTheRule(op.getInputs().get(i), context);
        if (groupByChanged) {
            return true;
        }
    }
    // This rule can't be applied to this operator or its descendants.
    // Thus, remove the effects of this operator so that the depth-first-search can return to the parent.
    usedVars.clear();
    usedVars.addAll(collectedUsedVarsBeforeThisOpFromRoot);
    return false;
}