Examples with AbstractLogicalOperator org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator used on opensource projects

Example 51 with AbstractLogicalOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator in project asterixdb by apache.

the class IntroduceProjectsRule method introduceProjects.

protected boolean introduceProjects(AbstractLogicalOperator parentOp, int parentInputIndex, Mutable<ILogicalOperator> opRef, Set<LogicalVariable> parentUsedVars, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    boolean modified = false;
    usedVars.clear();
    VariableUtilities.getUsedVariables(op, usedVars);
    // In the top-down pass, maintain a set of variables that are used in op and all its parents.
    // This is a necessary step for the newly created project operator during this optimization,
    // since we already have all information from collectUsedVars() method for the other operators.
    HashSet<LogicalVariable> parentsUsedVars = new HashSet<>();
    parentsUsedVars.addAll(parentUsedVars);
    parentsUsedVars.addAll(usedVars);
    if (allUsedVarsAfterOpMap.get(op) != null) {
        parentsUsedVars.addAll(allUsedVarsAfterOpMap.get(op));
    }
    // Descend into children.
    for (int i = 0; i < op.getInputs().size(); i++) {
        Mutable<ILogicalOperator> inputOpRef = op.getInputs().get(i);
        if (introduceProjects(op, i, inputOpRef, parentsUsedVars, context)) {
            modified = true;
        }
    }
    if (modified) {
        context.computeAndSetTypeEnvironmentForOperator(op);
    }
    // In the bottom-up pass, determine which live variables are not used by op's parents.
    // Such variables are be projected away.
    liveVars.clear();
    VariableUtilities.getLiveVariables(op, liveVars);
    producedVars.clear();
    VariableUtilities.getProducedVariables(op, producedVars);
    liveVars.removeAll(producedVars);
    projectVars.clear();
    for (LogicalVariable liveVar : liveVars) {
        if (parentsUsedVars.contains(liveVar)) {
            projectVars.add(liveVar);
        }
    }
    // Some of the variables that are live at this op are not used above.
    if (projectVars.size() != liveVars.size()) {
        // Add a project operator under each of op's qualifying input branches.
        for (int i = 0; i < op.getInputs().size(); i++) {
            ILogicalOperator childOp = op.getInputs().get(i).getValue();
            liveVars.clear();
            VariableUtilities.getLiveVariables(childOp, liveVars);
            List<LogicalVariable> vars = new ArrayList<>();
            vars.addAll(projectVars);
            // Only retain those variables that are live in the i-th input branch.
            vars.retainAll(liveVars);
            if (vars.size() != liveVars.size()) {
                ProjectOperator projectOp = new ProjectOperator(vars);
                projectOp.getInputs().add(new MutableObject<ILogicalOperator>(childOp));
                op.getInputs().get(i).setValue(projectOp);
                context.computeAndSetTypeEnvironmentForOperator(projectOp);
                modified = true;
            }
        }
    } else if (op.getOperatorTag() == LogicalOperatorTag.PROJECT) {
        // Check if the existing project has become useless.
        liveVars.clear();
        VariableUtilities.getLiveVariables(op.getInputs().get(0).getValue(), liveVars);
        ProjectOperator projectOp = (ProjectOperator) op;
        List<LogicalVariable> projectVarsTemp = projectOp.getVariables();
        if (liveVars.size() == projectVarsTemp.size() && liveVars.containsAll(projectVarsTemp)) {
            boolean eliminateProject = true;
            // For UnionAll the variables must also be in exactly the correct order.
            if (parentOp.getOperatorTag() == LogicalOperatorTag.UNIONALL) {
                eliminateProject = canEliminateProjectBelowUnion((UnionAllOperator) parentOp, projectOp, parentInputIndex);
            }
            if (eliminateProject) {
                // The existing project has become useless. Remove it.
                parentOp.getInputs().get(parentInputIndex).setValue(op.getInputs().get(0).getValue());
            }
        }
    }
    if (modified) {
        context.computeAndSetTypeEnvironmentForOperator(op);
    }
    return modified;
}

Example 52 with AbstractLogicalOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator in project asterixdb by apache.

the class AbstractIntroduceGroupByCombinerRule method tryToPushRoot.

private boolean tryToPushRoot(Mutable<ILogicalOperator> root, GroupByOperator oldGbyOp, GroupByOperator newGbyOp, BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context, List<Mutable<ILogicalOperator>> toPushAccumulate, Set<SimilarAggregatesInfo> toReplaceSet) throws AlgebricksException {
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) root.getValue();
    if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
        return false;
    }
    AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
    // Finds nested group-by if any.
    AbstractLogicalOperator op3 = op2;
    while (op3.getOperatorTag() != LogicalOperatorTag.GROUP && op3.getInputs().size() == 1) {
        op3 = (AbstractLogicalOperator) op3.getInputs().get(0).getValue();
    }
    if (op3.getOperatorTag() != LogicalOperatorTag.GROUP) {
        AggregateOperator initAgg = (AggregateOperator) op1;
        Pair<Boolean, Mutable<ILogicalOperator>> pOpRef = tryToPushAgg(initAgg, newGbyOp, toReplaceSet, context);
        if (!pOpRef.first) {
            return false;
        }
        Mutable<ILogicalOperator> opRef = pOpRef.second;
        if (opRef != null) {
            toPushAccumulate.add(opRef);
        }
        bi.modifyGbyMap.put(oldGbyOp, gbyVars);
        return true;
    } else {
        GroupByOperator nestedGby = (GroupByOperator) op3;
        List<LogicalVariable> gbyVars2 = nestedGby.getGbyVarList();
        Set<LogicalVariable> freeVars = new HashSet<>();
        // Removes non-free variables defined in the nested plan.
        OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(nestedGby, freeVars);
        gbyVars2.retainAll(freeVars);
        List<LogicalVariable> concatGbyVars = new ArrayList<LogicalVariable>(gbyVars);
        concatGbyVars.addAll(gbyVars2);
        for (ILogicalPlan p : nestedGby.getNestedPlans()) {
            for (Mutable<ILogicalOperator> r2 : p.getRoots()) {
                if (!tryToPushRoot(r2, nestedGby, newGbyOp, bi, concatGbyVars, context, toPushAccumulate, toReplaceSet)) {
                    return false;
                }
            }
        }
        /***
             * Push the nested pipeline which provides the input to the nested group operator into newGbyOp (the combined gby op).
             * The change is to fix asterixdb issue 782.
             */
        // Finds the reference of the bottom-most operator in the pipeline that
        // should not be pushed to the combiner group-by.
        Mutable<ILogicalOperator> currentOpRef = new MutableObject<ILogicalOperator>(nestedGby);
        Mutable<ILogicalOperator> bottomOpRef = findBottomOpRefStayInOldGby(currentOpRef);
        // Adds the used variables in the pipeline from <code>currentOpRef</code> to <code>bottomOpRef</code>
        // into the group-by keys for the introduced combiner group-by operator.
        Set<LogicalVariable> usedVars = collectUsedFreeVariables(currentOpRef, bottomOpRef);
        for (LogicalVariable usedVar : usedVars) {
            if (!concatGbyVars.contains(usedVar)) {
                concatGbyVars.add(usedVar);
            }
        }
        // Retains the nested pipeline above the identified operator in the old group-by operator.
        // Pushes the nested pipeline under the select operator into the new group-by operator.
        Mutable<ILogicalOperator> oldNtsRef = findNtsRef(currentOpRef);
        ILogicalOperator opToCombiner = bottomOpRef.getValue().getInputs().get(0).getValue();
        if (opToCombiner.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
            // No pipeline other than the aggregate operator needs to push to combiner.
            return true;
        }
        bottomOpRef.getValue().getInputs().set(0, new MutableObject<ILogicalOperator>(oldNtsRef.getValue()));
        Mutable<ILogicalOperator> newGbyNestedOpRef = findNtsRef(toPushAccumulate.get(0));
        NestedTupleSourceOperator newNts = (NestedTupleSourceOperator) newGbyNestedOpRef.getValue();
        newGbyNestedOpRef.setValue(opToCombiner);
        oldNtsRef.setValue(newNts);
        return true;
    }
}

Example 53 with AbstractLogicalOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator in project asterixdb by apache.

the class ExtractCommonOperatorsRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    if (op.getOperatorTag() != LogicalOperatorTag.WRITE && op.getOperatorTag() != LogicalOperatorTag.WRITE_RESULT && op.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT) {
        return false;
    }
    boolean rewritten = false;
    boolean changed;
    if (!roots.isEmpty()) {
        do {
            changed = false;
            // applying the rewriting until fixpoint
            topDownMaterialization(roots);
            genCandidates(context);
            removeTrivialShare();
            if (!equivalenceClasses.isEmpty()) {
                changed = rewrite(context);
            }
            if (!rewritten) {
                rewritten = changed;
            }
            equivalenceClasses.clear();
            childrenToParents.clear();
            opToCandidateInputs.clear();
            clusterMap.clear();
            clusterWaitForMap.clear();
            // Resets lastUsedClusterId to 0.
            lastUsedClusterId = 0;
        } while (changed);
        roots.clear();
    }
    return rewritten;
}

Example 54 with AbstractLogicalOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator in project asterixdb by apache.

the class ExtractCommonOperatorsRule method rewritePre.

@Override
public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    if (op.getOperatorTag() != LogicalOperatorTag.WRITE && op.getOperatorTag() != LogicalOperatorTag.WRITE_RESULT && op.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT) {
        return false;
    }
    MutableObject<ILogicalOperator> mutableOp = new MutableObject<>(op);
    if (!roots.contains(mutableOp)) {
        roots.add(mutableOp);
    }
    return false;
}

Example 55 with AbstractLogicalOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator in project asterixdb by apache.

the class ExtractCommonOperatorsRule method removeTrivialShare.

private void removeTrivialShare() {
    for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
        for (int i = candidates.size() - 1; i >= 0; i--) {
            Mutable<ILogicalOperator> opRef = candidates.get(i);
            AbstractLogicalOperator aop = (AbstractLogicalOperator) opRef.getValue();
            if (aop.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
                aop = (AbstractLogicalOperator) aop.getInputs().get(0).getValue();
            }
            if (aop.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
                candidates.remove(i);
            }
        }
    }
    for (int i = equivalenceClasses.size() - 1; i >= 0; i--) {
        if (equivalenceClasses.get(i).size() < 2) {
            equivalenceClasses.remove(i);
        }
    }
}