Examples with ILogicalPlan org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan used on opensource projects

Example 51 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class PushProjectDownRule method pushThroughOp.

private static Pair<Boolean, Boolean> pushThroughOp(HashSet<LogicalVariable> toPush, Mutable<ILogicalOperator> opRef2, ILogicalOperator initialOp, IOptimizationContext context) throws AlgebricksException {
    List<LogicalVariable> initProjectList = new ArrayList<LogicalVariable>(toPush);
    AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
    do {
        if (op2.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE || op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE || op2.getOperatorTag() == LogicalOperatorTag.PROJECT || op2.getOperatorTag() == LogicalOperatorTag.REPLICATE || op2.getOperatorTag() == LogicalOperatorTag.SPLIT || op2.getOperatorTag() == LogicalOperatorTag.UNIONALL) {
            return new Pair<Boolean, Boolean>(false, false);
        }
        if (!op2.isMap()) {
            break;
        }
        LinkedList<LogicalVariable> usedVars = new LinkedList<LogicalVariable>();
        VariableUtilities.getUsedVariables(op2, usedVars);
        toPush.addAll(usedVars);
        LinkedList<LogicalVariable> producedVars = new LinkedList<LogicalVariable>();
        VariableUtilities.getProducedVariables(op2, producedVars);
        toPush.removeAll(producedVars);
        // we assume pipelineable ops. have only one input
        opRef2 = op2.getInputs().get(0);
        op2 = (AbstractLogicalOperator) opRef2.getValue();
    } while (true);
    LinkedList<LogicalVariable> produced2 = new LinkedList<LogicalVariable>();
    VariableUtilities.getProducedVariables(op2, produced2);
    LinkedList<LogicalVariable> used2 = new LinkedList<LogicalVariable>();
    VariableUtilities.getUsedVariables(op2, used2);
    boolean canCommuteProjection = initProjectList.containsAll(toPush) && initProjectList.containsAll(produced2) && initProjectList.containsAll(used2);
    // get rid of useless decor vars.
    if (!canCommuteProjection && op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
        boolean gbyChanged = false;
        GroupByOperator gby = (GroupByOperator) op2;
        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> newDecorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
            LogicalVariable decorVar = GroupByOperator.getDecorVariable(p);
            if (!toPush.contains(decorVar)) {
                used2.remove(decorVar);
                gbyChanged = true;
            } else {
                newDecorList.add(p);
            }
        }
        gby.getDecorList().clear();
        gby.getDecorList().addAll(newDecorList);
        if (gbyChanged) {
            context.computeAndSetTypeEnvironmentForOperator(gby);
        }
    }
    used2.clear();
    VariableUtilities.getUsedVariables(op2, used2);
    // remember that toPush is a Set
    toPush.addAll(used2);
    toPush.removeAll(produced2);
    if (toPush.isEmpty()) {
        return new Pair<Boolean, Boolean>(false, false);
    }
    boolean smthWasPushed = false;
    for (Mutable<ILogicalOperator> c : op2.getInputs()) {
        if (pushNeededProjections(toPush, c, context, initialOp)) {
            smthWasPushed = true;
        }
    }
    if (op2.hasNestedPlans()) {
        AbstractOperatorWithNestedPlans n = (AbstractOperatorWithNestedPlans) op2;
        for (ILogicalPlan p : n.getNestedPlans()) {
            for (Mutable<ILogicalOperator> r : p.getRoots()) {
                if (pushNeededProjections(toPush, r, context, initialOp)) {
                    smthWasPushed = true;
                }
            }
        }
    }
    return new Pair<Boolean, Boolean>(smthWasPushed, canCommuteProjection);
}

Example 52 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class PushGroupByIntoSortRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    ILogicalOperator op1 = opRef.getValue();
    if (op1 == null) {
        return false;
    }
    boolean changed = false;
    for (Mutable<ILogicalOperator> childRef : op1.getInputs()) {
        AbstractLogicalOperator op = (AbstractLogicalOperator) childRef.getValue();
        if (op.getOperatorTag() == LogicalOperatorTag.GROUP) {
            PhysicalOperatorTag opTag = op.getPhysicalOperator().getOperatorTag();
            GroupByOperator groupByOperator = (GroupByOperator) op;
            if (opTag == PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY) {
                Mutable<ILogicalOperator> op2Ref = op.getInputs().get(0).getValue().getInputs().get(0);
                AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
                if (op2.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.STABLE_SORT) {
                    AbstractStableSortPOperator sortPhysicalOperator = (AbstractStableSortPOperator) op2.getPhysicalOperator();
                    if (groupByOperator.getNestedPlans().size() != 1) {
                        //an aggregate and a nested-tuple-source.
                        continue;
                    }
                    ILogicalPlan p0 = groupByOperator.getNestedPlans().get(0);
                    if (p0.getRoots().size() != 1) {
                        //an aggregate and a nested-tuple-source.
                        continue;
                    }
                    Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
                    AbstractLogicalOperator r0Logical = (AbstractLogicalOperator) r0.getValue();
                    if (r0Logical.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
                        //we only rewrite aggregation function; do nothing for running aggregates
                        continue;
                    }
                    AggregateOperator aggOp = (AggregateOperator) r0.getValue();
                    AbstractLogicalOperator aggInputOp = (AbstractLogicalOperator) aggOp.getInputs().get(0).getValue();
                    if (aggInputOp.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
                        continue;
                    }
                    boolean hasIntermediateAggregate = generateMergeAggregationExpressions(groupByOperator, context);
                    if (!hasIntermediateAggregate) {
                        continue;
                    }
                    //replace preclustered gby with sort gby
                    if (!groupByOperator.isGroupAll()) {
                        op.setPhysicalOperator(new SortGroupByPOperator(groupByOperator.getGroupByList(), context.getPhysicalOptimizationConfig().getMaxFramesExternalGroupBy(), sortPhysicalOperator.getSortColumns()));
                    }
                    // remove the stable sort operator
                    op.getInputs().clear();
                    op.getInputs().addAll(op2.getInputs());
                    changed = true;
                }
            }
            continue;
        } else {
            continue;
        }
    }
    return changed;
}

Example 53 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class PushGroupByIntoSortRule method generateMergeAggregationExpressions.

private boolean generateMergeAggregationExpressions(GroupByOperator gby, IOptimizationContext context) throws AlgebricksException {
    if (gby.getNestedPlans().size() != 1) {
        throw new AlgebricksException("External/sort group-by currently works only for one nested plan with one root containing" + "an aggregate and a nested-tuple-source.");
    }
    ILogicalPlan p0 = gby.getNestedPlans().get(0);
    if (p0.getRoots().size() != 1) {
        throw new AlgebricksException("External/sort group-by currently works only for one nested plan with one root containing" + "an aggregate and a nested-tuple-source.");
    }
    IMergeAggregationExpressionFactory mergeAggregationExpressionFactory = context.getMergeAggregationExpressionFactory();
    Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
    AggregateOperator aggOp = (AggregateOperator) r0.getValue();
    List<Mutable<ILogicalExpression>> aggFuncRefs = aggOp.getExpressions();
    List<LogicalVariable> originalAggVars = aggOp.getVariables();
    int n = aggOp.getExpressions().size();
    List<Mutable<ILogicalExpression>> mergeExpressionRefs = new ArrayList<Mutable<ILogicalExpression>>();
    for (int i = 0; i < n; i++) {
        ILogicalExpression mergeExpr = mergeAggregationExpressionFactory.createMergeAggregation(originalAggVars.get(i), aggFuncRefs.get(i).getValue(), context);
        if (mergeExpr == null) {
            return false;
        }
        mergeExpressionRefs.add(new MutableObject<ILogicalExpression>(mergeExpr));
    }
    aggOp.setMergeExpressions(mergeExpressionRefs);
    return true;
}

Example 54 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class EliminateSubplanRule method elimSubplanOverEts.

private void elimSubplanOverEts(Mutable<ILogicalOperator> opRef, IOptimizationContext ctx) throws AlgebricksException {
    SubplanOperator subplan = (SubplanOperator) opRef.getValue();
    for (ILogicalPlan p : subplan.getNestedPlans()) {
        for (Mutable<ILogicalOperator> r : p.getRoots()) {
            OperatorManipulationUtil.ntsToEts(r, ctx);
        }
    }
    LinkedList<Mutable<ILogicalOperator>> allRoots = subplan.allRootsInReverseOrder();
    if (allRoots.size() == 1) {
        opRef.setValue(allRoots.get(0).getValue());
    } else {
        ILogicalOperator topOp = null;
        for (Mutable<ILogicalOperator> r : allRoots) {
            if (topOp == null) {
                topOp = r.getValue();
            } else {
                InnerJoinOperator j = new InnerJoinOperator(new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
                j.getInputs().add(new MutableObject<ILogicalOperator>(topOp));
                j.getInputs().add(r);
                ctx.setOutputTypeEnvironment(j, j.computeOutputTypeEnvironment(ctx));
                topOp = j;
            }
        }
        opRef.setValue(topOp);
    }
}

Example 55 with ILogicalPlan

use of org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan in project asterixdb by apache.

the class LoadRecordFieldsRule method pushFieldAssign.

private static void pushFieldAssign(AssignOperator a2, AbstractLogicalOperator topOp, IOptimizationContext context) throws AlgebricksException {
    if (topOp.getInputs().size() == 1 && !topOp.hasNestedPlans()) {
        Mutable<ILogicalOperator> topChild = topOp.getInputs().get(0);
        // plugAccessAboveOp(a2, topChild, context);
        List<Mutable<ILogicalOperator>> a2InptList = a2.getInputs();
        a2InptList.clear();
        a2InptList.add(topChild);
        // and link it as child in the op. tree
        topOp.getInputs().set(0, new MutableObject<>(a2));
        findAndEliminateRedundantFieldAccess(a2, context);
    } else {
        // e.g., a join
        LinkedList<LogicalVariable> usedInAccess = new LinkedList<LogicalVariable>();
        VariableUtilities.getUsedVariables(a2, usedInAccess);
        LinkedList<LogicalVariable> produced2 = new LinkedList<LogicalVariable>();
        VariableUtilities.getProducedVariables(topOp, produced2);
        if (OperatorPropertiesUtil.disjoint(produced2, usedInAccess)) {
            for (Mutable<ILogicalOperator> inp : topOp.getInputs()) {
                HashSet<LogicalVariable> v2 = new HashSet<LogicalVariable>();
                VariableUtilities.getLiveVariables(inp.getValue(), v2);
                if (!OperatorPropertiesUtil.disjoint(usedInAccess, v2)) {
                    pushAccessAboveOpRef(a2, inp, context);
                    return;
                }
            }
            if (topOp.hasNestedPlans()) {
                AbstractOperatorWithNestedPlans nestedOp = (AbstractOperatorWithNestedPlans) topOp;
                for (ILogicalPlan plan : nestedOp.getNestedPlans()) {
                    for (Mutable<ILogicalOperator> root : plan.getRoots()) {
                        HashSet<LogicalVariable> v2 = new HashSet<LogicalVariable>();
                        VariableUtilities.getLiveVariables(root.getValue(), v2);
                        if (!OperatorPropertiesUtil.disjoint(usedInAccess, v2)) {
                            pushAccessAboveOpRef(a2, root, context);
                            return;
                        }
                    }
                }
            }
            throw new AlgebricksException("Field access " + getFirstExpr(a2) + " does not correspond to any input of operator " + topOp);
        }
    }
}