Search in sources :

Example 1 with SubplanOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator in project asterixdb by apache.

the class PushAggregateIntoNestedSubplanRule method pushSubplanAsAggIntoNestedSubplan.

private boolean pushSubplanAsAggIntoNestedSubplan(Mutable<ILogicalOperator> subplanOpRef, AbstractOperatorWithNestedPlans nspOp, LogicalVariable varFromNestedAgg, Map<LogicalVariable, Integer> nspAggVars, Map<LogicalVariable, AbstractOperatorWithNestedPlans> nspWithAgg, Map<LogicalVariable, Integer> nspAggVarToPlanIndex, IOptimizationContext context) throws AlgebricksException {
    SubplanOperator subplan = (SubplanOperator) subplanOpRef.getValue();
    // only free var can be varFromNestedAgg
    HashSet<LogicalVariable> freeVars = new HashSet<>();
    OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, freeVars);
    for (LogicalVariable vFree : freeVars) {
        if (!vFree.equals(varFromNestedAgg)) {
            return false;
        }
    }
    List<ILogicalPlan> plans = subplan.getNestedPlans();
    if (plans.size() > 1) {
        return false;
    }
    ILogicalPlan p = plans.get(0);
    if (p.getRoots().size() > 1) {
        return false;
    }
    Mutable<ILogicalOperator> opRef = p.getRoots().get(0);
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    if (op.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
        return false;
    }
    AggregateOperator aggInSubplanOp = (AggregateOperator) op;
    LogicalVariable unnestVar = null;
    boolean pushableNestedSubplan = false;
    while (op.getInputs().size() == 1) {
        opRef = op.getInputs().get(0);
        op = (AbstractLogicalOperator) opRef.getValue();
        switch(op.getOperatorTag()) {
            case ASSIGN:
                break;
            case UNNEST:
                UnnestOperator unnest = (UnnestOperator) op;
                if (unnest.getPositionalVariable() != null) {
                    // TODO currently subplan with both accumulating and running aggregate is not supported.
                    return false;
                }
                ILogicalExpression expr = unnest.getExpressionRef().getValue();
                if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
                    return false;
                }
                AbstractFunctionCallExpression fun = (AbstractFunctionCallExpression) expr;
                if (fun.getFunctionIdentifier() != BuiltinFunctions.SCAN_COLLECTION) {
                    return false;
                }
                ILogicalExpression arg0 = fun.getArguments().get(0).getValue();
                if (arg0.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                    return false;
                }
                VariableReferenceExpression varExpr = (VariableReferenceExpression) arg0;
                if (!varExpr.getVariableReference().equals(varFromNestedAgg)) {
                    return false;
                }
                opRef = op.getInputs().get(0);
                op = (AbstractLogicalOperator) opRef.getValue();
                if (op.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
                    return false;
                }
                pushableNestedSubplan = true;
                unnestVar = unnest.getVariable();
                break;
            default:
                return false;
        }
    }
    if (!pushableNestedSubplan) {
        return false;
    }
    for (int i = 0; i < nspOp.getNestedPlans().size(); i++) {
        Mutable<ILogicalOperator> nspAggRef = nspOp.getNestedPlans().get(i).getRoots().get(0);
        AggregateOperator nspAgg = (AggregateOperator) nspAggRef.getValue();
        Mutable<ILogicalOperator> nspAggChildRef = nspAgg.getInputs().get(0);
        LogicalVariable listifyVar = findListifiedVariable(nspAgg, varFromNestedAgg);
        if (listifyVar == null) {
            continue;
        }
        OperatorManipulationUtil.substituteVarRec(aggInSubplanOp, unnestVar, listifyVar, true, context);
        nspAgg.getVariables().addAll(aggInSubplanOp.getVariables());
        nspAgg.getExpressions().addAll(aggInSubplanOp.getExpressions());
        for (LogicalVariable v : aggInSubplanOp.getVariables()) {
            nspWithAgg.put(v, nspOp);
            nspAggVars.put(v, 0);
            nspAggVarToPlanIndex.put(v, i);
        }
        Mutable<ILogicalOperator> opRef1InSubplan = aggInSubplanOp.getInputs().get(0);
        if (!opRef1InSubplan.getValue().getInputs().isEmpty()) {
            Mutable<ILogicalOperator> opRef2InSubplan = opRef1InSubplan.getValue().getInputs().get(0);
            AbstractLogicalOperator op2InSubplan = (AbstractLogicalOperator) opRef2InSubplan.getValue();
            if (op2InSubplan.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
                List<Mutable<ILogicalOperator>> nspInpList = nspAgg.getInputs();
                nspInpList.clear();
                nspInpList.add(opRef1InSubplan);
                while (true) {
                    opRef2InSubplan = opRef1InSubplan.getValue().getInputs().get(0);
                    op2InSubplan = (AbstractLogicalOperator) opRef2InSubplan.getValue();
                    if (op2InSubplan.getOperatorTag() == LogicalOperatorTag.UNNEST) {
                        List<Mutable<ILogicalOperator>> opInpList = opRef1InSubplan.getValue().getInputs();
                        opInpList.clear();
                        opInpList.add(nspAggChildRef);
                        break;
                    }
                    opRef1InSubplan = opRef2InSubplan;
                    if (opRef1InSubplan.getValue().getInputs().isEmpty()) {
                        throw new IllegalStateException("PushAggregateIntoNestedSubplanRule: could not find UNNEST.");
                    }
                }
            }
        }
        subplanOpRef.setValue(subplan.getInputs().get(0).getValue());
        OperatorPropertiesUtil.typeOpRec(nspAggRef, context);
    }
    return true;
}
Also used : LogicalVariable(org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable) AbstractLogicalOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) AbstractFunctionCallExpression(org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression) SubplanOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator) UnnestOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator) Mutable(org.apache.commons.lang3.mutable.Mutable) ILogicalExpression(org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression) VariableReferenceExpression(org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression) AggregateOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator) ILogicalPlan(org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan) HashSet(java.util.HashSet)

Example 2 with SubplanOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator in project asterixdb by apache.

the class NestGroupByRule method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
    if (op1.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
        return false;
    }
    SubplanOperator subplan = (SubplanOperator) op1;
    if (subplan.getNestedPlans().size() != 1) {
        return false;
    }
    ILogicalPlan p = subplan.getNestedPlans().get(0);
    if (p.getRoots().size() != 1) {
        return false;
    }
    Set<LogicalVariable> free = new HashSet<LogicalVariable>();
    OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, free);
    if (free.size() != 1) {
        return false;
    }
    LogicalVariable fVar = null;
    for (LogicalVariable v : free) {
        fVar = v;
        break;
    }
    AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
    if (op2.getOperatorTag() != LogicalOperatorTag.GROUP) {
        return false;
    }
    GroupByOperator gby = (GroupByOperator) op2;
    if (gby.getNestedPlans().size() != 1) {
        return false;
    }
    ILogicalPlan p2 = gby.getNestedPlans().get(0);
    if (p2.getRoots().size() != 1) {
        return false;
    }
    Mutable<ILogicalOperator> r2 = p2.getRoots().get(0);
    AbstractLogicalOperator opr2 = (AbstractLogicalOperator) r2.getValue();
    if (opr2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
        return false;
    }
    AggregateOperator aggOuter = (AggregateOperator) opr2;
    int posInAggList = aggOuter.getVariables().indexOf(fVar);
    if (posInAggList < 0) {
        return false;
    }
    AbstractLogicalOperator outerAggSon = (AbstractLogicalOperator) aggOuter.getInputs().get(0).getValue();
    if (outerAggSon.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
        return false;
    }
    ILogicalExpression eAgg = aggOuter.getExpressions().get(posInAggList).getValue();
    if (eAgg.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
        return false;
    }
    AbstractFunctionCallExpression listifyCall = (AbstractFunctionCallExpression) eAgg;
    if (listifyCall.getFunctionIdentifier() != BuiltinFunctions.LISTIFY) {
        return false;
    }
    ILogicalExpression argListify = listifyCall.getArguments().get(0).getValue();
    if (argListify.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
        return false;
    }
    Mutable<ILogicalOperator> r = p.getRoots().get(0);
    AbstractLogicalOperator opInS = (AbstractLogicalOperator) r.getValue();
    if (opInS.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
        return false;
    }
    AggregateOperator aggInner = (AggregateOperator) opInS;
    do {
        opInS = (AbstractLogicalOperator) opInS.getInputs().get(0).getValue();
    } while (opInS.getOperatorTag() == LogicalOperatorTag.ASSIGN);
    if (opInS.getOperatorTag() != LogicalOperatorTag.GROUP) {
        return false;
    }
    AbstractLogicalOperator unnestParent = opInS;
    AbstractLogicalOperator opUnder = (AbstractLogicalOperator) opInS.getInputs().get(0).getValue();
    // skip Assigns
    while (opUnder.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
        unnestParent = opUnder;
        opUnder = (AbstractLogicalOperator) opUnder.getInputs().get(0).getValue();
    }
    if (opUnder.getOperatorTag() != LogicalOperatorTag.UNNEST) {
        return false;
    }
    UnnestOperator unnest = (UnnestOperator) opUnder;
    AbstractLogicalOperator unnestSon = (AbstractLogicalOperator) unnest.getInputs().get(0).getValue();
    if (unnestSon.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
        return false;
    }
    NestedTupleSourceOperator innerNts = (NestedTupleSourceOperator) unnestSon;
    ILogicalExpression eUnnest = unnest.getExpressionRef().getValue();
    if (eUnnest.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
        return false;
    }
    AbstractFunctionCallExpression uf = (AbstractFunctionCallExpression) eUnnest;
    if (uf.getFunctionIdentifier() != BuiltinFunctions.SCAN_COLLECTION) {
        return false;
    }
    ILogicalExpression scanArg = uf.getArguments().get(0).getValue();
    if (scanArg.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
        return false;
    }
    if (((VariableReferenceExpression) scanArg).getVariableReference() != fVar) {
        return false;
    }
    LogicalVariable uVar = unnest.getVariable();
    GroupByOperator innerGby = (GroupByOperator) opInS;
    Set<LogicalVariable> freeInInnerGby = new HashSet<LogicalVariable>();
    OperatorPropertiesUtil.getFreeVariablesInSubplans(innerGby, freeInInnerGby);
    for (LogicalVariable v : freeInInnerGby) {
        if (v != uVar) {
            return false;
        }
    }
    unnestParent.getInputs().get(0).setValue(innerNts);
    LogicalVariable listifiedVar = ((VariableReferenceExpression) argListify).getVariableReference();
    substInSubplan(aggInner, uVar, listifiedVar, context);
    gby.getNestedPlans().add(p);
    innerNts.getDataSourceReference().setValue(gby);
    opRef.setValue(gby);
    OperatorPropertiesUtil.typePlan(p, context);
    OperatorPropertiesUtil.typePlan(p2, context);
    context.computeAndSetTypeEnvironmentForOperator(gby);
    return true;
}
Also used : LogicalVariable(org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable) NestedTupleSourceOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator) GroupByOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator) AbstractLogicalOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) AbstractFunctionCallExpression(org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression) SubplanOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator) UnnestOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator) ILogicalExpression(org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression) VariableReferenceExpression(org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression) AggregateOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator) ILogicalPlan(org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan) HashSet(java.util.HashSet)

Example 3 with SubplanOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator in project asterixdb by apache.

the class LangExpressionToPlanTranslator method constructSubplanOperatorForBranch.

/**
     * Constructs a subplan operator for a branch in a if-else (or case) expression.
     *
     * @param inputOp,
     *            the input operator.
     * @param selectExpr,
     *            the expression to select tuples that are processed by this branch.
     * @param branchExpression,
     *            the expression to be evaluated in this branch.
     * @return a pair of the constructed subplan operator and the output variable for the branch.
     * @throws CompilationException
     */
protected Pair<ILogicalOperator, LogicalVariable> constructSubplanOperatorForBranch(ILogicalOperator inputOp, Mutable<ILogicalExpression> selectExpr, Expression branchExpression) throws CompilationException {
    context.enterSubplan();
    SubplanOperator subplanOp = new SubplanOperator();
    subplanOp.getInputs().add(new MutableObject<>(inputOp));
    Mutable<ILogicalOperator> nestedSource = new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(subplanOp)));
    SelectOperator select = new SelectOperator(selectExpr, false, null);
    // The select operator cannot be moved up and down, otherwise it will cause typing issues (ASTERIXDB-1203).
    OperatorPropertiesUtil.markMovable(select, false);
    select.getInputs().add(nestedSource);
    Pair<ILogicalOperator, LogicalVariable> pBranch = branchExpression.accept(this, new MutableObject<>(select));
    LogicalVariable branchVar = context.newVar();
    AggregateOperator aggOp = new AggregateOperator(Collections.singletonList(branchVar), Collections.singletonList(new MutableObject<>(new AggregateFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.LISTIFY), false, Collections.singletonList(new MutableObject<>(new VariableReferenceExpression(pBranch.second)))))));
    aggOp.getInputs().add(new MutableObject<>(pBranch.first));
    ILogicalPlan planForBranch = new ALogicalPlanImpl(new MutableObject<>(aggOp));
    subplanOp.getNestedPlans().add(planForBranch);
    context.exitSubplan();
    return new Pair<>(subplanOp, branchVar);
}
Also used : LogicalVariable(org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable) AggregateFunctionCallExpression(org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression) NestedTupleSourceOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator) SubplanOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator) SelectOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator) ALogicalPlanImpl(org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl) VariableReferenceExpression(org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression) AggregateOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator) ILogicalPlan(org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan) MutableObject(org.apache.commons.lang3.mutable.MutableObject) GbyVariableExpressionPair(org.apache.asterix.lang.common.expression.GbyVariableExpressionPair) Pair(org.apache.hyracks.algebricks.common.utils.Pair) QuantifiedPair(org.apache.asterix.lang.common.struct.QuantifiedPair)

Example 4 with SubplanOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator in project asterixdb by apache.

the class InsertUpsertCheckUtil method checkTopDown.

// Checks the query plan rooted at <code>op</code> top down to see whether there is an invalid returning expression
// for insert/upsert, i.e., a returning expression that contains dataset accesses.
private static boolean checkTopDown(ILogicalOperator op, boolean hasSubplanAboveWithDatasetAccess) {
    boolean metSubplanWithDataScan = hasSubplanAboveWithDatasetAccess;
    if (op.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
        SubplanOperator subplanOp = (SubplanOperator) op;
        metSubplanWithDataScan = containsDatasetAccess(subplanOp);
    }
    if (op.getOperatorTag() == LogicalOperatorTag.INSERT_DELETE_UPSERT && metSubplanWithDataScan) {
        return true;
    }
    for (Mutable<ILogicalOperator> inputOpRef : op.getInputs()) {
        if (checkTopDown(inputOpRef.getValue(), metSubplanWithDataScan)) {
            return true;
        }
    }
    return false;
}
Also used : SubplanOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator)

Example 5 with SubplanOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator in project asterixdb by apache.

the class EliminateSubplanRule method rewritePre.

/**
     * Eliminate Subplan above ETS
     * and Subplan that has only ops. with one input and no free vars.
     */
@Override
public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
    if (op.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
        return false;
    }
    SubplanOperator subplan = (SubplanOperator) op;
    Mutable<ILogicalOperator> outerRef = subplan.getInputs().get(0);
    AbstractLogicalOperator outerRefOp = (AbstractLogicalOperator) outerRef.getValue();
    if (outerRefOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
        elimSubplanOverEts(opRef, context);
        return true;
    }
    if (subplan.getNestedPlans().size() == 1 && subplan.getNestedPlans().get(0).getRoots().size() == 1 && !OperatorPropertiesUtil.hasFreeVariables(subplan)) {
        if (elimOneSubplanWithNoFreeVars(opRef)) {
            return true;
        }
    }
    return false;
}
Also used : SubplanOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator) AbstractLogicalOperator(org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator) ILogicalOperator(org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator)

Aggregations

SubplanOperator (org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator)29 ILogicalOperator (org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator)26 ILogicalPlan (org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan)19 LogicalVariable (org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable)18 AbstractLogicalOperator (org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator)16 ILogicalExpression (org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression)14 Mutable (org.apache.commons.lang3.mutable.Mutable)11 NestedTupleSourceOperator (org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator)11 HashSet (java.util.HashSet)10 VariableReferenceExpression (org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression)10 ArrayList (java.util.ArrayList)9 MutableObject (org.apache.commons.lang3.mutable.MutableObject)9 AggregateOperator (org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator)7 GroupByOperator (org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator)6 Pair (org.apache.hyracks.algebricks.common.utils.Pair)5 AssignOperator (org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator)5 InnerJoinOperator (org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator)5 SelectOperator (org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator)5 UnnestOperator (org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator)5 ScalarFunctionCallExpression (org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression)4