Examples with ALogicalPlanImpl org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl used on opensource projects

Example 11 with ALogicalPlanImpl

use of org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl in project asterixdb by apache.

the class InlineSubplanInputForNestedTupleSourceRule method applyGeneralFlattening.

private Pair<Boolean, LinkedHashMap<LogicalVariable, LogicalVariable>> applyGeneralFlattening(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    SubplanOperator subplanOp = (SubplanOperator) opRef.getValue();
    if (!SubplanFlatteningUtil.containsOperators(subplanOp, ImmutableSet.of(LogicalOperatorTag.DATASOURCESCAN, LogicalOperatorTag.INNERJOIN, // We don't have nested runtime for union-all and distinct hence we have to include them here.
    LogicalOperatorTag.LEFTOUTERJOIN, LogicalOperatorTag.UNIONALL, LogicalOperatorTag.DISTINCT))) {
        return new Pair<>(false, new LinkedHashMap<>());
    }
    Mutable<ILogicalOperator> inputOpRef = subplanOp.getInputs().get(0);
    ILogicalOperator inputOpBackup = inputOpRef.getValue();
    // Creates parameters for the left outer join operator.
    Pair<ILogicalOperator, Set<LogicalVariable>> primaryOpAndVars = EquivalenceClassUtils.findOrCreatePrimaryKeyOpAndVariables(inputOpBackup, true, context);
    ILogicalOperator inputOp = primaryOpAndVars.first;
    Set<LogicalVariable> primaryKeyVars = primaryOpAndVars.second;
    inputOpRef.setValue(inputOp);
    Set<LogicalVariable> inputLiveVars = new HashSet<>();
    VariableUtilities.getLiveVariables(inputOp, inputLiveVars);
    Pair<Map<LogicalVariable, LogicalVariable>, List<Pair<IOrder, Mutable<ILogicalExpression>>>> varMapAndOrderExprs = SubplanFlatteningUtil.inlineAllNestedTupleSource(subplanOp, context);
    Map<LogicalVariable, LogicalVariable> varMap = varMapAndOrderExprs.first;
    if (varMap == null) {
        inputOpRef.setValue(inputOpBackup);
        return new Pair<>(false, new LinkedHashMap<>());
    }
    Mutable<ILogicalOperator> lowestAggregateRefInSubplan = SubplanFlatteningUtil.findLowestAggregate(subplanOp.getNestedPlans().get(0).getRoots().get(0));
    Mutable<ILogicalOperator> rightInputOpRef = lowestAggregateRefInSubplan.getValue().getInputs().get(0);
    ILogicalOperator rightInputOp = rightInputOpRef.getValue();
    // Creates a variable to indicate whether a left input tuple is killed in the plan rooted at rightInputOp.
    LogicalVariable assignVar = context.newVar();
    ILogicalOperator assignOp = new AssignOperator(assignVar, new MutableObject<>(ConstantExpression.TRUE));
    assignOp.getInputs().add(rightInputOpRef);
    context.computeAndSetTypeEnvironmentForOperator(assignOp);
    rightInputOpRef = new MutableObject<>(assignOp);
    // Constructs the join predicate for the leftOuter join.
    List<Mutable<ILogicalExpression>> joinPredicates = new ArrayList<>();
    for (LogicalVariable liveVar : primaryKeyVars) {
        List<Mutable<ILogicalExpression>> arguments = new ArrayList<>();
        arguments.add(new MutableObject<>(new VariableReferenceExpression(liveVar)));
        LogicalVariable rightVar = varMap.get(liveVar);
        arguments.add(new MutableObject<>(new VariableReferenceExpression(rightVar)));
        ILogicalExpression expr = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AlgebricksBuiltinFunctions.EQ), arguments);
        joinPredicates.add(new MutableObject<>(expr));
    }
    ILogicalExpression joinExpr = joinPredicates.size() > 1 ? new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AlgebricksBuiltinFunctions.AND), joinPredicates) : joinPredicates.size() > 0 ? joinPredicates.get(0).getValue() : ConstantExpression.TRUE;
    LeftOuterJoinOperator leftOuterJoinOp = new LeftOuterJoinOperator(new MutableObject<>(joinExpr), inputOpRef, rightInputOpRef);
    OperatorManipulationUtil.computeTypeEnvironmentBottomUp(rightInputOp, context);
    context.computeAndSetTypeEnvironmentForOperator(leftOuterJoinOp);
    // Creates group-by operator.
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByDecorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
    List<ILogicalPlan> nestedPlans = new ArrayList<>();
    GroupByOperator groupbyOp = new GroupByOperator(groupByList, groupByDecorList, nestedPlans);
    LinkedHashMap<LogicalVariable, LogicalVariable> replacedVarMap = new LinkedHashMap<>();
    for (LogicalVariable liveVar : primaryKeyVars) {
        LogicalVariable newVar = context.newVar();
        groupByList.add(new Pair<>(newVar, new MutableObject<>(new VariableReferenceExpression(liveVar))));
        // Adds variables for replacements in ancestors.
        replacedVarMap.put(liveVar, newVar);
    }
    for (LogicalVariable liveVar : inputLiveVars) {
        if (primaryKeyVars.contains(liveVar)) {
            continue;
        }
        groupByDecorList.add(new Pair<>(null, new MutableObject<>(new VariableReferenceExpression(liveVar))));
    }
    // Sets up the nested plan for the groupby operator.
    Mutable<ILogicalOperator> aggOpRef = subplanOp.getNestedPlans().get(0).getRoots().get(0);
    // Clears the input of the lowest aggregate.
    lowestAggregateRefInSubplan.getValue().getInputs().clear();
    Mutable<ILogicalOperator> currentOpRef = lowestAggregateRefInSubplan;
    // Adds an optional order operator.
    List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExprs = varMapAndOrderExprs.second;
    if (!orderExprs.isEmpty()) {
        OrderOperator orderOp = new OrderOperator(orderExprs);
        currentOpRef = new MutableObject<>(orderOp);
        lowestAggregateRefInSubplan.getValue().getInputs().add(currentOpRef);
    }
    // Adds a select operator into the nested plan for group-by to remove tuples with NULL on {@code assignVar}, i.e.,
    // subplan input tuples that are filtered out within a subplan.
    Mutable<ILogicalExpression> filterVarExpr = new MutableObject<>(new VariableReferenceExpression(assignVar));
    List<Mutable<ILogicalExpression>> args = new ArrayList<>();
    args.add(filterVarExpr);
    List<Mutable<ILogicalExpression>> argsForNotFunction = new ArrayList<>();
    argsForNotFunction.add(new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.IS_MISSING), args)));
    SelectOperator selectOp = new SelectOperator(new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.NOT), argsForNotFunction)), false, null);
    currentOpRef.getValue().getInputs().add(new MutableObject<>(selectOp));
    selectOp.getInputs().add(new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(groupbyOp))));
    List<Mutable<ILogicalOperator>> nestedRoots = new ArrayList<>();
    nestedRoots.add(aggOpRef);
    nestedPlans.add(new ALogicalPlanImpl(nestedRoots));
    groupbyOp.getInputs().add(new MutableObject<>(leftOuterJoinOp));
    // Replaces subplan with the group-by operator.
    opRef.setValue(groupbyOp);
    OperatorManipulationUtil.computeTypeEnvironmentBottomUp(groupbyOp, context);
    // Recursively applys this rule to the nested plan of the subplan operator,
    // for the case where there are nested subplan operators within {@code subplanOp}.
    Pair<Boolean, LinkedHashMap<LogicalVariable, LogicalVariable>> result = rewriteSubplanOperator(rightInputOpRef, context);
    VariableUtilities.substituteVariables(leftOuterJoinOp, result.second, context);
    VariableUtilities.substituteVariables(groupbyOp, result.second, context);
    // No var mapping from the right input operator should be populated up.
    return new Pair<>(true, replacedVarMap);
}

Example 12 with ALogicalPlanImpl

use of org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl in project asterixdb by apache.

the class SqlppExpressionToPlanTranslator method visit.

@Override
public Pair<ILogicalOperator, LogicalVariable> visit(JoinClause joinClause, Mutable<ILogicalOperator> inputRef) throws CompilationException {
    Mutable<ILogicalOperator> leftInputRef = uncorrelatedLeftBranchStack.pop();
    if (joinClause.getJoinType() == JoinType.INNER) {
        Pair<ILogicalOperator, LogicalVariable> rightBranch = generateUnnestForBinaryCorrelateRightBranch(joinClause, inputRef, true);
        // A join operator with condition TRUE.
        AbstractBinaryJoinOperator joinOperator = new InnerJoinOperator(new MutableObject<ILogicalExpression>(ConstantExpression.TRUE), leftInputRef, new MutableObject<ILogicalOperator>(rightBranch.first));
        Mutable<ILogicalOperator> joinOpRef = new MutableObject<>(joinOperator);
        // Add an additional filter operator.
        Pair<ILogicalExpression, Mutable<ILogicalOperator>> conditionExprOpPair = langExprToAlgExpression(joinClause.getConditionExpression(), joinOpRef);
        SelectOperator filter = new SelectOperator(new MutableObject<ILogicalExpression>(conditionExprOpPair.first), false, null);
        filter.getInputs().add(conditionExprOpPair.second);
        return new Pair<>(filter, rightBranch.second);
    } else {
        // Creates a subplan operator.
        SubplanOperator subplanOp = new SubplanOperator();
        Mutable<ILogicalOperator> ntsRef = new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(subplanOp)));
        subplanOp.getInputs().add(leftInputRef);
        // Enters the translation for a subplan.
        context.enterSubplan();
        // Adds an unnest operator to unnest to right expression.
        Pair<ILogicalOperator, LogicalVariable> rightBranch = generateUnnestForBinaryCorrelateRightBranch(joinClause, ntsRef, true);
        AbstractUnnestNonMapOperator rightUnnestOp = (AbstractUnnestNonMapOperator) rightBranch.first;
        // Adds an additional filter operator for the join condition.
        Pair<ILogicalExpression, Mutable<ILogicalOperator>> conditionExprOpPair = langExprToAlgExpression(joinClause.getConditionExpression(), new MutableObject<ILogicalOperator>(rightUnnestOp));
        SelectOperator filter = new SelectOperator(new MutableObject<ILogicalExpression>(conditionExprOpPair.first), false, null);
        filter.getInputs().add(conditionExprOpPair.second);
        ILogicalOperator currentTopOp = filter;
        LogicalVariable varToListify;
        boolean hasRightPosVar = rightUnnestOp.getPositionalVariable() != null;
        if (hasRightPosVar) {
            // Creates record to get correlation between the two aggregate variables.
            ScalarFunctionCallExpression recordCreationFunc = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.CLOSED_RECORD_CONSTRUCTOR), // Field name for the listified right unnest var.
            new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AString("unnestvar")))), // The listified right unnest var
            new MutableObject<ILogicalExpression>(new VariableReferenceExpression(rightUnnestOp.getVariable())), // Field name for the listified right unnest positional var.
            new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AString("posvar")))), // The listified right unnest positional var.
            new MutableObject<ILogicalExpression>(new VariableReferenceExpression(rightUnnestOp.getPositionalVariable())));
            // Assigns the record constructor function to a record variable.
            LogicalVariable recordVar = context.newVar();
            AssignOperator assignOp = new AssignOperator(recordVar, new MutableObject<ILogicalExpression>(recordCreationFunc));
            assignOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTopOp));
            // Sets currentTopOp and varToListify for later usages.
            currentTopOp = assignOp;
            varToListify = recordVar;
        } else {
            varToListify = rightUnnestOp.getVariable();
        }
        // Adds an aggregate operator to listfy unnest variables.
        AggregateFunctionCallExpression fListify = BuiltinFunctions.makeAggregateFunctionExpression(BuiltinFunctions.LISTIFY, mkSingletonArrayList(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(varToListify))));
        LogicalVariable aggVar = context.newSubplanOutputVar();
        AggregateOperator aggOp = new AggregateOperator(mkSingletonArrayList(aggVar), mkSingletonArrayList(new MutableObject<ILogicalExpression>(fListify)));
        aggOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTopOp));
        // Exits the translation of a subplan.
        context.exitSubplan();
        // Sets the nested subplan of the subplan operator.
        ILogicalPlan subplan = new ALogicalPlanImpl(new MutableObject<ILogicalOperator>(aggOp));
        subplanOp.getNestedPlans().add(subplan);
        // Outer unnest the aggregated var from the subplan.
        LogicalVariable outerUnnestVar = context.newVar();
        LeftOuterUnnestOperator outerUnnestOp = new LeftOuterUnnestOperator(outerUnnestVar, new MutableObject<ILogicalExpression>(makeUnnestExpression(new VariableReferenceExpression(aggVar))));
        outerUnnestOp.getInputs().add(new MutableObject<ILogicalOperator>(subplanOp));
        currentTopOp = outerUnnestOp;
        if (hasRightPosVar) {
            ScalarFunctionCallExpression fieldAccessForRightUnnestVar = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.FIELD_ACCESS_BY_INDEX), new MutableObject<ILogicalExpression>(new VariableReferenceExpression(outerUnnestVar)), new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AInt32(0)))));
            ScalarFunctionCallExpression fieldAccessForRightPosVar = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.FIELD_ACCESS_BY_INDEX), new MutableObject<ILogicalExpression>(new VariableReferenceExpression(outerUnnestVar)), new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AInt32(1)))));
            // Creates variables for assign.
            LogicalVariable rightUnnestVar = context.newVar();
            LogicalVariable rightPosVar = context.newVar();
            // Relate the assigned variables to the variable expression in AST.
            context.setVar(joinClause.getRightVariable(), rightUnnestVar);
            context.setVar(joinClause.getPositionalVariable(), rightPosVar);
            // Varaibles to assign.
            List<LogicalVariable> assignVars = new ArrayList<>();
            assignVars.add(rightUnnestVar);
            assignVars.add(rightPosVar);
            // Expressions for assign.
            List<Mutable<ILogicalExpression>> assignExprs = new ArrayList<>();
            assignExprs.add(new MutableObject<ILogicalExpression>(fieldAccessForRightUnnestVar));
            assignExprs.add(new MutableObject<ILogicalExpression>(fieldAccessForRightPosVar));
            // Creates the assign operator.
            AssignOperator assignOp = new AssignOperator(assignVars, assignExprs);
            assignOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTopOp));
            currentTopOp = assignOp;
        } else {
            context.setVar(joinClause.getRightVariable(), outerUnnestVar);
        }
        return new Pair<>(currentTopOp, null);
    }
}

Example 13 with ALogicalPlanImpl

use of org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl in project asterixdb by apache.

the class LangExpressionToPlanTranslator method visit.

@Override
public Pair<ILogicalOperator, LogicalVariable> visit(GroupbyClause gc, Mutable<ILogicalOperator> tupSource) throws CompilationException {
    Mutable<ILogicalOperator> topOp = tupSource;
    if (gc.hasGroupVar()) {
        List<Pair<Expression, Identifier>> groupFieldList = gc.getGroupFieldList();
        List<Mutable<ILogicalExpression>> groupRecordConstructorArgList = new ArrayList<>();
        for (Pair<Expression, Identifier> groupField : groupFieldList) {
            ILogicalExpression groupFieldNameExpr = langExprToAlgExpression(new LiteralExpr(new StringLiteral(groupField.second.getValue())), topOp).first;
            groupRecordConstructorArgList.add(new MutableObject<>(groupFieldNameExpr));
            ILogicalExpression groupFieldExpr = langExprToAlgExpression(groupField.first, topOp).first;
            groupRecordConstructorArgList.add(new MutableObject<>(groupFieldExpr));
        }
        LogicalVariable groupVar = context.newVarFromExpression(gc.getGroupVar());
        AssignOperator groupVarAssignOp = new AssignOperator(groupVar, new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR), groupRecordConstructorArgList)));
        groupVarAssignOp.getInputs().add(topOp);
        topOp = new MutableObject<>(groupVarAssignOp);
    }
    GroupByOperator gOp = new GroupByOperator();
    for (GbyVariableExpressionPair ve : gc.getGbyPairList()) {
        VariableExpr vexpr = ve.getVar();
        LogicalVariable v = vexpr == null ? context.newVar() : context.newVarFromExpression(vexpr);
        Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = langExprToAlgExpression(ve.getExpr(), topOp);
        gOp.addGbyExpression(v, eo.first);
        topOp = eo.second;
    }
    for (GbyVariableExpressionPair ve : gc.getDecorPairList()) {
        VariableExpr vexpr = ve.getVar();
        LogicalVariable v = vexpr == null ? context.newVar() : context.newVarFromExpression(vexpr);
        Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = langExprToAlgExpression(ve.getExpr(), topOp);
        gOp.addDecorExpression(v, eo.first);
        topOp = eo.second;
    }
    gOp.getInputs().add(topOp);
    for (Entry<Expression, VariableExpr> entry : gc.getWithVarMap().entrySet()) {
        Pair<ILogicalExpression, Mutable<ILogicalOperator>> listifyInput = langExprToAlgExpression(entry.getKey(), new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(gOp))));
        List<Mutable<ILogicalExpression>> flArgs = new ArrayList<>(1);
        flArgs.add(new MutableObject<>(listifyInput.first));
        AggregateFunctionCallExpression fListify = BuiltinFunctions.makeAggregateFunctionExpression(BuiltinFunctions.LISTIFY, flArgs);
        LogicalVariable aggVar = context.newVar();
        AggregateOperator agg = new AggregateOperator(mkSingletonArrayList(aggVar), mkSingletonArrayList(new MutableObject<>(fListify)));
        agg.getInputs().add(listifyInput.second);
        ILogicalPlan plan = new ALogicalPlanImpl(new MutableObject<>(agg));
        gOp.getNestedPlans().add(plan);
        // Hide the variable that was part of the "with", replacing it with
        // the one bound by the aggregation op.
        context.setVar(entry.getValue(), aggVar);
    }
    gOp.setGroupAll(gc.isGroupAll());
    gOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, gc.hasHashGroupByHint());
    return new Pair<>(gOp, null);
}

Example 14 with ALogicalPlanImpl

use of org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl in project asterixdb by apache.

the class AbstractIntroduceGroupByCombinerRule method tryToPushSubplan.

private Pair<Boolean, ILogicalPlan> tryToPushSubplan(ILogicalPlan nestedPlan, GroupByOperator oldGbyOp, GroupByOperator newGbyOp, BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context) throws AlgebricksException {
    List<Mutable<ILogicalOperator>> pushedRoots = new ArrayList<Mutable<ILogicalOperator>>();
    Set<SimilarAggregatesInfo> toReplaceSet = new HashSet<SimilarAggregatesInfo>();
    for (Mutable<ILogicalOperator> r : nestedPlan.getRoots()) {
        if (!tryToPushRoot(r, oldGbyOp, newGbyOp, bi, gbyVars, context, pushedRoots, toReplaceSet)) {
            // For now, if we cannot push everything, give up.
            return new Pair<Boolean, ILogicalPlan>(false, null);
        }
    }
    if (pushedRoots.isEmpty()) {
        return new Pair<Boolean, ILogicalPlan>(true, null);
    } else {
        // Replaces the aggregation expressions in the original group-by op with new ones.
        ILogicalPlan newPlan = new ALogicalPlanImpl(pushedRoots);
        ILogicalPlan plan = fingIdenticalPlan(newGbyOp, newPlan);
        replaceOriginalAggFuncs(toReplaceSet);
        if (plan == null) {
            return new Pair<Boolean, ILogicalPlan>(true, newPlan);
        } else {
            // Does not add a nested subplan to newGbyOp if there already exists an isomorphic plan.
            Set<LogicalVariable> originalVars = new ListSet<LogicalVariable>();
            Set<LogicalVariable> newVars = new ListSet<LogicalVariable>();
            for (Mutable<ILogicalOperator> rootRef : pushedRoots) {
                VariableUtilities.getProducedVariables(rootRef.getValue(), originalVars);
            }
            for (Mutable<ILogicalOperator> rootRef : plan.getRoots()) {
                VariableUtilities.getProducedVariables(rootRef.getValue(), newVars);
            }
            // Replaces variable exprs referring to the variables produced by newPlan by
            // those produced by plan.
            Iterator<LogicalVariable> originalVarIter = originalVars.iterator();
            Iterator<LogicalVariable> newVarIter = newVars.iterator();
            while (originalVarIter.hasNext()) {
                LogicalVariable originalVar = originalVarIter.next();
                LogicalVariable newVar = newVarIter.next();
                for (SimilarAggregatesInfo sai : toReplaceSet) {
                    for (AggregateExprInfo aei : sai.simAggs) {
                        ILogicalExpression afce = aei.aggExprRef.getValue();
                        afce.substituteVar(originalVar, newVar);
                    }
                }
            }
            return new Pair<Boolean, ILogicalPlan>(true, null);
        }
    }
}

Example 15 with ALogicalPlanImpl

use of org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl in project asterixdb by apache.

the class LogicalOperatorDeepCopyWithNewVariablesVisitor method deepCopyPlan.

private ILogicalPlan deepCopyPlan(ILogicalPlan plan, ILogicalOperator arg) throws AlgebricksException {
    List<Mutable<ILogicalOperator>> rootsCopy = deepCopyOperatorReferenceList(plan.getRoots(), arg);
    ILogicalPlan planCopy = new ALogicalPlanImpl(rootsCopy);
    return planCopy;
}