Examples with GroupByOperator org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator used on opensource projects

Example 31 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class PushGroupByThroughProduct method rewritePost.

@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
    if (op1.getOperatorTag() != LogicalOperatorTag.GROUP) {
        return false;
    }
    Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
    AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
    if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
        return false;
    }
    InnerJoinOperator join = (InnerJoinOperator) op2;
    if (!OperatorPropertiesUtil.isAlwaysTrueCond(join.getCondition().getValue())) {
        // not a product
        return false;
    }
    GroupByOperator gby = (GroupByOperator) op1;
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorToPush = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorNotToPush = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
    Mutable<ILogicalOperator> opLeftRef = join.getInputs().get(0);
    ILogicalOperator opLeft = opLeftRef.getValue();
    switch(canPushThrough(gby, opLeft, decorToPush, decorNotToPush)) {
        case REPEATED_DECORS:
            {
                return false;
            }
        case TRUE:
            {
                push(opRef, opRef2, 0, decorToPush, decorNotToPush, context);
                return true;
            }
        case FALSE:
            {
                decorToPush.clear();
                Mutable<ILogicalOperator> opRightRef = join.getInputs().get(1);
                ILogicalOperator opRight = opRightRef.getValue();
                if (canPushThrough(gby, opRight, decorToPush, decorNotToPush) == PushTestResult.TRUE) {
                    push(opRef, opRef2, 1, decorToPush, decorNotToPush, context);
                    return true;
                } else {
                    return false;
                }
            }
        default:
            {
                throw new IllegalStateException();
            }
    }
}

Example 32 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class InlineSubplanInputForNestedTupleSourceRule method applySpecialFlattening.

private Pair<Boolean, LinkedHashMap<LogicalVariable, LogicalVariable>> applySpecialFlattening(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    SubplanOperator subplanOp = (SubplanOperator) opRef.getValue();
    Mutable<ILogicalOperator> inputOpRef = subplanOp.getInputs().get(0);
    LinkedHashMap<LogicalVariable, LogicalVariable> replacedVarMap = new LinkedHashMap<>();
    // Recursively applies this rule to the nested plan of the subplan operator,
    // for the case where there are nested subplan operators within {@code subplanOp}.
    Pair<Boolean, LinkedHashMap<LogicalVariable, LogicalVariable>> result = rewriteSubplanOperator(subplanOp.getNestedPlans().get(0).getRoots().get(0), context);
    ILogicalOperator inputOpBackup = inputOpRef.getValue();
    // Gets live variables and covering variables from the subplan's input operator.
    Pair<ILogicalOperator, Set<LogicalVariable>> primaryOpAndVars = EquivalenceClassUtils.findOrCreatePrimaryKeyOpAndVariables(inputOpBackup, false, context);
    ILogicalOperator inputOp = primaryOpAndVars.first;
    Set<LogicalVariable> primaryKeyVars = primaryOpAndVars.second;
    inputOpRef.setValue(inputOp);
    Set<LogicalVariable> liveVars = new HashSet<>();
    VariableUtilities.getLiveVariables(inputOp, liveVars);
    Pair<Set<LogicalVariable>, Mutable<ILogicalOperator>> notNullVarsAndTopJoinRef = SubplanFlatteningUtil.inlineLeftNtsInSubplanJoin(subplanOp, context);
    if (notNullVarsAndTopJoinRef.first == null) {
        inputOpRef.setValue(inputOpBackup);
        return new Pair<>(false, replacedVarMap);
    }
    Set<LogicalVariable> notNullVars = notNullVarsAndTopJoinRef.first;
    Mutable<ILogicalOperator> topJoinRef = notNullVarsAndTopJoinRef.second;
    // Creates a group-by operator.
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByDecorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
    GroupByOperator groupbyOp = new GroupByOperator(groupByList, groupByDecorList, subplanOp.getNestedPlans());
    for (LogicalVariable coverVar : primaryKeyVars) {
        LogicalVariable newVar = context.newVar();
        groupByList.add(new Pair<>(newVar, new MutableObject<>(new VariableReferenceExpression(coverVar))));
        // Adds variables for replacements in ancestors.
        replacedVarMap.put(coverVar, newVar);
    }
    for (LogicalVariable liveVar : liveVars) {
        if (primaryKeyVars.contains(liveVar)) {
            continue;
        }
        groupByDecorList.add(new Pair<>(null, new MutableObject<>(new VariableReferenceExpression(liveVar))));
    }
    groupbyOp.getInputs().add(new MutableObject<>(topJoinRef.getValue()));
    if (!notNullVars.isEmpty()) {
        // Adds a select operator into the nested plan for group-by to remove tuples with NULL on {@code assignVar}, i.e.,
        // subplan input tuples that are filtered out within a subplan.
        List<Mutable<ILogicalExpression>> nullCheckExprRefs = new ArrayList<>();
        for (LogicalVariable notNullVar : notNullVars) {
            Mutable<ILogicalExpression> filterVarExpr = new MutableObject<>(new VariableReferenceExpression(notNullVar));
            List<Mutable<ILogicalExpression>> args = new ArrayList<>();
            args.add(filterVarExpr);
            List<Mutable<ILogicalExpression>> argsForNotFunction = new ArrayList<>();
            argsForNotFunction.add(new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.IS_MISSING), args)));
            nullCheckExprRefs.add(new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.NOT), argsForNotFunction)));
        }
        Mutable<ILogicalExpression> selectExprRef = nullCheckExprRefs.size() > 1 ? new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.AND), nullCheckExprRefs)) : nullCheckExprRefs.get(0);
        SelectOperator selectOp = new SelectOperator(selectExprRef, false, null);
        topJoinRef.setValue(selectOp);
        selectOp.getInputs().add(new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(groupbyOp))));
    } else {
        // The original join operator in the Subplan is a left-outer join.
        // Therefore, no null-check variable is injected and no SelectOperator needs to be added.
        topJoinRef.setValue(new NestedTupleSourceOperator(new MutableObject<>(groupbyOp)));
    }
    opRef.setValue(groupbyOp);
    OperatorManipulationUtil.computeTypeEnvironmentBottomUp(groupbyOp, context);
    VariableUtilities.substituteVariables(groupbyOp, result.second, context);
    replacedVarMap.putAll(result.second);
    return new Pair<>(true, replacedVarMap);
}

Example 33 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class InlineSubplanInputForNestedTupleSourceRule method applyGeneralFlattening.

private Pair<Boolean, LinkedHashMap<LogicalVariable, LogicalVariable>> applyGeneralFlattening(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
    SubplanOperator subplanOp = (SubplanOperator) opRef.getValue();
    if (!SubplanFlatteningUtil.containsOperators(subplanOp, ImmutableSet.of(LogicalOperatorTag.DATASOURCESCAN, LogicalOperatorTag.INNERJOIN, // We don't have nested runtime for union-all and distinct hence we have to include them here.
    LogicalOperatorTag.LEFTOUTERJOIN, LogicalOperatorTag.UNIONALL, LogicalOperatorTag.DISTINCT))) {
        return new Pair<>(false, new LinkedHashMap<>());
    }
    Mutable<ILogicalOperator> inputOpRef = subplanOp.getInputs().get(0);
    ILogicalOperator inputOpBackup = inputOpRef.getValue();
    // Creates parameters for the left outer join operator.
    Pair<ILogicalOperator, Set<LogicalVariable>> primaryOpAndVars = EquivalenceClassUtils.findOrCreatePrimaryKeyOpAndVariables(inputOpBackup, true, context);
    ILogicalOperator inputOp = primaryOpAndVars.first;
    Set<LogicalVariable> primaryKeyVars = primaryOpAndVars.second;
    inputOpRef.setValue(inputOp);
    Set<LogicalVariable> inputLiveVars = new HashSet<>();
    VariableUtilities.getLiveVariables(inputOp, inputLiveVars);
    Pair<Map<LogicalVariable, LogicalVariable>, List<Pair<IOrder, Mutable<ILogicalExpression>>>> varMapAndOrderExprs = SubplanFlatteningUtil.inlineAllNestedTupleSource(subplanOp, context);
    Map<LogicalVariable, LogicalVariable> varMap = varMapAndOrderExprs.first;
    if (varMap == null) {
        inputOpRef.setValue(inputOpBackup);
        return new Pair<>(false, new LinkedHashMap<>());
    }
    Mutable<ILogicalOperator> lowestAggregateRefInSubplan = SubplanFlatteningUtil.findLowestAggregate(subplanOp.getNestedPlans().get(0).getRoots().get(0));
    Mutable<ILogicalOperator> rightInputOpRef = lowestAggregateRefInSubplan.getValue().getInputs().get(0);
    ILogicalOperator rightInputOp = rightInputOpRef.getValue();
    // Creates a variable to indicate whether a left input tuple is killed in the plan rooted at rightInputOp.
    LogicalVariable assignVar = context.newVar();
    ILogicalOperator assignOp = new AssignOperator(assignVar, new MutableObject<>(ConstantExpression.TRUE));
    assignOp.getInputs().add(rightInputOpRef);
    context.computeAndSetTypeEnvironmentForOperator(assignOp);
    rightInputOpRef = new MutableObject<>(assignOp);
    // Constructs the join predicate for the leftOuter join.
    List<Mutable<ILogicalExpression>> joinPredicates = new ArrayList<>();
    for (LogicalVariable liveVar : primaryKeyVars) {
        List<Mutable<ILogicalExpression>> arguments = new ArrayList<>();
        arguments.add(new MutableObject<>(new VariableReferenceExpression(liveVar)));
        LogicalVariable rightVar = varMap.get(liveVar);
        arguments.add(new MutableObject<>(new VariableReferenceExpression(rightVar)));
        ILogicalExpression expr = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AlgebricksBuiltinFunctions.EQ), arguments);
        joinPredicates.add(new MutableObject<>(expr));
    }
    ILogicalExpression joinExpr = joinPredicates.size() > 1 ? new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AlgebricksBuiltinFunctions.AND), joinPredicates) : joinPredicates.size() > 0 ? joinPredicates.get(0).getValue() : ConstantExpression.TRUE;
    LeftOuterJoinOperator leftOuterJoinOp = new LeftOuterJoinOperator(new MutableObject<>(joinExpr), inputOpRef, rightInputOpRef);
    OperatorManipulationUtil.computeTypeEnvironmentBottomUp(rightInputOp, context);
    context.computeAndSetTypeEnvironmentForOperator(leftOuterJoinOp);
    // Creates group-by operator.
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByDecorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
    List<ILogicalPlan> nestedPlans = new ArrayList<>();
    GroupByOperator groupbyOp = new GroupByOperator(groupByList, groupByDecorList, nestedPlans);
    LinkedHashMap<LogicalVariable, LogicalVariable> replacedVarMap = new LinkedHashMap<>();
    for (LogicalVariable liveVar : primaryKeyVars) {
        LogicalVariable newVar = context.newVar();
        groupByList.add(new Pair<>(newVar, new MutableObject<>(new VariableReferenceExpression(liveVar))));
        // Adds variables for replacements in ancestors.
        replacedVarMap.put(liveVar, newVar);
    }
    for (LogicalVariable liveVar : inputLiveVars) {
        if (primaryKeyVars.contains(liveVar)) {
            continue;
        }
        groupByDecorList.add(new Pair<>(null, new MutableObject<>(new VariableReferenceExpression(liveVar))));
    }
    // Sets up the nested plan for the groupby operator.
    Mutable<ILogicalOperator> aggOpRef = subplanOp.getNestedPlans().get(0).getRoots().get(0);
    // Clears the input of the lowest aggregate.
    lowestAggregateRefInSubplan.getValue().getInputs().clear();
    Mutable<ILogicalOperator> currentOpRef = lowestAggregateRefInSubplan;
    // Adds an optional order operator.
    List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExprs = varMapAndOrderExprs.second;
    if (!orderExprs.isEmpty()) {
        OrderOperator orderOp = new OrderOperator(orderExprs);
        currentOpRef = new MutableObject<>(orderOp);
        lowestAggregateRefInSubplan.getValue().getInputs().add(currentOpRef);
    }
    // Adds a select operator into the nested plan for group-by to remove tuples with NULL on {@code assignVar}, i.e.,
    // subplan input tuples that are filtered out within a subplan.
    Mutable<ILogicalExpression> filterVarExpr = new MutableObject<>(new VariableReferenceExpression(assignVar));
    List<Mutable<ILogicalExpression>> args = new ArrayList<>();
    args.add(filterVarExpr);
    List<Mutable<ILogicalExpression>> argsForNotFunction = new ArrayList<>();
    argsForNotFunction.add(new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.IS_MISSING), args)));
    SelectOperator selectOp = new SelectOperator(new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.NOT), argsForNotFunction)), false, null);
    currentOpRef.getValue().getInputs().add(new MutableObject<>(selectOp));
    selectOp.getInputs().add(new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(groupbyOp))));
    List<Mutable<ILogicalOperator>> nestedRoots = new ArrayList<>();
    nestedRoots.add(aggOpRef);
    nestedPlans.add(new ALogicalPlanImpl(nestedRoots));
    groupbyOp.getInputs().add(new MutableObject<>(leftOuterJoinOp));
    // Replaces subplan with the group-by operator.
    opRef.setValue(groupbyOp);
    OperatorManipulationUtil.computeTypeEnvironmentBottomUp(groupbyOp, context);
    // Recursively applys this rule to the nested plan of the subplan operator,
    // for the case where there are nested subplan operators within {@code subplanOp}.
    Pair<Boolean, LinkedHashMap<LogicalVariable, LogicalVariable>> result = rewriteSubplanOperator(rightInputOpRef, context);
    VariableUtilities.substituteVariables(leftOuterJoinOp, result.second, context);
    VariableUtilities.substituteVariables(groupbyOp, result.second, context);
    // No var mapping from the right input operator should be populated up.
    return new Pair<>(true, replacedVarMap);
}

Example 34 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class LangExpressionToPlanTranslator method visit.

@Override
public Pair<ILogicalOperator, LogicalVariable> visit(GroupbyClause gc, Mutable<ILogicalOperator> tupSource) throws CompilationException {
    Mutable<ILogicalOperator> topOp = tupSource;
    if (gc.hasGroupVar()) {
        List<Pair<Expression, Identifier>> groupFieldList = gc.getGroupFieldList();
        List<Mutable<ILogicalExpression>> groupRecordConstructorArgList = new ArrayList<>();
        for (Pair<Expression, Identifier> groupField : groupFieldList) {
            ILogicalExpression groupFieldNameExpr = langExprToAlgExpression(new LiteralExpr(new StringLiteral(groupField.second.getValue())), topOp).first;
            groupRecordConstructorArgList.add(new MutableObject<>(groupFieldNameExpr));
            ILogicalExpression groupFieldExpr = langExprToAlgExpression(groupField.first, topOp).first;
            groupRecordConstructorArgList.add(new MutableObject<>(groupFieldExpr));
        }
        LogicalVariable groupVar = context.newVarFromExpression(gc.getGroupVar());
        AssignOperator groupVarAssignOp = new AssignOperator(groupVar, new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR), groupRecordConstructorArgList)));
        groupVarAssignOp.getInputs().add(topOp);
        topOp = new MutableObject<>(groupVarAssignOp);
    }
    GroupByOperator gOp = new GroupByOperator();
    for (GbyVariableExpressionPair ve : gc.getGbyPairList()) {
        VariableExpr vexpr = ve.getVar();
        LogicalVariable v = vexpr == null ? context.newVar() : context.newVarFromExpression(vexpr);
        Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = langExprToAlgExpression(ve.getExpr(), topOp);
        gOp.addGbyExpression(v, eo.first);
        topOp = eo.second;
    }
    for (GbyVariableExpressionPair ve : gc.getDecorPairList()) {
        VariableExpr vexpr = ve.getVar();
        LogicalVariable v = vexpr == null ? context.newVar() : context.newVarFromExpression(vexpr);
        Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = langExprToAlgExpression(ve.getExpr(), topOp);
        gOp.addDecorExpression(v, eo.first);
        topOp = eo.second;
    }
    gOp.getInputs().add(topOp);
    for (Entry<Expression, VariableExpr> entry : gc.getWithVarMap().entrySet()) {
        Pair<ILogicalExpression, Mutable<ILogicalOperator>> listifyInput = langExprToAlgExpression(entry.getKey(), new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(gOp))));
        List<Mutable<ILogicalExpression>> flArgs = new ArrayList<>(1);
        flArgs.add(new MutableObject<>(listifyInput.first));
        AggregateFunctionCallExpression fListify = BuiltinFunctions.makeAggregateFunctionExpression(BuiltinFunctions.LISTIFY, flArgs);
        LogicalVariable aggVar = context.newVar();
        AggregateOperator agg = new AggregateOperator(mkSingletonArrayList(aggVar), mkSingletonArrayList(new MutableObject<>(fListify)));
        agg.getInputs().add(listifyInput.second);
        ILogicalPlan plan = new ALogicalPlanImpl(new MutableObject<>(agg));
        gOp.getNestedPlans().add(plan);
        // Hide the variable that was part of the "with", replacing it with
        // the one bound by the aggregation op.
        context.setVar(entry.getValue(), aggVar);
    }
    gOp.setGroupAll(gc.isGroupAll());
    gOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, gc.hasHashGroupByHint());
    return new Pair<>(gOp, null);
}

Example 35 with GroupByOperator

use of org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator in project asterixdb by apache.

the class RequiredCapacityVisitorTest method testUnPartitionedGroupBy.

@Test
public void testUnPartitionedGroupBy() throws AlgebricksException {
    IClusterCapacity clusterCapacity = new ClusterCapacity();
    RequiredCapacityVisitor visitor = makeComputationCapacityVisitor(PARALLELISM, clusterCapacity);
    // Constructs a parallel group-by query plan.
    GroupByOperator globalGby = makeGroupByOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    ExchangeOperator exchange = new ExchangeOperator();
    exchange.setPhysicalOperator(new OneToOneExchangePOperator());
    exchange.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    GroupByOperator localGby = makeGroupByOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    globalGby.getInputs().add(new MutableObject<>(exchange));
    exchange.getInputs().add(new MutableObject<>(localGby));
    // Verifies the calculated cluster capacity requirement for the test quer plan.
    globalGby.accept(visitor, null);
    Assert.assertTrue(clusterCapacity.getAggregatedCores() == 1);
    Assert.assertTrue(clusterCapacity.getAggregatedMemoryByteSize() == 2 * MEMORY_BUDGET + FRAME_SIZE);
}