Examples with AggregateFunctionCallExpression org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression used on opensource projects

Example 6 with AggregateFunctionCallExpression

use of org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression in project asterixdb by apache.

the class InlineAllNtsInSubplanVisitor method wrapLimitInGroupBy.

private Pair<ILogicalOperator, LogicalVariable> wrapLimitInGroupBy(ILogicalOperator op, LogicalVariable recordVar, Set<LogicalVariable> inputLiveVars) throws AlgebricksException {
    GroupByOperator gbyOp = new GroupByOperator();
    List<Pair<LogicalVariable, LogicalVariable>> keyVarNewVarPairs = new ArrayList<>();
    for (LogicalVariable keyVar : correlatedKeyVars) {
        // This limits the visitor can only be applied to a nested logical
        // plan inside a Subplan operator,
        // where the keyVarsToEnforce forms a candidate key which can
        // uniquely identify a tuple out of the nested-tuple-source.
        LogicalVariable newVar = context.newVar();
        gbyOp.getGroupByList().add(new Pair<>(newVar, new MutableObject<>(new VariableReferenceExpression(keyVar))));
        keyVarNewVarPairs.add(new Pair<>(keyVar, newVar));
    }
    // Creates an aggregate operator doing LISTIFY, as the root of the
    // nested plan of the added group-by operator.
    List<LogicalVariable> aggVarList = new ArrayList<LogicalVariable>();
    List<Mutable<ILogicalExpression>> aggExprList = new ArrayList<Mutable<ILogicalExpression>>();
    LogicalVariable aggVar = context.newVar();
    List<Mutable<ILogicalExpression>> aggArgList = new ArrayList<>();
    aggVarList.add(aggVar);
    // Creates an aggregation function expression.
    aggArgList.add(new MutableObject<>(new VariableReferenceExpression(recordVar)));
    ILogicalExpression aggExpr = new AggregateFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.LISTIFY), false, aggArgList);
    aggExprList.add(new MutableObject<>(aggExpr));
    AggregateOperator aggOp = new AggregateOperator(aggVarList, aggExprList);
    // Adds the original limit operator as the input operator to the added
    // aggregate operator.
    aggOp.getInputs().add(new MutableObject<>(op));
    op.getInputs().clear();
    ILogicalOperator currentOp = op;
    if (!orderingExprs.isEmpty()) {
        OrderOperator orderOp = new OrderOperator(cloneOrderingExpression(orderingExprs));
        op.getInputs().add(new MutableObject<>(orderOp));
        currentOp = orderOp;
    }
    // Adds a nested tuple source operator as the input operator to the
    // limit operator.
    NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(gbyOp));
    currentOp.getInputs().add(new MutableObject<>(nts));
    // Sets the root of the added nested plan to the aggregate operator.
    ILogicalPlan nestedPlan = new ALogicalPlanImpl();
    nestedPlan.getRoots().add(new MutableObject<>(aggOp));
    // Sets the nested plan for the added group-by operator.
    gbyOp.getNestedPlans().add(nestedPlan);
    // Updates variable mapping for ancestor operators.
    for (Pair<LogicalVariable, LogicalVariable> keyVarNewVar : keyVarNewVarPairs) {
        updateInputToOutputVarMapping(keyVarNewVar.first, keyVarNewVar.second, false);
    }
    return new Pair<>(gbyOp, aggVar);
}

Example 7 with AggregateFunctionCallExpression

use of org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression in project asterixdb by apache.

the class SetAsterixPhysicalOperatorsRule method computeDefaultPhysicalOp.

private static void computeDefaultPhysicalOp(AbstractLogicalOperator op, IOptimizationContext context) throws AlgebricksException {
    PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig();
    if (op.getOperatorTag().equals(LogicalOperatorTag.GROUP)) {
        GroupByOperator gby = (GroupByOperator) op;
        if (gby.getNestedPlans().size() == 1) {
            ILogicalPlan p0 = gby.getNestedPlans().get(0);
            if (p0.getRoots().size() == 1) {
                Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
                if (((AbstractLogicalOperator) (r0.getValue())).getOperatorTag().equals(LogicalOperatorTag.AGGREGATE)) {
                    AggregateOperator aggOp = (AggregateOperator) r0.getValue();
                    boolean serializable = true;
                    for (Mutable<ILogicalExpression> exprRef : aggOp.getExpressions()) {
                        AbstractFunctionCallExpression expr = (AbstractFunctionCallExpression) exprRef.getValue();
                        if (!BuiltinFunctions.isAggregateFunctionSerializable(expr.getFunctionIdentifier())) {
                            serializable = false;
                            break;
                        }
                    }
                    if ((gby.getAnnotations().get(OperatorAnnotations.USE_HASH_GROUP_BY) == Boolean.TRUE || gby.getAnnotations().get(OperatorAnnotations.USE_EXTERNAL_GROUP_BY) == Boolean.TRUE)) {
                        boolean setToExternalGby = false;
                        if (serializable) {
                            // if serializable, use external group-by
                            // now check whether the serialized version aggregation function has corresponding intermediate agg
                            boolean hasIntermediateAgg = true;
                            IMergeAggregationExpressionFactory mergeAggregationExpressionFactory = context.getMergeAggregationExpressionFactory();
                            List<LogicalVariable> originalVariables = aggOp.getVariables();
                            List<Mutable<ILogicalExpression>> aggExprs = aggOp.getExpressions();
                            int aggNum = aggExprs.size();
                            for (int i = 0; i < aggNum; i++) {
                                AbstractFunctionCallExpression expr = (AbstractFunctionCallExpression) aggExprs.get(i).getValue();
                                AggregateFunctionCallExpression serialAggExpr = BuiltinFunctions.makeSerializableAggregateFunctionExpression(expr.getFunctionIdentifier(), expr.getArguments());
                                if (mergeAggregationExpressionFactory.createMergeAggregation(originalVariables.get(i), serialAggExpr, context) == null) {
                                    hasIntermediateAgg = false;
                                    break;
                                }
                            }
                            // Check whether there are multiple aggregates in the sub plan.
                            // Currently, we don't support multiple aggregates in one external group-by.
                            boolean multipleAggOpsFound = false;
                            ILogicalOperator r1Logical = aggOp;
                            while (r1Logical.hasInputs()) {
                                r1Logical = r1Logical.getInputs().get(0).getValue();
                                if (r1Logical.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
                                    multipleAggOpsFound = true;
                                    break;
                                }
                            }
                            if (hasIntermediateAgg && !multipleAggOpsFound) {
                                for (int i = 0; i < aggNum; i++) {
                                    AbstractFunctionCallExpression expr = (AbstractFunctionCallExpression) aggExprs.get(i).getValue();
                                    AggregateFunctionCallExpression serialAggExpr = BuiltinFunctions.makeSerializableAggregateFunctionExpression(expr.getFunctionIdentifier(), expr.getArguments());
                                    aggOp.getExpressions().get(i).setValue(serialAggExpr);
                                }
                                ExternalGroupByPOperator externalGby = new ExternalGroupByPOperator(gby.getGroupByList(), physicalOptimizationConfig.getMaxFramesExternalGroupBy(), (long) physicalOptimizationConfig.getMaxFramesExternalGroupBy() * physicalOptimizationConfig.getFrameSize());
                                generateMergeAggregationExpressions(gby, context);
                                op.setPhysicalOperator(externalGby);
                                setToExternalGby = true;
                            }
                        }
                        if (!setToExternalGby) {
                            // if not serializable or no intermediate agg, use pre-clustered group-by
                            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList = gby.getGroupByList();
                            List<LogicalVariable> columnList = new ArrayList<LogicalVariable>(gbyList.size());
                            for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) {
                                ILogicalExpression expr = p.second.getValue();
                                if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                                    VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
                                    columnList.add(varRef.getVariableReference());
                                }
                            }
                            op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList, gby.isGroupAll()));
                        }
                    }
                } else if (((AbstractLogicalOperator) (r0.getValue())).getOperatorTag().equals(LogicalOperatorTag.RUNNINGAGGREGATE)) {
                    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList = gby.getGroupByList();
                    List<LogicalVariable> columnList = new ArrayList<LogicalVariable>(gbyList.size());
                    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) {
                        ILogicalExpression expr = p.second.getValue();
                        if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                            VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
                            columnList.add(varRef.getVariableReference());
                        }
                    }
                    op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList, gby.isGroupAll()));
                } else {
                    throw new AlgebricksException("Unsupported nested operator within a group-by: " + ((AbstractLogicalOperator) (r0.getValue())).getOperatorTag().name());
                }
            }
        }
    }
    if (op.getPhysicalOperator() == null) {
        switch(op.getOperatorTag()) {
            case INNERJOIN:
                {
                    JoinUtils.setJoinAlgorithmAndExchangeAlgo((InnerJoinOperator) op, context);
                    break;
                }
            case LEFTOUTERJOIN:
                {
                    JoinUtils.setJoinAlgorithmAndExchangeAlgo((LeftOuterJoinOperator) op, context);
                    break;
                }
            case UNNEST_MAP:
            case LEFT_OUTER_UNNEST_MAP:
                {
                    ILogicalExpression unnestExpr = null;
                    unnestExpr = ((AbstractUnnestMapOperator) op).getExpressionRef().getValue();
                    if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
                        AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
                        FunctionIdentifier fid = f.getFunctionIdentifier();
                        if (!fid.equals(BuiltinFunctions.INDEX_SEARCH)) {
                            throw new IllegalStateException();
                        }
                        AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
                        jobGenParams.readFromFuncArgs(f.getArguments());
                        MetadataProvider mp = (MetadataProvider) context.getMetadataProvider();
                        DataSourceId dataSourceId = new DataSourceId(jobGenParams.getDataverseName(), jobGenParams.getDatasetName());
                        Dataset dataset = mp.findDataset(jobGenParams.getDataverseName(), jobGenParams.getDatasetName());
                        IDataSourceIndex<String, DataSourceId> dsi = mp.findDataSourceIndex(jobGenParams.getIndexName(), dataSourceId);
                        INodeDomain storageDomain = mp.findNodeDomain(dataset.getNodeGroupName());
                        if (dsi == null) {
                            throw new AlgebricksException("Could not find index " + jobGenParams.getIndexName() + " for dataset " + dataSourceId);
                        }
                        IndexType indexType = jobGenParams.getIndexType();
                        boolean requiresBroadcast = jobGenParams.getRequiresBroadcast();
                        switch(indexType) {
                            case BTREE:
                                {
                                    BTreeJobGenParams btreeJobGenParams = new BTreeJobGenParams();
                                    btreeJobGenParams.readFromFuncArgs(f.getArguments());
                                    op.setPhysicalOperator(new BTreeSearchPOperator(dsi, storageDomain, requiresBroadcast, btreeJobGenParams.isPrimaryIndex(), btreeJobGenParams.isEqCondition(), btreeJobGenParams.getLowKeyVarList(), btreeJobGenParams.getHighKeyVarList()));
                                    break;
                                }
                            case RTREE:
                                {
                                    op.setPhysicalOperator(new RTreeSearchPOperator(dsi, storageDomain, requiresBroadcast));
                                    break;
                                }
                            case SINGLE_PARTITION_WORD_INVIX:
                            case SINGLE_PARTITION_NGRAM_INVIX:
                                {
                                    op.setPhysicalOperator(new InvertedIndexPOperator(dsi, storageDomain, requiresBroadcast, false));
                                    break;
                                }
                            case LENGTH_PARTITIONED_WORD_INVIX:
                            case LENGTH_PARTITIONED_NGRAM_INVIX:
                                {
                                    op.setPhysicalOperator(new InvertedIndexPOperator(dsi, storageDomain, requiresBroadcast, true));
                                    break;
                                }
                            default:
                                {
                                    throw new NotImplementedException(indexType + " indexes are not implemented.");
                                }
                        }
                    }
                    break;
                }
        }
    }
    if (op.hasNestedPlans()) {
        AbstractOperatorWithNestedPlans nested = (AbstractOperatorWithNestedPlans) op;
        for (ILogicalPlan p : nested.getNestedPlans()) {
            setPhysicalOperators(p, context);
        }
    }
    for (Mutable<ILogicalOperator> opRef : op.getInputs()) {
        computeDefaultPhysicalOp((AbstractLogicalOperator) opRef.getValue(), context);
    }
}

Example 8 with AggregateFunctionCallExpression

use of org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression in project asterixdb by apache.

the class QueryLogicalExpressionJobGen method createSerializableAggregateFunctionFactory.

@Override
public ISerializedAggregateEvaluatorFactory createSerializableAggregateFunctionFactory(AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException {
    IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context);
    IFunctionDescriptor fd = getFunctionDescriptor(expr, env, context);
    switch(fd.getFunctionDescriptorTag()) {
        case AGGREGATE:
            {
                if (BuiltinFunctions.isAggregateFunctionSerializable(fd.getIdentifier())) {
                    AggregateFunctionCallExpression serialAggExpr = BuiltinFunctions.makeSerializableAggregateFunctionExpression(fd.getIdentifier(), expr.getArguments());
                    IFunctionDescriptor afdd = getFunctionDescriptor(serialAggExpr, env, context);
                    return afdd.createSerializableAggregateEvaluatorFactory(args);
                } else {
                    throw new AlgebricksException("Trying to create a serializable aggregate from a non-serializable aggregate function descriptor. (fi=" + expr.getFunctionIdentifier() + ")");
                }
            }
        case SERIALAGGREGATE:
            {
                return fd.createSerializableAggregateEvaluatorFactory(args);
            }
        default:
            throw new IllegalStateException("Invalid function descriptor " + fd.getFunctionDescriptorTag() + " expected " + FunctionDescriptorTag.SERIALAGGREGATE + " or " + FunctionDescriptorTag.AGGREGATE);
    }
}

Example 9 with AggregateFunctionCallExpression

use of org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression in project asterixdb by apache.

the class PushAggregateIntoNestedSubplanRule method extractAggFunctionsFromExpression.

/**
     * @param exprRef
     * @param nspWithAgg
     * @param context
     * @return a pair whose first member is a boolean which is true iff
     *         something was changed in the expression tree rooted at expr. The
     *         second member is the result of transforming expr.
     * @throws AlgebricksException
     */
private Pair<Boolean, ILogicalExpression> extractAggFunctionsFromExpression(Mutable<ILogicalExpression> exprRef, Map<LogicalVariable, AbstractOperatorWithNestedPlans> nspWithAgg, Map<ILogicalExpression, ILogicalExpression> aggregateExprToVarExpr, IOptimizationContext context) throws AlgebricksException {
    ILogicalExpression expr = exprRef.getValue();
    switch(expr.getExpressionTag()) {
        case FUNCTION_CALL:
            AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr;
            FunctionIdentifier fi = BuiltinFunctions.getAggregateFunction(fce.getFunctionIdentifier());
            if (fi != null) {
                ILogicalExpression a1 = fce.getArguments().get(0).getValue();
                if (a1.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                    LogicalVariable argVar = ((VariableReferenceExpression) a1).getVariableReference();
                    AbstractOperatorWithNestedPlans nspOp = nspWithAgg.get(argVar);
                    if (nspOp != null) {
                        if (!aggregateExprToVarExpr.containsKey(expr)) {
                            LogicalVariable newVar = context.newVar();
                            AggregateFunctionCallExpression aggFun = BuiltinFunctions.makeAggregateFunctionExpression(fi, fce.getArguments());
                            rewriteAggregateInNestedSubplan(argVar, nspOp, aggFun, newVar, context);
                            ILogicalExpression newVarExpr = new VariableReferenceExpression(newVar);
                            aggregateExprToVarExpr.put(expr, newVarExpr);
                            return new Pair<>(Boolean.TRUE, newVarExpr);
                        } else {
                            ILogicalExpression varExpr = aggregateExprToVarExpr.get(expr);
                            return new Pair<>(Boolean.TRUE, varExpr);
                        }
                    }
                }
            }
            boolean change = false;
            for (Mutable<ILogicalExpression> a : fce.getArguments()) {
                Pair<Boolean, ILogicalExpression> aggArg = extractAggFunctionsFromExpression(a, nspWithAgg, aggregateExprToVarExpr, context);
                if (aggArg.first.booleanValue()) {
                    a.setValue(aggArg.second);
                    change = true;
                }
            }
            return new Pair<>(change, fce);
        case VARIABLE:
        case CONSTANT:
            return new Pair<>(Boolean.FALSE, expr);
        default:
            throw new IllegalArgumentException();
    }
}

Example 10 with AggregateFunctionCallExpression

use of org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression in project asterixdb by apache.

the class PushAggregateIntoNestedSubplanRule method rewriteAggregateInNestedSubplan.

private void rewriteAggregateInNestedSubplan(LogicalVariable oldAggVar, AbstractOperatorWithNestedPlans nspOp, AggregateFunctionCallExpression aggFun, LogicalVariable newAggVar, IOptimizationContext context) throws AlgebricksException {
    for (int j = 0; j < nspOp.getNestedPlans().size(); j++) {
        AggregateOperator aggOp = (AggregateOperator) nspOp.getNestedPlans().get(j).getRoots().get(0).getValue();
        int n = aggOp.getVariables().size();
        for (int i = 0; i < n; i++) {
            LogicalVariable v = aggOp.getVariables().get(i);
            if (v.equals(oldAggVar)) {
                AbstractFunctionCallExpression oldAggExpr = (AbstractFunctionCallExpression) aggOp.getExpressions().get(i).getValue();
                AggregateFunctionCallExpression newAggFun = BuiltinFunctions.makeAggregateFunctionExpression(aggFun.getFunctionIdentifier(), new ArrayList<>());
                for (Mutable<ILogicalExpression> arg : oldAggExpr.getArguments()) {
                    ILogicalExpression cloned = arg.getValue().cloneExpression();
                    newAggFun.getArguments().add(new MutableObject<>(cloned));
                }
                aggOp.getVariables().add(newAggVar);
                aggOp.getExpressions().add(new MutableObject<>(newAggFun));
                context.computeAndSetTypeEnvironmentForOperator(aggOp);
                break;
            }
        }
    }
}