Examples with GenericUDAFEvaluator org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator used on opensource projects

Example 16 with GenericUDAFEvaluator

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator in project hive by apache.

the class WindowingTableFunction method finishPartition.

/*
   * (non-Javadoc)
   * 
   * @see
   * org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator#finishPartition()
   * 
   * for fns that are not ISupportStreamingModeForWindowing give them the
   * remaining rows (rows whose span went beyond the end of the partition) for
   * rest of the functions invoke terminate.
   * 
   * while numOutputRows < numInputRows for each Fn that doesn't have enough o/p
   * invoke getNextObj if there is no O/p then flag this as an error.
   */
@Override
public List<Object> finishPartition() throws HiveException {
    /*
     * Once enough rows have been output, there is no need to generate more output.
     */
    if (streamingState.rankLimitReached()) {
        return null;
    }
    WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef();
    for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) {
        WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i);
        GenericUDAFEvaluator fnEval = wFn.getWFnEval();
        int numRowsRemaining = wFn.getWindowFrame().getEnd().getRelativeOffset();
        if (fnEval != null && fnEval instanceof ISupportStreamingModeForWindowing) {
            fnEval.terminate(streamingState.aggBuffers[i]);
            WindowingFunctionInfoHelper wFnInfo = getWindowingFunctionInfoHelper(wFn.getName());
            if (!wFnInfo.isSupportsWindow()) {
                numRowsRemaining = ((ISupportStreamingModeForWindowing) fnEval).getRowsRemainingAfterTerminate();
            }
            if (numRowsRemaining != BoundarySpec.UNBOUNDED_AMOUNT) {
                while (numRowsRemaining > 0) {
                    Object out = ((ISupportStreamingModeForWindowing) fnEval).getNextResult(streamingState.aggBuffers[i]);
                    if (out != null) {
                        streamingState.fnOutputs[i].add(out == ISupportStreamingModeForWindowing.NULL_RESULT ? null : out);
                    }
                    numRowsRemaining--;
                }
            }
        } else {
            while (numRowsRemaining > 0) {
                int rowToProcess = streamingState.rollingPart.size() - numRowsRemaining;
                if (rowToProcess >= 0) {
                    Object out = evaluateWindowFunction(wFn, rowToProcess, streamingState.rollingPart);
                    streamingState.fnOutputs[i].add(out);
                }
                numRowsRemaining--;
            }
        }
    }
    List<Object> oRows = new ArrayList<Object>();
    while (!streamingState.rollingPart.processedAllRows() && !streamingState.rankLimitReached()) {
        boolean hasRow = streamingState.hasOutputRow();
        if (!hasRow && !streamingState.rankLimitReached()) {
            throw new HiveException("Internal Error: cannot generate all output rows for a Partition");
        }
        if (hasRow) {
            oRows.add(streamingState.nextOutputRow());
        }
    }
    return oRows.size() == 0 ? null : oRows;
}

Example 17 with GenericUDAFEvaluator

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator in project hive by apache.

the class Registry method getGenericUDAFEvaluator.

/**
   * Get the GenericUDAF evaluator for the name and argumentClasses.
   *
   * @param name         the name of the UDAF
   * @param argumentOIs
   * @param isDistinct
   * @param isAllColumns
   * @return The UDAF evaluator
   */
@SuppressWarnings("deprecation")
public GenericUDAFEvaluator getGenericUDAFEvaluator(String name, List<ObjectInspector> argumentOIs, boolean isWindowing, boolean isDistinct, boolean isAllColumns) throws SemanticException {
    GenericUDAFResolver udafResolver = getGenericUDAFResolver(name);
    if (udafResolver == null) {
        return null;
    }
    GenericUDAFEvaluator udafEvaluator;
    ObjectInspector[] args = new ObjectInspector[argumentOIs.size()];
    // generics + arrays.
    for (int ii = 0; ii < argumentOIs.size(); ++ii) {
        args[ii] = argumentOIs.get(ii);
    }
    GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo(args, isWindowing, isDistinct, isAllColumns);
    if (udafResolver instanceof GenericUDAFResolver2) {
        udafEvaluator = ((GenericUDAFResolver2) udafResolver).getEvaluator(paramInfo);
    } else {
        udafEvaluator = udafResolver.getEvaluator(paramInfo.getParameters());
    }
    return udafEvaluator;
}

Example 18 with GenericUDAFEvaluator

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator in project hive by apache.

the class TestStreamingSum method _agg.

public static <T, TW> void _agg(GenericUDAFResolver fnR, TypeInfo[] inputTypes, Iterator<T> inVals, TypeHandler<T, TW> typeHandler, TW[] in, ObjectInspector[] inputOIs, int inSz, int numPreceding, int numFollowing, Iterator<T> outVals) throws HiveException {
    GenericUDAFEvaluator fn = fnR.getEvaluator(inputTypes);
    fn.init(Mode.COMPLETE, inputOIs);
    fn = fn.getWindowingEvaluator(wdwFrame(numPreceding, numFollowing));
    AggregationBuffer agg = fn.getNewAggregationBuffer();
    ISupportStreamingModeForWindowing oS = (ISupportStreamingModeForWindowing) fn;
    int outSz = 0;
    while (inVals.hasNext()) {
        typeHandler.set(inVals.next(), in[0]);
        fn.aggregate(agg, in);
        Object out = oS.getNextResult(agg);
        if (out != null) {
            if (out == ISupportStreamingModeForWindowing.NULL_RESULT) {
                out = null;
            } else {
                try {
                    out = typeHandler.get((TW) out);
                } catch (ClassCastException ce) {
                }
            }
            Assert.assertEquals(out, outVals.next());
            outSz++;
        }
    }
    fn.terminate(agg);
    while (outSz < inSz) {
        Object out = oS.getNextResult(agg);
        if (out == ISupportStreamingModeForWindowing.NULL_RESULT) {
            out = null;
        } else {
            try {
                out = typeHandler.get((TW) out);
            } catch (ClassCastException ce) {
            }
        }
        Assert.assertEquals(out, outVals.next());
        outSz++;
    }
}

Example 19 with GenericUDAFEvaluator

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator in project hive by apache.

the class PTFTranslator method setupWdwFnEvaluator.

static void setupWdwFnEvaluator(WindowFunctionDef def) throws HiveException {
    List<PTFExpressionDef> args = def.getArgs();
    List<ObjectInspector> argOIs = new ArrayList<ObjectInspector>();
    ObjectInspector[] funcArgOIs = null;
    if (args != null) {
        for (PTFExpressionDef arg : args) {
            argOIs.add(arg.getOI());
        }
        funcArgOIs = new ObjectInspector[args.size()];
        funcArgOIs = argOIs.toArray(funcArgOIs);
    }
    GenericUDAFEvaluator wFnEval = FunctionRegistry.getGenericWindowingEvaluator(def.getName(), argOIs, def.isDistinct(), def.isStar());
    ObjectInspector OI = wFnEval.init(GenericUDAFEvaluator.Mode.COMPLETE, funcArgOIs);
    def.setWFnEval(wFnEval);
    def.setOI(OI);
}

Example 20 with GenericUDAFEvaluator

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator in project hive by apache.

the class SemanticAnalyzer method getGenericUDAFEvaluator.

/**
   * Returns the GenericUDAFEvaluator for the aggregation. This is called once
   * for each GroupBy aggregation.
   */
public static GenericUDAFEvaluator getGenericUDAFEvaluator(String aggName, ArrayList<ExprNodeDesc> aggParameters, ASTNode aggTree, boolean isDistinct, boolean isAllColumns) throws SemanticException {
    ArrayList<ObjectInspector> originalParameterTypeInfos = getWritableObjectInspector(aggParameters);
    GenericUDAFEvaluator result = FunctionRegistry.getGenericUDAFEvaluator(aggName, originalParameterTypeInfos, isDistinct, isAllColumns);
    if (null == result) {
        String reason = "Looking for UDAF Evaluator\"" + aggName + "\" with parameters " + originalParameterTypeInfos;
        throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((ASTNode) aggTree.getChild(0), reason));
    }
    return result;
}