Examples with GenericUDAFEvaluator org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator used on opensource projects

Example 11 with GenericUDAFEvaluator

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator in project SQLWindowing by hbutani.

the class WindowingTableFunction method executeFnwithWindow.

static ArrayList<Object> executeFnwithWindow(QueryDef qDef, WindowFunctionDef wFnDef, Partition iPart) throws HiveException, WindowingException {
    ArrayList<Object> vals = new ArrayList<Object>();
    GenericUDAFEvaluator fEval = wFnDef.getEvaluator();
    Object[] args = new Object[wFnDef.getArgs().size()];
    for (int i = 0; i < iPart.size(); i++) {
        AggregationBuffer aggBuffer = fEval.getNewAggregationBuffer();
        Range rng = getRange(wFnDef, i, iPart);
        PartitionIterator<Object> rItr = rng.iterator();
        RuntimeUtils.connectLeadLagFunctionsToPartition(qDef, rItr);
        while (rItr.hasNext()) {
            Object row = rItr.next();
            int j = 0;
            for (ArgDef arg : wFnDef.getArgs()) {
                args[j++] = arg.getExprEvaluator().evaluate(row);
            }
            fEval.aggregate(aggBuffer, args);
        }
        Object out = fEval.evaluate(aggBuffer);
        out = ObjectInspectorUtils.copyToStandardObject(out, wFnDef.getOI(), ObjectInspectorCopyOption.WRITABLE);
        vals.add(out);
    }
    return vals;
}

Example 12 with GenericUDAFEvaluator

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator in project SQLWindowing by hbutani.

the class WindowFunctionTranslation method setupEvaluator.

static void setupEvaluator(WindowFunctionDef wFnDef) throws WindowingException {
    try {
        WindowFunctionSpec wSpec = wFnDef.getSpec();
        ArrayList<ArgDef> args = wFnDef.getArgs();
        ArrayList<ObjectInspector> argOIs = getWritableObjectInspector(args);
        GenericUDAFEvaluator wFnEval = org.apache.hadoop.hive.ql.exec.FunctionRegistry.getGenericUDAFEvaluator(wSpec.getName(), argOIs, wSpec.isDistinct(), wSpec.isStar());
        ObjectInspector[] funcArgOIs = null;
        if (args != null) {
            funcArgOIs = new ObjectInspector[args.size()];
            int i = 0;
            for (ArgDef arg : args) {
                funcArgOIs[i++] = arg.getOI();
            }
        }
        ObjectInspector OI = wFnEval.init(GenericUDAFEvaluator.Mode.COMPLETE, funcArgOIs);
        wFnDef.setEvaluator(wFnEval);
        wFnDef.setOI(OI);
    } catch (HiveException he) {
        throw new WindowingException(he);
    }
}

Example 13 with GenericUDAFEvaluator

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator in project hive by apache.

the class GroupByDesc method isDistinctLike.

/**
   * Checks if this grouping is like distinct, which means that all non-distinct grouping
   * columns behave like they were distinct - for example min and max operators.
   */
public boolean isDistinctLike() {
    ArrayList<AggregationDesc> aggregators = getAggregators();
    for (AggregationDesc ad : aggregators) {
        if (!ad.getDistinct()) {
            GenericUDAFEvaluator udafEval = ad.getGenericUDAFEvaluator();
            UDFType annot = AnnotationUtils.getAnnotation(udafEval.getClass(), UDFType.class);
            if (annot == null || !annot.distinctLike()) {
                return false;
            }
        }
    }
    return true;
}

Example 14 with GenericUDAFEvaluator

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator in project hive by apache.

the class WindowingTableFunction method initializeStreaming.

@Override
public void initializeStreaming(Configuration cfg, StructObjectInspector inputOI, boolean isMapSide) throws HiveException {
    int[] span = setCanAcceptInputAsStream(cfg);
    if (!canAcceptInputAsStream) {
        return;
    }
    WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef();
    for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) {
        WindowFunctionDef wFnDef = tabDef.getWindowFunctions().get(i);
        WindowFrameDef wdwFrame = wFnDef.getWindowFrame();
        GenericUDAFEvaluator fnEval = wFnDef.getWFnEval();
        GenericUDAFEvaluator streamingEval = fnEval.getWindowingEvaluator(wdwFrame);
        if (streamingEval != null) {
            wFnDef.setWFnEval(streamingEval);
            if (wFnDef.isPivotResult()) {
                ListObjectInspector listOI = (ListObjectInspector) wFnDef.getOI();
                wFnDef.setOI(listOI.getListElementObjectInspector());
            }
        }
    }
    if (tabDef.getRankLimit() != -1) {
        rnkLimitDef = new RankLimit(tabDef.getRankLimit(), tabDef.getRankLimitFunction(), tabDef.getWindowFunctions());
    }
    streamingState = new StreamingState(cfg, inputOI, isMapSide, tabDef, span[0], span[1]);
}

Example 15 with GenericUDAFEvaluator

use of org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator in project hive by apache.

the class WindowingTableFunction method setCanAcceptInputAsStream.

/*
   * (non-Javadoc)
   * 
   * @see
   * org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator#canAcceptInputAsStream
   * ()
   * 
   * WindowTableFunction supports streaming if all functions meet one of these
   * conditions: 1. The Function implements ISupportStreamingModeForWindowing 2.
   * Or returns a non null Object for the getWindowingEvaluator, that implements
   * ISupportStreamingModeForWindowing. 3. Is an invocation on a 'fixed' window.
   * So no Unbounded Preceding or Following.
   */
@SuppressWarnings("resource")
private int[] setCanAcceptInputAsStream(Configuration cfg) throws HiveException {
    canAcceptInputAsStream = false;
    if (ptfDesc.getLlInfo().getLeadLagExprs() != null) {
        return null;
    }
    WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef();
    int startPos = Integer.MAX_VALUE;
    int endPos = Integer.MIN_VALUE;
    for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) {
        WindowFunctionDef wFnDef = tabDef.getWindowFunctions().get(i);
        WindowFrameDef wdwFrame = wFnDef.getWindowFrame();
        GenericUDAFEvaluator fnEval = wFnDef.getWFnEval();
        boolean streamingPossible = streamingPossible(cfg, wFnDef);
        GenericUDAFEvaluator streamingEval = streamingPossible ? fnEval.getWindowingEvaluator(wdwFrame) : null;
        if (streamingEval != null && streamingEval instanceof ISupportStreamingModeForWindowing) {
            continue;
        }
        BoundaryDef start = wdwFrame.getStart();
        BoundaryDef end = wdwFrame.getEnd();
        if (wdwFrame.getWindowType() == WindowType.ROWS) {
            if (!end.isUnbounded() && !start.isUnbounded()) {
                startPos = Math.min(startPos, wdwFrame.getStart().getRelativeOffset());
                endPos = Math.max(endPos, wdwFrame.getEnd().getRelativeOffset());
                continue;
            }
        }
        return null;
    }
    int windowLimit = HiveConf.getIntVar(cfg, ConfVars.HIVEJOINCACHESIZE);
    if (windowLimit < (endPos - startPos + 1)) {
        return null;
    }
    canAcceptInputAsStream = true;
    return new int[] { startPos, endPos };
}