Search in sources :

Example 1 with WindowTableFunctionDef

use of org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef in project hive by apache.

the class WindowingTableFunction method execute.

@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public void execute(PTFPartitionIterator<Object> pItr, PTFPartition outP) throws HiveException {
    ArrayList<List<?>> oColumns = new ArrayList<List<?>>();
    PTFPartition iPart = pItr.getPartition();
    StructObjectInspector inputOI = iPart.getOutputOI();
    WindowTableFunctionDef wTFnDef = (WindowTableFunctionDef) getTableDef();
    for (WindowFunctionDef wFn : wTFnDef.getWindowFunctions()) {
        boolean processWindow = processWindow(wFn.getWindowFrame());
        pItr.reset();
        if (!processWindow) {
            Object out = evaluateFunctionOnPartition(wFn, iPart);
            if (!wFn.isPivotResult()) {
                out = new SameList(iPart.size(), out);
            }
            oColumns.add((List<?>) out);
        } else {
            oColumns.add(executeFnwithWindow(wFn, iPart));
        }
    }
    for (int i = 0; i < iPart.size(); i++) {
        ArrayList oRow = new ArrayList();
        Object iRow = iPart.getAt(i);
        for (int j = 0; j < oColumns.size(); j++) {
            oRow.add(oColumns.get(j).get(i));
        }
        for (StructField f : inputOI.getAllStructFieldRefs()) {
            oRow.add(inputOI.getStructFieldData(iRow, f));
        }
        outP.append(oRow);
    }
}
Also used : StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField) PTFPartition(org.apache.hadoop.hive.ql.exec.PTFPartition) ArrayList(java.util.ArrayList) WindowTableFunctionDef(org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef) AbstractList(java.util.AbstractList) ArrayList(java.util.ArrayList) List(java.util.List) WindowFunctionDef(org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector)

Example 2 with WindowTableFunctionDef

use of org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef in project hive by apache.

the class WindowingTableFunction method initializeWindowingFunctionInfoHelpers.

private void initializeWindowingFunctionInfoHelpers() throws SemanticException {
    // to the object during the map/reduce tasks.
    if (windowingFunctionHelpers != null) {
        return;
    }
    windowingFunctionHelpers = new HashMap<String, WindowingFunctionInfoHelper>();
    WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef();
    for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) {
        WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i);
        WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFn.getName());
        boolean supportsWindow = wFnInfo.isSupportsWindow();
        windowingFunctionHelpers.put(wFn.getName(), new WindowingFunctionInfoHelper(supportsWindow));
    }
}
Also used : WindowFunctionInfo(org.apache.hadoop.hive.ql.exec.WindowFunctionInfo) WindowTableFunctionDef(org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef) WindowFunctionDef(org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef)

Example 3 with WindowTableFunctionDef

use of org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef in project hive by apache.

the class WindowingTableFunction method processRow.

/*
   * (non-Javadoc)
   * 
   * @see
   * org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator#processRow(java
   * .lang.Object)
   * 
   * - hand row to each Function, provided there are enough rows for Function's
   * window. - call getNextObject on each Function. - output as many rows as
   * possible, based on minimum sz of Output List
   */
@Override
public List<Object> processRow(Object row) throws HiveException {
    /*
     * Once enough rows have been output, there is no need to process input rows.
     */
    if (streamingState.rankLimitReached()) {
        return null;
    }
    streamingState.rollingPart.append(row);
    WindowTableFunctionDef tabDef = (WindowTableFunctionDef) tableDef;
    for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) {
        WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i);
        GenericUDAFEvaluator fnEval = wFn.getWFnEval();
        int a = 0;
        if (wFn.getArgs() != null) {
            for (PTFExpressionDef arg : wFn.getArgs()) {
                streamingState.funcArgs[i][a++] = arg.getExprEvaluator().evaluate(row);
            }
        }
        if (fnEval != null && fnEval instanceof ISupportStreamingModeForWindowing) {
            fnEval.aggregate(streamingState.aggBuffers[i], streamingState.funcArgs[i]);
            Object out = ((ISupportStreamingModeForWindowing) fnEval).getNextResult(streamingState.aggBuffers[i]);
            if (out != null) {
                streamingState.fnOutputs[i].add(out == ISupportStreamingModeForWindowing.NULL_RESULT ? null : out);
            }
        } else {
            int rowToProcess = streamingState.rollingPart.rowToProcess(wFn.getWindowFrame());
            if (rowToProcess >= 0) {
                Object out = evaluateWindowFunction(wFn, rowToProcess, streamingState.rollingPart);
                streamingState.fnOutputs[i].add(out);
            }
        }
    }
    List<Object> oRows = new ArrayList<Object>();
    while (true) {
        boolean hasRow = streamingState.hasOutputRow();
        if (!hasRow) {
            break;
        }
        oRows.add(streamingState.nextOutputRow());
    }
    return oRows.size() == 0 ? null : oRows;
}
Also used : ISupportStreamingModeForWindowing(org.apache.hadoop.hive.ql.udf.generic.ISupportStreamingModeForWindowing) GenericUDAFEvaluator(org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator) WindowTableFunctionDef(org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef) ArrayList(java.util.ArrayList) PTFExpressionDef(org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef) WindowFunctionDef(org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef)

Example 4 with WindowTableFunctionDef

use of org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef in project hive by apache.

the class WindowingTableFunction method iterator.

@SuppressWarnings("rawtypes")
@Override
public Iterator<Object> iterator(PTFPartitionIterator<Object> pItr) throws HiveException {
    WindowTableFunctionDef wTFnDef = (WindowTableFunctionDef) getTableDef();
    ArrayList<Object> output = new ArrayList<Object>();
    List<?>[] outputFromPivotFunctions = new List<?>[wTFnDef.getWindowFunctions().size()];
    ArrayList<Integer> wFnsWithWindows = new ArrayList<Integer>();
    PTFPartition iPart = pItr.getPartition();
    int i = 0;
    for (WindowFunctionDef wFn : wTFnDef.getWindowFunctions()) {
        boolean processWindow = processWindow(wFn.getWindowFrame());
        pItr.reset();
        if (!processWindow && !wFn.isPivotResult()) {
            Object out = evaluateFunctionOnPartition(wFn, iPart);
            output.add(out);
        } else if (wFn.isPivotResult()) {
            GenericUDAFEvaluator streamingEval = wFn.getWFnEval().getWindowingEvaluator(wFn.getWindowFrame());
            if (streamingEval != null && streamingEval instanceof ISupportStreamingModeForWindowing) {
                ISupportStreamingModeForWindowing strEval = (ISupportStreamingModeForWindowing) streamingEval;
                if (strEval.getRowsRemainingAfterTerminate() == 0) {
                    wFn.setWFnEval(streamingEval);
                    if (wFn.getOI() instanceof ListObjectInspector) {
                        ListObjectInspector listOI = (ListObjectInspector) wFn.getOI();
                        wFn.setOI(listOI.getListElementObjectInspector());
                    }
                    output.add(null);
                    wFnsWithWindows.add(i);
                } else {
                    outputFromPivotFunctions[i] = (List) evaluateFunctionOnPartition(wFn, iPart);
                    output.add(null);
                }
            } else {
                outputFromPivotFunctions[i] = (List) evaluateFunctionOnPartition(wFn, iPart);
                output.add(null);
            }
        } else {
            output.add(null);
            wFnsWithWindows.add(i);
        }
        i++;
    }
    for (i = 0; i < iPart.getOutputOI().getAllStructFieldRefs().size(); i++) {
        output.add(null);
    }
    if (wTFnDef.getRankLimit() != -1) {
        rnkLimitDef = new RankLimit(wTFnDef.getRankLimit(), wTFnDef.getRankLimitFunction(), wTFnDef.getWindowFunctions());
    }
    return new WindowingIterator(iPart, output, outputFromPivotFunctions, ArrayUtils.toPrimitive(wFnsWithWindows.toArray(new Integer[wFnsWithWindows.size()])));
}
Also used : ISupportStreamingModeForWindowing(org.apache.hadoop.hive.ql.udf.generic.ISupportStreamingModeForWindowing) GenericUDAFEvaluator(org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator) PTFPartition(org.apache.hadoop.hive.ql.exec.PTFPartition) WindowTableFunctionDef(org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef) ArrayList(java.util.ArrayList) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) AbstractList(java.util.AbstractList) ArrayList(java.util.ArrayList) List(java.util.List) WindowFunctionDef(org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef)

Example 5 with WindowTableFunctionDef

use of org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef in project hive by apache.

the class WindowingTableFunction method startPartition.

/*
   * (non-Javadoc)
   * 
   * @see
   * org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator#startPartition()
   */
@Override
public void startPartition() throws HiveException {
    WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef();
    streamingState.reset(tabDef);
}
Also used : WindowTableFunctionDef(org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef)

Aggregations

WindowTableFunctionDef (org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef)10 WindowFunctionDef (org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef)8 ArrayList (java.util.ArrayList)5 GenericUDAFEvaluator (org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator)5 ISupportStreamingModeForWindowing (org.apache.hadoop.hive.ql.udf.generic.ISupportStreamingModeForWindowing)4 ListObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector)3 AbstractList (java.util.AbstractList)2 List (java.util.List)2 PTFPartition (org.apache.hadoop.hive.ql.exec.PTFPartition)2 WindowFrameDef (org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef)2 StructObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector)2 ArrayDeque (java.util.ArrayDeque)1 WindowFunctionInfo (org.apache.hadoop.hive.ql.exec.WindowFunctionInfo)1 HiveException (org.apache.hadoop.hive.ql.metadata.HiveException)1 PTFQueryInputSpec (org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputSpec)1 PartitioningSpec (org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitioningSpec)1 WindowExpressionSpec (org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowExpressionSpec)1 WindowFunctionSpec (org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec)1 PTFDesc (org.apache.hadoop.hive.ql.plan.PTFDesc)1 BoundaryDef (org.apache.hadoop.hive.ql.plan.ptf.BoundaryDef)1