Examples with WindowFunctionDef org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef used on opensource projects

Example 11 with WindowFunctionDef

use of org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef in project hive by apache.

the class WindowingTableFunction method setCanAcceptInputAsStream.

/*
   * (non-Javadoc)
   * 
   * @see
   * org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator#canAcceptInputAsStream
   * ()
   * 
   * WindowTableFunction supports streaming if all functions meet one of these
   * conditions: 1. The Function implements ISupportStreamingModeForWindowing 2.
   * Or returns a non null Object for the getWindowingEvaluator, that implements
   * ISupportStreamingModeForWindowing. 3. Is an invocation on a 'fixed' window.
   * So no Unbounded Preceding or Following.
   */
@SuppressWarnings("resource")
private int[] setCanAcceptInputAsStream(Configuration cfg) throws HiveException {
    canAcceptInputAsStream = false;
    if (ptfDesc.getLlInfo().getLeadLagExprs() != null) {
        return null;
    }
    WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef();
    int startPos = Integer.MAX_VALUE;
    int endPos = Integer.MIN_VALUE;
    for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) {
        WindowFunctionDef wFnDef = tabDef.getWindowFunctions().get(i);
        WindowFrameDef wdwFrame = wFnDef.getWindowFrame();
        GenericUDAFEvaluator fnEval = wFnDef.getWFnEval();
        boolean streamingPossible = streamingPossible(cfg, wFnDef);
        GenericUDAFEvaluator streamingEval = streamingPossible ? fnEval.getWindowingEvaluator(wdwFrame) : null;
        if (streamingEval != null && streamingEval instanceof ISupportStreamingModeForWindowing) {
            continue;
        }
        BoundaryDef start = wdwFrame.getStart();
        BoundaryDef end = wdwFrame.getEnd();
        if (wdwFrame.getWindowType() == WindowType.ROWS) {
            if (!end.isUnbounded() && !start.isUnbounded()) {
                startPos = Math.min(startPos, wdwFrame.getStart().getRelativeOffset());
                endPos = Math.max(endPos, wdwFrame.getEnd().getRelativeOffset());
                continue;
            }
        }
        return null;
    }
    int windowLimit = HiveConf.getIntVar(cfg, ConfVars.HIVEJOINCACHESIZE);
    if (windowLimit < (endPos - startPos + 1)) {
        return null;
    }
    canAcceptInputAsStream = true;
    return new int[] { startPos, endPos };
}

Example 12 with WindowFunctionDef

use of org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef in project hive by apache.

the class WindowingTableFunction method finishPartition.

/*
   * (non-Javadoc)
   * 
   * @see
   * org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator#finishPartition()
   * 
   * for fns that are not ISupportStreamingModeForWindowing give them the
   * remaining rows (rows whose span went beyond the end of the partition) for
   * rest of the functions invoke terminate.
   * 
   * while numOutputRows < numInputRows for each Fn that doesn't have enough o/p
   * invoke getNextObj if there is no O/p then flag this as an error.
   */
@Override
public List<Object> finishPartition() throws HiveException {
    /*
     * Once enough rows have been output, there is no need to generate more output.
     */
    if (streamingState.rankLimitReached()) {
        return null;
    }
    WindowTableFunctionDef tabDef = (WindowTableFunctionDef) getTableDef();
    for (int i = 0; i < tabDef.getWindowFunctions().size(); i++) {
        WindowFunctionDef wFn = tabDef.getWindowFunctions().get(i);
        GenericUDAFEvaluator fnEval = wFn.getWFnEval();
        int numRowsRemaining = wFn.getWindowFrame().getEnd().getRelativeOffset();
        if (fnEval != null && fnEval instanceof ISupportStreamingModeForWindowing) {
            fnEval.terminate(streamingState.aggBuffers[i]);
            WindowingFunctionInfoHelper wFnInfo = getWindowingFunctionInfoHelper(wFn.getName());
            if (!wFnInfo.isSupportsWindow()) {
                numRowsRemaining = ((ISupportStreamingModeForWindowing) fnEval).getRowsRemainingAfterTerminate();
            }
            if (numRowsRemaining != BoundarySpec.UNBOUNDED_AMOUNT) {
                while (numRowsRemaining > 0) {
                    Object out = ((ISupportStreamingModeForWindowing) fnEval).getNextResult(streamingState.aggBuffers[i]);
                    if (out != null) {
                        streamingState.fnOutputs[i].add(out == ISupportStreamingModeForWindowing.NULL_RESULT ? null : out);
                    }
                    numRowsRemaining--;
                }
            }
        } else {
            while (numRowsRemaining > 0) {
                int rowToProcess = streamingState.rollingPart.size() - numRowsRemaining;
                if (rowToProcess >= 0) {
                    Object out = evaluateWindowFunction(wFn, rowToProcess, streamingState.rollingPart);
                    streamingState.fnOutputs[i].add(out);
                }
                numRowsRemaining--;
            }
        }
    }
    List<Object> oRows = new ArrayList<Object>();
    while (!streamingState.rollingPart.processedAllRows() && !streamingState.rankLimitReached()) {
        boolean hasRow = streamingState.hasOutputRow();
        if (!hasRow && !streamingState.rankLimitReached()) {
            throw new HiveException("Internal Error: cannot generate all output rows for a Partition");
        }
        if (hasRow) {
            oRows.add(streamingState.nextOutputRow());
        }
    }
    return oRows.size() == 0 ? null : oRows;
}

Example 13 with WindowFunctionDef

use of org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef in project hive by apache.

the class Vectorizer method fillInPTFEvaluators.

private static void fillInPTFEvaluators(List<WindowFunctionDef> windowsFunctions, String[] evaluatorFunctionNames, WindowFrameDef[] evaluatorWindowFrameDefs, List<ExprNodeDesc>[] evaluatorInputExprNodeDescLists) throws HiveException {
    final int functionCount = windowsFunctions.size();
    for (int i = 0; i < functionCount; i++) {
        WindowFunctionDef winFunc = windowsFunctions.get(i);
        evaluatorFunctionNames[i] = winFunc.getName();
        evaluatorWindowFrameDefs[i] = winFunc.getWindowFrame();
        List<PTFExpressionDef> args = winFunc.getArgs();
        if (args != null) {
            List<ExprNodeDesc> exprNodeDescList = new ArrayList<ExprNodeDesc>();
            for (PTFExpressionDef arg : args) {
                exprNodeDescList.add(arg.getExprNode());
            }
            evaluatorInputExprNodeDescLists[i] = exprNodeDescList;
        }
    }
}