Examples with ArgDef com.sap.hadoop.windowing.query2.definition.ArgDef used on opensource projects

Example 1 with ArgDef

use of com.sap.hadoop.windowing.query2.definition.ArgDef in project SQLWindowing by hbutani.

the class WindowingTableFunction method executeFnwithWindow.

static ArrayList<Object> executeFnwithWindow(QueryDef qDef, WindowFunctionDef wFnDef, Partition iPart) throws HiveException, WindowingException {
    ArrayList<Object> vals = new ArrayList<Object>();
    GenericUDAFEvaluator fEval = wFnDef.getEvaluator();
    Object[] args = new Object[wFnDef.getArgs().size()];
    for (int i = 0; i < iPart.size(); i++) {
        AggregationBuffer aggBuffer = fEval.getNewAggregationBuffer();
        Range rng = getRange(wFnDef, i, iPart);
        PartitionIterator<Object> rItr = rng.iterator();
        RuntimeUtils.connectLeadLagFunctionsToPartition(qDef, rItr);
        while (rItr.hasNext()) {
            Object row = rItr.next();
            int j = 0;
            for (ArgDef arg : wFnDef.getArgs()) {
                args[j++] = arg.getExprEvaluator().evaluate(row);
            }
            fEval.aggregate(aggBuffer, args);
        }
        Object out = fEval.evaluate(aggBuffer);
        out = ObjectInspectorUtils.copyToStandardObject(out, wFnDef.getOI(), ObjectInspectorCopyOption.WRITABLE);
        vals.add(out);
    }
    return vals;
}

Example 2 with ArgDef

use of com.sap.hadoop.windowing.query2.definition.ArgDef in project SQLWindowing by hbutani.

the class WindowFunctionTranslation method setupEvaluator.

static void setupEvaluator(WindowFunctionDef wFnDef) throws WindowingException {
    try {
        WindowFunctionSpec wSpec = wFnDef.getSpec();
        ArrayList<ArgDef> args = wFnDef.getArgs();
        ArrayList<ObjectInspector> argOIs = getWritableObjectInspector(args);
        GenericUDAFEvaluator wFnEval = org.apache.hadoop.hive.ql.exec.FunctionRegistry.getGenericUDAFEvaluator(wSpec.getName(), argOIs, wSpec.isDistinct(), wSpec.isStar());
        ObjectInspector[] funcArgOIs = null;
        if (args != null) {
            funcArgOIs = new ObjectInspector[args.size()];
            int i = 0;
            for (ArgDef arg : args) {
                funcArgOIs[i++] = arg.getOI();
            }
        }
        ObjectInspector OI = wFnEval.init(GenericUDAFEvaluator.Mode.COMPLETE, funcArgOIs);
        wFnDef.setEvaluator(wFnEval);
        wFnDef.setOI(OI);
    } catch (HiveException he) {
        throw new WindowingException(he);
    }
}

Example 3 with ArgDef

use of com.sap.hadoop.windowing.query2.definition.ArgDef in project SQLWindowing by hbutani.

the class InputTranslation method translate.

/*
	 * <ol>
	 * <li> Get the <code>TableFunctionResolver</code> for this Function from the FunctionRegistry.
	 * <li> Create the TableFuncDef object.
	 * <li> Get the InputInfo for the input to this function.
	 * <li> Translate the Arguments to this Function in the Context of the InputInfo.
	 * <li> ask the  TableFunctionResolver to create a TableFunctionEvaluator based on the Args passed in.
	 * <li> ask the TableFunctionEvaluator to setup the Map-side ObjectInspector. Gives a chance to functions that 
	 * reshape the Input before it is partitioned to define the Shape after raw data is transformed.
	 * <li> Setup the Window Definition for this Function. The Window Definition is resolved wrt to the InputDef's
	 * Shape or the MapOI, for Functions that reshape the raw input.
	 * <li> ask the TableFunctionEvaluator to setup the Output ObjectInspector for this Function.
	 * <li> setup a Serde for the Output partition based on the OutputOI. 
	 * </ol> 
	 */
private static TableFuncDef translate(QueryDef qDef, TableFuncSpec tSpec, QueryInputDef inputDef) throws WindowingException {
    QueryTranslationInfo tInfo = qDef.getTranslationInfo();
    TableFunctionResolver tFn = FunctionRegistry.getTableFunctionResolver(tSpec.getName());
    if (tFn == null) {
        throw new WindowingException(sprintf("Unknown Table Function %s", tSpec.getName()));
    }
    TableFuncDef tDef = new TableFuncDef();
    tDef.setSpec(tSpec);
    tDef.setInput(inputDef);
    InputInfo iInfo = tInfo.getInputInfo(inputDef);
    /*
		 * translate args
		 */
    ArrayList<ASTNode> args = tSpec.getArgs();
    if (args != null) {
        for (ASTNode expr : args) {
            ArgDef argDef = translateTableFunctionArg(qDef, tDef, iInfo, expr);
            tDef.addArg(argDef);
        }
    }
    tFn.initialize(qDef, tDef);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    tDef.setFunction(tEval);
    tFn.setupRawInputOI();
    tDef.setWindow(WindowSpecTranslation.translateWindow(qDef, tDef));
    tFn.setupOutputOI();
    TranslateUtils.setupSerdeAndOI(tDef, inputDef, tInfo, tEval);
    return tDef;
}

Example 4 with ArgDef

use of com.sap.hadoop.windowing.query2.definition.ArgDef in project SQLWindowing by hbutani.

the class WindowingTableFunction method execute.

@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public void execute(PartitionIterator<Object> pItr, Partition outP) throws WindowingException {
    ArrayList<List<?>> oColumns = new ArrayList<List<?>>();
    Partition iPart = pItr.getPartition();
    StructObjectInspector inputOI;
    try {
        inputOI = (StructObjectInspector) iPart.getSerDe().getObjectInspector();
    } catch (SerDeException se) {
        throw new WindowingException(se);
    }
    try {
        for (WindowFunctionDef wFn : wFnDefs) {
            boolean processWindow = wFn.getWindow() != null;
            pItr.reset();
            if (!processWindow) {
                GenericUDAFEvaluator fEval = wFn.getEvaluator();
                Object[] args = new Object[wFn.getArgs().size()];
                AggregationBuffer aggBuffer = fEval.getNewAggregationBuffer();
                while (pItr.hasNext()) {
                    Object row = pItr.next();
                    int i = 0;
                    for (ArgDef arg : wFn.getArgs()) {
                        args[i++] = arg.getExprEvaluator().evaluate(row);
                    }
                    fEval.aggregate(aggBuffer, args);
                }
                Object out = fEval.evaluate(aggBuffer);
                WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFn.getSpec().getName());
                if (!wFnInfo.isPivotResult()) {
                    out = new SameList(iPart.size(), out);
                }
                oColumns.add((List<?>) out);
            } else {
                oColumns.add(executeFnwithWindow(getQueryDef(), wFn, iPart));
            }
        }
        for (int i = 0; i < iPart.size(); i++) {
            ArrayList oRow = new ArrayList();
            Object iRow = iPart.getAt(i);
            for (StructField f : inputOI.getAllStructFieldRefs()) {
                oRow.add(inputOI.getStructFieldData(iRow, f));
            }
            for (int j = 0; j < oColumns.size(); j++) {
                oRow.add(oColumns.get(j).get(i));
            }
            outP.append(oRow);
        }
    } catch (HiveException he) {
        throw new WindowingException(he);
    }
}

Example 5 with ArgDef

use of com.sap.hadoop.windowing.query2.definition.ArgDef in project SQLWindowing by hbutani.

the class TranslateUtils method buildArgDef.

public static ArgDef buildArgDef(QueryDef qDef, InputInfo iInfo, ASTNode arg) throws WindowingException {
    ArgDef argDef = new ArgDef();
    ExprNodeDesc exprNode = TranslateUtils.buildExprNode(arg, iInfo.getTypeCheckCtx());
    ExprNodeEvaluator exprEval = WindowingExprNodeEvaluatorFactory.get(qDef.getTranslationInfo(), exprNode);
    ObjectInspector oi = initExprNodeEvaluator(qDef, exprNode, exprEval, iInfo);
    argDef.setExpression(arg);
    argDef.setExprNode(exprNode);
    argDef.setExprEvaluator(exprEval);
    argDef.setOI(oi);
    return argDef;
}