Examples with TableFuncDef com.sap.hadoop.windowing.query2.definition.TableFuncDef used on opensource projects

Example 11 with TableFuncDef

use of com.sap.hadoop.windowing.query2.definition.TableFuncDef in project SQLWindowing by hbutani.

the class PTFOperator method initializeOp.

/*
	 * 1. Find out if the operator is invoked at Map-Side or Reduce-side 
	 * 2. Get the deserialized QueryDef 
	 * 3. Reconstruct the transient variables in QueryDef 
	 * 4. Create input partition to store rows coming from previous operator
	 */
@Override
protected void initializeOp(Configuration jobConf) throws HiveException {
    hiveConf = new HiveConf(jobConf, PTFOperator.class);
    // if the parent is ExtractOperator, this invocation is from reduce-side
    Operator<? extends OperatorDesc> parentOp = getParentOperators().get(0);
    if (parentOp instanceof ExtractOperator) {
        isMapOperator = false;
    } else {
        isMapOperator = true;
    }
    // use the string from PTFDesc to get deserialized QueryDef
    qDef = (QueryDef) SerializationUtils.deserialize(new ByteArrayInputStream(conf.getQueryDefStr().getBytes()));
    try {
        reconstructQueryDef(hiveConf);
        inputPart = RuntimeUtils.createFirstPartitionForChain(qDef, inputObjInspectors[0], hiveConf, isMapOperator);
    } catch (WindowingException we) {
        throw new HiveException("Cannot create input partition for PTFOperator.", we);
    }
    // OI for ReduceSinkOperator is taken from TODO
    if (isMapOperator) {
        TableFuncDef tDef = RuntimeUtils.getFirstTableFunction(qDef);
        outputObjInspector = tDef.getMapOI();
    } else {
        outputObjInspector = qDef.getSelectList().getOI();
    }
    setupKeysWrapper(inputObjInspectors[0]);
    super.initializeOp(jobConf);
}

Example 12 with TableFuncDef

use of com.sap.hadoop.windowing.query2.definition.TableFuncDef in project SQLWindowing by hbutani.

the class PTFOperator method processMapFunction.

protected void processMapFunction() throws HiveException {
    try {
        TableFuncDef tDef = RuntimeUtils.getFirstTableFunction(qDef);
        Partition outPart = tDef.getFunction().transformRawInput(inputPart);
        PartitionIterator<Object> pItr = outPart.iterator();
        while (pItr.hasNext()) {
            Object oRow = pItr.next();
            forward(oRow, outputObjInspector);
        }
    } catch (WindowingException we) {
        throw new HiveException("Cannot close PTFOperator.", we);
    }
}

Example 13 with TableFuncDef

use of com.sap.hadoop.windowing.query2.definition.TableFuncDef in project SQLWindowing by hbutani.

the class WindowFunctionTranslation method translate.

public static WindowFunctionDef translate(QueryDef qDef, TableFuncDef windowTableFnDef, WindowFunctionSpec wFnSpec) throws WindowingException {
    QueryTranslationInfo tInfo = qDef.getTranslationInfo();
    InputInfo iInfo = tInfo.getInputInfo(windowTableFnDef.getInput());
    WindowFunctionDef wFnDef = new WindowFunctionDef();
    wFnDef.setSpec(wFnSpec);
    /*
		 * translate args
		 */
    ArrayList<ASTNode> args = wFnSpec.getArgs();
    if (args != null) {
        for (ASTNode expr : args) {
            ArgDef argDef = translateWindowFunctionArg(qDef, windowTableFnDef, iInfo, expr);
            wFnDef.addArg(argDef);
        }
    }
    if (RANKING_FUNCS.contains(wFnSpec.getName())) {
        setupRankingArgs(qDef, windowTableFnDef, wFnDef, wFnSpec);
    }
    WindowDef wDef = translateWindowSpec(qDef, iInfo, wFnSpec);
    wFnDef.setWindow(wDef);
    validateWindowDefForWFn(windowTableFnDef, wFnDef);
    setupEvaluator(wFnDef);
    return wFnDef;
}

Example 14 with TableFuncDef

use of com.sap.hadoop.windowing.query2.definition.TableFuncDef in project SQLWindowing by hbutani.

the class RuntimeUtils method createFirstPartitionForChain.

/**
	 * Create a new partition.
	 * The input OI is used to evaluate rows appended to the partition.
	 * The serde is determined based on whether the query has a map-phase 
	 * or not. The OI on the serde is used by PTFs to evaluate output of the 
	 * partition. 
	 * @param qDef
	 * @param oi
	 * @param hiveConf
	 * @return
	 * @throws WindowingException
	 */
public static Partition createFirstPartitionForChain(QueryDef qDef, ObjectInspector oi, HiveConf hiveConf, boolean isMapSide) throws WindowingException {
    TableFuncDef tabDef = getFirstTableFunction(qDef);
    TableFunctionEvaluator tEval = tabDef.getFunction();
    String partClassName = tEval.getPartitionClass();
    int partMemSize = tEval.getPartitionMemSize();
    Partition part = null;
    SerDe serde = tabDef.getInput().getSerde();
    part = new Partition(partClassName, partMemSize, serde, (StructObjectInspector) oi);
    return part;
}