Examples with QueryDef com.sap.hadoop.windowing.query2.definition.QueryDef used on opensource projects

Example 36 with QueryDef

use of com.sap.hadoop.windowing.query2.definition.QueryDef in project SQLWindowing by hbutani.

the class PTFOperator method reconstructQueryDef.

/**
	 * Initialize the visitor to use the QueryDefDeserializer Use the order
	 * defined in QueryDefWalker to visit the QueryDef
	 * 
	 * @param hiveConf
	 * @throws WindowingException
	 */
protected void reconstructQueryDef(HiveConf hiveConf) throws WindowingException {
    QueryDefVisitor qdd = new QueryDefDeserializer(hiveConf, inputObjInspectors[0]);
    QueryDefWalker qdw = new QueryDefWalker(qdd);
    qdw.walk(qDef);
}

Example 37 with QueryDef

use of com.sap.hadoop.windowing.query2.definition.QueryDef in project SQLWindowing by hbutani.

the class PTFOperator method initializeOp.

/*
	 * 1. Find out if the operator is invoked at Map-Side or Reduce-side 
	 * 2. Get the deserialized QueryDef 
	 * 3. Reconstruct the transient variables in QueryDef 
	 * 4. Create input partition to store rows coming from previous operator
	 */
@Override
protected void initializeOp(Configuration jobConf) throws HiveException {
    hiveConf = new HiveConf(jobConf, PTFOperator.class);
    // if the parent is ExtractOperator, this invocation is from reduce-side
    Operator<? extends OperatorDesc> parentOp = getParentOperators().get(0);
    if (parentOp instanceof ExtractOperator) {
        isMapOperator = false;
    } else {
        isMapOperator = true;
    }
    // use the string from PTFDesc to get deserialized QueryDef
    qDef = (QueryDef) SerializationUtils.deserialize(new ByteArrayInputStream(conf.getQueryDefStr().getBytes()));
    try {
        reconstructQueryDef(hiveConf);
        inputPart = RuntimeUtils.createFirstPartitionForChain(qDef, inputObjInspectors[0], hiveConf, isMapOperator);
    } catch (WindowingException we) {
        throw new HiveException("Cannot create input partition for PTFOperator.", we);
    }
    // OI for ReduceSinkOperator is taken from TODO
    if (isMapOperator) {
        TableFuncDef tDef = RuntimeUtils.getFirstTableFunction(qDef);
        outputObjInspector = tDef.getMapOI();
    } else {
        outputObjInspector = qDef.getSelectList().getOI();
    }
    setupKeysWrapper(inputObjInspectors[0]);
    super.initializeOp(jobConf);
}

Example 38 with QueryDef

use of com.sap.hadoop.windowing.query2.definition.QueryDef in project SQLWindowing by hbutani.

the class TranslateUtils method initExprNodeEvaluator.

public static ObjectInspector initExprNodeEvaluator(QueryDef qDef, ExprNodeDesc exprNode, ExprNodeEvaluator exprEval, InputInfo iInfo) throws WindowingException {
    ObjectInspector OI;
    try {
        OI = exprEval.initialize(iInfo.getOI());
    } catch (HiveException he) {
        throw new WindowingException(he);
    }
    /*
		 * if there are any LeadLag functions in this Expression Tree: - setup a
		 * duplicate Evaluator for the 1st arg of the LLFuncDesc - initialize it
		 * using the InputInfo provided for this Expr tree - set the duplicate
		 * evaluator on the LLUDF instance.
		 */
    LeadLagInfo llInfo = qDef.getTranslationInfo().getLLInfo();
    List<ExprNodeGenericFuncDesc> llFuncExprs = llInfo.getLLFuncExprsInTopExpr(exprNode);
    if (llFuncExprs != null) {
        for (ExprNodeGenericFuncDesc llFuncExpr : llFuncExprs) {
            ExprNodeDesc firstArg = llFuncExpr.getChildren().get(0);
            ExprNodeEvaluator dupExprEval = WindowingExprNodeEvaluatorFactory.get(qDef.getTranslationInfo(), firstArg);
            try {
                dupExprEval.initialize(iInfo.getOI());
            } catch (HiveException he) {
                throw new WindowingException(he);
            }
            GenericUDFLeadLag llFn = (GenericUDFLeadLag) llFuncExpr.getGenericUDF();
            llFn.setExprEvaluator(dupExprEval);
        }
    }
    return OI;
}

Example 39 with QueryDef

use of com.sap.hadoop.windowing.query2.definition.QueryDef in project SQLWindowing by hbutani.

the class WhereTranslation method translate.

public static void translate(QueryDef qDef) throws WindowingException {
    QueryTranslationInfo tInfo = qDef.getTranslationInfo();
    QuerySpec spec = qDef.getSpec();
    ASTNode wExpr = spec.getWhereExpr();
    if (wExpr == null)
        return;
    WhereDef whDef = new WhereDef();
    whDef.setExpression(wExpr);
    QueryInputDef iDef = qDef.getInput();
    InputInfo iInfo = tInfo.getInputInfo(iDef);
    ExprNodeDesc exprNode = TranslateUtils.buildExprNode(wExpr, iInfo.getTypeCheckCtx());
    ExprNodeEvaluator exprEval = WindowingExprNodeEvaluatorFactory.get(tInfo, exprNode);
    ObjectInspector oi = TranslateUtils.initExprNodeEvaluator(qDef, exprNode, exprEval, iInfo);
    try {
        ObjectInspectorConverters.getConverter(oi, PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
    } catch (Throwable t) {
        throw new WindowingException("Where Expr must be convertible to a boolean value", t);
    }
    whDef.setExprNode(exprNode);
    whDef.setExprEvaluator(exprEval);
    whDef.setOI(oi);
    qDef.setWhere(whDef);
}

Example 40 with QueryDef

use of com.sap.hadoop.windowing.query2.definition.QueryDef in project SQLWindowing by hbutani.

the class WindowFunctionTranslation method translate.

public static WindowFunctionDef translate(QueryDef qDef, TableFuncDef windowTableFnDef, WindowFunctionSpec wFnSpec) throws WindowingException {
    QueryTranslationInfo tInfo = qDef.getTranslationInfo();
    InputInfo iInfo = tInfo.getInputInfo(windowTableFnDef.getInput());
    WindowFunctionDef wFnDef = new WindowFunctionDef();
    wFnDef.setSpec(wFnSpec);
    /*
		 * translate args
		 */
    ArrayList<ASTNode> args = wFnSpec.getArgs();
    if (args != null) {
        for (ASTNode expr : args) {
            ArgDef argDef = translateWindowFunctionArg(qDef, windowTableFnDef, iInfo, expr);
            wFnDef.addArg(argDef);
        }
    }
    if (RANKING_FUNCS.contains(wFnSpec.getName())) {
        setupRankingArgs(qDef, windowTableFnDef, wFnDef, wFnSpec);
    }
    WindowDef wDef = translateWindowSpec(qDef, iInfo, wFnSpec);
    wFnDef.setWindow(wDef);
    validateWindowDefForWFn(windowTableFnDef, wFnDef);
    setupEvaluator(wFnDef);
    return wFnDef;
}