Examples with ASTNode org.apache.hadoop.hive.ql.parse.ASTNode used on opensource projects

Example 11 with ASTNode

use of org.apache.hadoop.hive.ql.parse.ASTNode in project SQLWindowing by hbutani.

the class WindowingTypeCheckProcFactory method processGByExpr.

/**
	 * Function to do groupby subexpression elimination. This is called by all
	 * the processors initially. As an example, consider the query select a+b,
	 * count(1) from T group by a+b; Then a+b is already precomputed in the
	 * group by operators key, so we substitute a+b in the select list with the
	 * internal column name of the a+b expression that appears in the in input
	 * row resolver.
	 * 
	 * @param nd
	 *            The node that is being inspected.
	 * @param procCtx
	 *            The processor context.
	 * 
	 * @return exprNodeColumnDesc.
	 */
public static ExprNodeDesc processGByExpr(Node nd, Object procCtx) throws SemanticException {
    // We recursively create the exprNodeDesc. Base cases: when we encounter
    // a column ref, we convert that into an exprNodeColumnDesc; when we
    // encounter
    // a constant, we convert that into an exprNodeConstantDesc. For others
    // we
    // just
    // build the exprNodeFuncDesc with recursively built children.
    ASTNode expr = (ASTNode) nd;
    TypeCheckCtx ctx = (TypeCheckCtx) procCtx;
    RowResolver input = ctx.getInputRR();
    ExprNodeDesc desc = null;
    // If the current subExpression is pre-calculated, as in Group-By etc.
    ColumnInfo colInfo = input.getExpression(expr);
    if (colInfo != null) {
        desc = new ExprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName(), colInfo.getTabAlias(), colInfo.getIsVirtualCol());
        //			}
        return desc;
    }
    return desc;
}

Example 12 with ASTNode

use of org.apache.hadoop.hive.ql.parse.ASTNode in project SQLWindowing by hbutani.

the class WindowingTypeCheckProcFactory method genExprNode.

public static HashMap<Node, Object> genExprNode(ASTNode expr, TypeCheckCtx tcCtx) throws SemanticException {
    // Create the walker, the rules dispatcher and the context.
    // create a walker which walks the tree in a DFS manner while
    // maintaining
    // the operator stack. The dispatcher
    // generates the plan from the operator tree
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("R1", Windowing2Parser.NULL + "%"), getNullExprProcessor());
    opRules.put(new RuleRegExp("R2", Windowing2Parser.Number + "%|" + Windowing2Parser.TinyintLiteral + "%|" + Windowing2Parser.SmallintLiteral + "%|" + Windowing2Parser.BigintLiteral + "%"), getNumExprProcessor());
    opRules.put(new RuleRegExp("R3", Windowing2Parser.Identifier + "%|" + Windowing2Parser.StringLiteral + "%|" + Windowing2Parser.CHARSETLITERAL + "%|" + Windowing2Parser.STRINGLITERALSEQUENCE + "%|" + "%|" + Windowing2Parser.IF + "%|" + Windowing2Parser.CASE + "%|" + Windowing2Parser.WHEN + "%|" + Windowing2Parser.IN + "%|" + Windowing2Parser.ARRAY + "%|" + Windowing2Parser.MAP + "%|" + Windowing2Parser.BETWEEN + "%|" + Windowing2Parser.STRUCT + "%"), getStrExprProcessor());
    opRules.put(new RuleRegExp("R4", Windowing2Parser.TRUE + "%|" + Windowing2Parser.FALSE + "%"), getBoolExprProcessor());
    opRules.put(new RuleRegExp("R5", Windowing2Parser.TABLEORCOL + "%"), getColumnExprProcessor());
    // The dispatcher fires the processor corresponding to the closest
    // matching
    // rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(getDefaultExprProcessor(), opRules, tcCtx);
    GraphWalker ogw = new DefaultGraphWalker(disp);
    // Create a list of topop nodes
    ArrayList<Node> topNodes = new ArrayList<Node>();
    topNodes.add(expr);
    HashMap<Node, Object> nodeOutputs = new HashMap<Node, Object>();
    ogw.startWalking(topNodes, nodeOutputs);
    return nodeOutputs;
}

Example 13 with ASTNode

use of org.apache.hadoop.hive.ql.parse.ASTNode in project SQLWindowing by hbutani.

the class WindowSpecTranslation method translateColumn.

static void translateColumn(QueryDef qDef, ColumnDef cDef, InputInfo iInfo, ColumnSpec cSpec) throws WindowingException {
    String colTabName = cSpec.getTableName();
    if (colTabName != null && !colTabName.equals(iInfo.getAlias())) {
        throw new WindowingException(sprintf("Unknown Table Reference in column", cSpec));
    }
    ASTNode expr = TranslateUtils.buildASTNode(cSpec.getColumnName());
    ExprNodeDesc exprNode = TranslateUtils.buildExprNode(expr, iInfo.getTypeCheckCtx());
    ExprNodeEvaluator exprEval = WindowingExprNodeEvaluatorFactory.get(qDef.getTranslationInfo(), exprNode);
    ObjectInspector oi = TranslateUtils.initExprNodeEvaluator(qDef, exprNode, exprEval, iInfo);
    cDef.setExpression(expr);
    cDef.setExprNode(exprNode);
    cDef.setExprEvaluator(exprEval);
    cDef.setOI(oi);
    cDef.setAlias(cSpec.getColumnName());
}

Example 14 with ASTNode

use of org.apache.hadoop.hive.ql.parse.ASTNode in project SQLWindowing by hbutani.

the class SerializationUtils method addAntlrPersistenceDelegates.

public static void addAntlrPersistenceDelegates(XMLEncoder e) {
    e.setPersistenceDelegate(ASTNode.class, new PersistenceDelegate() {

        protected Expression instantiate(Object oldInstance, Encoder out) {
            return new Expression(oldInstance, oldInstance.getClass(), "new", new Object[] { ((ASTNode) oldInstance).getToken() });
        }
    });
    e.setPersistenceDelegate(CommonTree.class, new PersistenceDelegate() {

        protected Expression instantiate(Object oldInstance, Encoder out) {
            return new Expression(oldInstance, oldInstance.getClass(), "new", new Object[] { ((CommonTree) oldInstance).getToken() });
        }
    });
    e.setPersistenceDelegate(BaseTree.class, new PersistenceDelegate() {

        protected Expression instantiate(Object oldInstance, Encoder out) {
            return new Expression(oldInstance, oldInstance.getClass(), "new", new Object[] {});
        }

        @SuppressWarnings("rawtypes")
        protected void initialize(Class type, Object oldInstance, Object newInstance, Encoder out) {
            super.initialize(type, oldInstance, newInstance, out);
            BaseTree t = (BaseTree) oldInstance;
            for (int i = 0; i < t.getChildCount(); i++) {
                out.writeStatement(new Statement(oldInstance, "addChild", new Object[] { t.getChild(i) }));
            }
        }
    });
    e.setPersistenceDelegate(CommonToken.class, new PersistenceDelegate() {

        protected Expression instantiate(Object oldInstance, Encoder out) {
            return new Expression(oldInstance, oldInstance.getClass(), "new", new Object[] { ((CommonToken) oldInstance).getType(), ((CommonToken) oldInstance).getText() });
        }
    });
}

Example 15 with ASTNode

use of org.apache.hadoop.hive.ql.parse.ASTNode in project SQLWindowing by hbutani.

the class InputTranslation method translate.

/*
	 * <ol>
	 * <li> Get the <code>TableFunctionResolver</code> for this Function from the FunctionRegistry.
	 * <li> Create the TableFuncDef object.
	 * <li> Get the InputInfo for the input to this function.
	 * <li> Translate the Arguments to this Function in the Context of the InputInfo.
	 * <li> ask the  TableFunctionResolver to create a TableFunctionEvaluator based on the Args passed in.
	 * <li> ask the TableFunctionEvaluator to setup the Map-side ObjectInspector. Gives a chance to functions that 
	 * reshape the Input before it is partitioned to define the Shape after raw data is transformed.
	 * <li> Setup the Window Definition for this Function. The Window Definition is resolved wrt to the InputDef's
	 * Shape or the MapOI, for Functions that reshape the raw input.
	 * <li> ask the TableFunctionEvaluator to setup the Output ObjectInspector for this Function.
	 * <li> setup a Serde for the Output partition based on the OutputOI. 
	 * </ol> 
	 */
private static TableFuncDef translate(QueryDef qDef, TableFuncSpec tSpec, QueryInputDef inputDef) throws WindowingException {
    QueryTranslationInfo tInfo = qDef.getTranslationInfo();
    TableFunctionResolver tFn = FunctionRegistry.getTableFunctionResolver(tSpec.getName());
    if (tFn == null) {
        throw new WindowingException(sprintf("Unknown Table Function %s", tSpec.getName()));
    }
    TableFuncDef tDef = new TableFuncDef();
    tDef.setSpec(tSpec);
    tDef.setInput(inputDef);
    InputInfo iInfo = tInfo.getInputInfo(inputDef);
    /*
		 * translate args
		 */
    ArrayList<ASTNode> args = tSpec.getArgs();
    if (args != null) {
        for (ASTNode expr : args) {
            ArgDef argDef = translateTableFunctionArg(qDef, tDef, iInfo, expr);
            tDef.addArg(argDef);
        }
    }
    tFn.initialize(qDef, tDef);
    TableFunctionEvaluator tEval = tFn.getEvaluator();
    tDef.setFunction(tEval);
    tFn.setupRawInputOI();
    tDef.setWindow(WindowSpecTranslation.translateWindow(qDef, tDef));
    tFn.setupOutputOI();
    TranslateUtils.setupSerdeAndOI(tDef, inputDef, tInfo, tEval);
    return tDef;
}