Search in sources :

Example 6 with ColumnDef

use of com.sap.hadoop.windowing.query2.definition.ColumnDef in project SQLWindowing by hbutani.

the class OutputTranslation method translateSelectExpr.

public static ColumnDef translateSelectExpr(QueryDef qDef, InputInfo iInfo, int colIdx, String alias, ASTNode expr) throws WindowingException {
    ColumnDef cDef = new ColumnDef((ColumnSpec) null);
    ExprNodeDesc exprNode = TranslateUtils.buildExprNode(expr, iInfo.getTypeCheckCtx());
    ExprNodeEvaluator exprEval = WindowingExprNodeEvaluatorFactory.get(qDef.getTranslationInfo(), exprNode);
    ObjectInspector oi = TranslateUtils.initExprNodeEvaluator(qDef, exprNode, exprEval, iInfo);
    cDef.setExpression(expr);
    cDef.setExprNode(exprNode);
    cDef.setExprEvaluator(exprEval);
    cDef.setOI(oi);
    cDef.setAlias(getAlias(alias, expr, colIdx));
    return cDef;
}
Also used : ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) ExprNodeEvaluator(org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator) ColumnDef(com.sap.hadoop.windowing.query2.definition.ColumnDef) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc)

Example 7 with ColumnDef

use of com.sap.hadoop.windowing.query2.definition.ColumnDef in project SQLWindowing by hbutani.

the class MRUtils method initialize.

/**
	 * Construct the data structures containing ExprNodeDesc for partition
	 * columns and order columns. Use the input definition to construct the list
	 * of output columns for the ReduceSinkOperator
	 * 
	 * @throws WindowingException
	 */
public void initialize() throws WindowingException {
    TableFuncDef tabDef = RuntimeUtils.getFirstTableFunction(qdef);
    hiveTableDef = tabDef.getHiveTableDef();
    InputInfo inputInfo;
    ArrayList<ColumnDef> partColList = tabDef.getWindow().getPartDef().getColumns();
    TableFunctionEvaluator tEval = tabDef.getFunction();
    /*
		 * If the query has a map phase, the inputInfo is retrieved from the map
		 * output info of the table function definition. This is constructed
		 * using the map output oi of the table function definition. If the
		 * query does not have a map phase, the inputInfo is retrieved from the
		 * QueryInputDef (either HiveTableDef or HiveQueryDef) of the query.
		 */
    if (tEval.isTransformsRawInput()) {
        inputInfo = qdef.getTranslationInfo().getMapInputInfo(tabDef);
    } else {
        inputInfo = qdef.getTranslationInfo().getInputInfo(hiveTableDef);
    }
    for (ColumnDef colDef : partColList) {
        partCols.add(colDef.getExprNode());
    }
    ArrayList<OrderColumnDef> orderColList = tabDef.getWindow().getOrderDef().getColumns();
    for (OrderColumnDef colDef : orderColList) {
        Order order = colDef.getOrder();
        if (order.name().equals("ASC")) {
            orderString.append('+');
        } else {
            orderString.append('-');
        }
        orderCols.add(colDef.getExprNode());
        outputColumnNames.add(colDef.getAlias());
    }
    RowResolver rr = inputInfo.getRowResolver();
    ArrayList<ColumnInfo> colInfoList = rr.getColumnInfos();
    for (ColumnInfo colInfo : colInfoList) {
        String internalName = colInfo.getInternalName();
        TypeInfo type = colInfo.getType();
        valueCols.add(TranslateUtils.getExprDesc(internalName, type));
        outputColumnNames.add(internalName);
    }
}
Also used : Order(com.sap.hadoop.metadata.Order) OrderColumnDef(com.sap.hadoop.windowing.query2.definition.OrderColumnDef) ColumnDef(com.sap.hadoop.windowing.query2.definition.ColumnDef) OrderColumnDef(com.sap.hadoop.windowing.query2.definition.OrderColumnDef) ColumnInfo(org.apache.hadoop.hive.ql.exec.ColumnInfo) RowResolver(org.apache.hadoop.hive.ql.parse.RowResolver) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) TableFuncDef(com.sap.hadoop.windowing.query2.definition.TableFuncDef) InputInfo(com.sap.hadoop.windowing.query2.translate.QueryTranslationInfo.InputInfo) TableFunctionEvaluator(com.sap.hadoop.windowing.functions2.TableFunctionEvaluator)

Example 8 with ColumnDef

use of com.sap.hadoop.windowing.query2.definition.ColumnDef in project SQLWindowing by hbutani.

the class OutputTranslation method setupOutputSerDe.

@SuppressWarnings("unchecked")
public static void setupOutputSerDe(HiveConf hCfg, SelectDef selectDef, QueryOutputDef oDef) throws WindowingException {
    String serDeClassName = oDef.getSpec().getSerDeClass();
    Properties serDeProps = oDef.getSpec().getSerDeProps();
    Class<? extends SerDe> serDeClass;
    SerDe serde;
    try {
        serDeClass = (Class<? extends SerDe>) Class.forName(serDeClassName);
        serde = serDeClass.newInstance();
    } catch (Exception e) {
        throw new WindowingException("Internal error, initializing output SerDe", e);
    }
    StringBuilder colNames = new StringBuilder();
    StringBuilder colTypes = new StringBuilder();
    boolean first = true;
    for (ColumnDef cDef : selectDef.getColumns()) {
        if (!first) {
            colNames.append(",");
            colTypes.append(",");
        } else
            first = false;
        colNames.append(cDef.getAlias());
        colTypes.append(TypeInfoUtils.getTypeInfoFromObjectInspector(cDef.getOI()).getTypeName());
    }
    serDeProps.setProperty(org.apache.hadoop.hive.serde.Constants.LIST_COLUMNS, colNames.toString());
    serDeProps.setProperty(org.apache.hadoop.hive.serde.Constants.LIST_COLUMN_TYPES, colTypes.toString());
    try {
        serde.initialize(hCfg, serDeProps);
    } catch (SerDeException se) {
        throw new WindowingException("Failed to initialize output SerDe", se);
    }
    oDef.setSerDe(serde);
}
Also used : SerDe(org.apache.hadoop.hive.serde2.SerDe) WindowingException(com.sap.hadoop.windowing.WindowingException) ColumnDef(com.sap.hadoop.windowing.query2.definition.ColumnDef) Properties(java.util.Properties) WindowingException(com.sap.hadoop.windowing.WindowingException) SerDeException(org.apache.hadoop.hive.serde2.SerDeException) HiveException(org.apache.hadoop.hive.ql.metadata.HiveException) SerDeException(org.apache.hadoop.hive.serde2.SerDeException)

Example 9 with ColumnDef

use of com.sap.hadoop.windowing.query2.definition.ColumnDef in project SQLWindowing by hbutani.

the class OutputTranslation method translateSelectExprs.

public static void translateSelectExprs(QueryDef qDef) throws WindowingException {
    QueryTranslationInfo tInfo = qDef.getTranslationInfo();
    QueryInputDef iDef = qDef.getInput();
    InputInfo iInfo = tInfo.getInputInfo(iDef);
    SelectDef selectDef = qDef.getSelectList();
    SelectSpec selectSpec = qDef.getSpec().getSelectList();
    Iterator<Object> selectExprsAndAliases = selectSpec.getColumnListAndAlias();
    int i = 0;
    ColumnDef cDef = null;
    while (selectExprsAndAliases.hasNext()) {
        Object[] o = (Object[]) selectExprsAndAliases.next();
        boolean isWnFn = ((Boolean) o[0]).booleanValue();
        if (isWnFn) {
            cDef = translateWindowFnAlias(qDef, iInfo, i++, (String) o[1]);
        } else {
            cDef = translateSelectExpr(qDef, iInfo, i++, (String) o[1], (ASTNode) o[2]);
        }
        selectDef.addColumn(cDef);
    }
    TranslateUtils.setupSelectOI(selectDef);
}
Also used : SelectDef(com.sap.hadoop.windowing.query2.definition.SelectDef) ColumnDef(com.sap.hadoop.windowing.query2.definition.ColumnDef) SelectSpec(com.sap.hadoop.windowing.query2.specification.SelectSpec) InputInfo(com.sap.hadoop.windowing.query2.translate.QueryTranslationInfo.InputInfo) QueryInputDef(com.sap.hadoop.windowing.query2.definition.QueryInputDef) ASTNode(org.apache.hadoop.hive.ql.parse.ASTNode)

Example 10 with ColumnDef

use of com.sap.hadoop.windowing.query2.definition.ColumnDef in project SQLWindowing by hbutani.

the class TranslateUtils method setupSelectOI.

/**
	 * Use the column aliases and OIs on each ColumnDef 
	 * to recreate the OI on the select list
	 * @param sDef
	 */
public static void setupSelectOI(SelectDef sDef) {
    ArrayList<ColumnDef> selColDefs = sDef.getColumns();
    ArrayList<String> colAliases = new ArrayList<String>();
    ArrayList<ObjectInspector> colOIs = new ArrayList<ObjectInspector>();
    for (ColumnDef colDef : selColDefs) {
        colAliases.add(colDef.getAlias());
        colOIs.add(colDef.getOI());
    }
    sDef.setOI(ObjectInspectorFactory.getStandardStructObjectInspector(colAliases, colOIs));
}
Also used : ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) PrimitiveObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector) ArrayList(java.util.ArrayList) ColumnDef(com.sap.hadoop.windowing.query2.definition.ColumnDef) OrderColumnDef(com.sap.hadoop.windowing.query2.definition.OrderColumnDef)

Aggregations

ColumnDef (com.sap.hadoop.windowing.query2.definition.ColumnDef)12 OrderColumnDef (com.sap.hadoop.windowing.query2.definition.OrderColumnDef)7 ObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector)4 WindowingException (com.sap.hadoop.windowing.WindowingException)3 PartitionDef (com.sap.hadoop.windowing.query2.definition.PartitionDef)3 ArrayList (java.util.ArrayList)3 ExprNodeEvaluator (org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator)3 OrderDef (com.sap.hadoop.windowing.query2.definition.OrderDef)2 OrderColumnSpec (com.sap.hadoop.windowing.query2.specification.OrderColumnSpec)2 InputInfo (com.sap.hadoop.windowing.query2.translate.QueryTranslationInfo.InputInfo)2 HiveException (org.apache.hadoop.hive.ql.metadata.HiveException)2 SerDe (org.apache.hadoop.hive.serde2.SerDe)2 SerDeException (org.apache.hadoop.hive.serde2.SerDeException)2 Order (com.sap.hadoop.metadata.Order)1 TableFunctionEvaluator (com.sap.hadoop.windowing.functions2.TableFunctionEvaluator)1 QueryInputDef (com.sap.hadoop.windowing.query2.definition.QueryInputDef)1 SelectDef (com.sap.hadoop.windowing.query2.definition.SelectDef)1 TableFuncDef (com.sap.hadoop.windowing.query2.definition.TableFuncDef)1 WhereDef (com.sap.hadoop.windowing.query2.definition.WhereDef)1 WindowFrameDef (com.sap.hadoop.windowing.query2.definition.WindowFrameDef)1