Examples with QueryTranslationInfo com.sap.hadoop.windowing.query2.translate.QueryTranslationInfo used on opensource projects

Example 6 with QueryTranslationInfo

use of com.sap.hadoop.windowing.query2.translate.QueryTranslationInfo in project SQLWindowing by hbutani.

the class OutputTranslation method translate.

public static void translate(QueryDef qDef) throws WindowingException {
    QueryTranslationInfo tInfo = qDef.getTranslationInfo();
    translateSelectExprs(qDef);
    validateOutputSpec(qDef);
    QueryOutputDef oDef = new QueryOutputDef();
    oDef.setOutputSpec(qDef.getSpec().getOutput());
    qDef.setOutput(oDef);
    setupOutputSerDe(tInfo.getHiveCfg(), qDef.getSelectList(), oDef);
}

Example 7 with QueryTranslationInfo

use of com.sap.hadoop.windowing.query2.translate.QueryTranslationInfo in project SQLWindowing by hbutani.

the class OutputTranslation method validateOutputSpec.

public static void validateOutputSpec(QueryDef qDef) throws WindowingException {
    QueryTranslationInfo tInfo = qDef.getTranslationInfo();
    QueryOutputSpec spec = qDef.getSpec().getOutput();
    // ensure outputPath is specified. It is optional in grammar because it is not required in Hive mode.
    if (spec == null || spec.getPath() == null) {
        throw new WindowingException("Query doesn't contain an output Path for results");
    }
    // if tableName is specified; validate it exists
    Table oTbl = null;
    if (spec.getHiveTable() != null) {
        oTbl = getHiveTableDetails(tInfo.getHiveCfg(), spec.getHiveTable(), qDef.getInput().getHiveTableSpec());
    }
    // validate serDeClass
    if (spec.getSerDeClass() == null) {
        if (oTbl != null && oTbl.getSd().getSerdeInfo().isSetSerializationLib()) {
            spec.setSerDeClass(oTbl.getSd().getSerdeInfo().getSerializationLib());
            if (oTbl.getSd().getSerdeInfo().isSetParameters()) {
                Iterator<Map.Entry<String, String>> props = oTbl.getSd().getSerdeInfo().getParameters().entrySet().iterator();
                while (props.hasNext()) {
                    Map.Entry<String, String> e = props.next();
                    spec.addSerdeProperty(e.getKey(), e.getValue());
                }
            }
        } else {
            spec.setSerDeClass(com.sap.hadoop.windowing.Constants.DEFAULT_SERDE_CLASSNAME);
            spec.addSerdeProperty(org.apache.hadoop.hive.serde.Constants.FIELD_DELIM, ",");
        }
    }
    try {
        Class.forName(spec.getSerDeClass());
    } catch (Throwable t) {
        throw new WindowingException(sprintf("Unknown SerDe Class %s", spec.getSerDeClass()), t);
    }
    // validate outputFormat
    if (spec.getOutputFormatClass() == null) {
        if (oTbl != null) {
            spec.setOutputFormatClass(oTbl.getSd().getOutputFormat());
        } else {
            spec.setOutputFormatClass(Constants.DEFAULT_OUTPUTFORMAT_CLASSNAME);
        }
    }
    try {
        Class.forName(spec.getOutputFormatClass());
    } catch (Throwable t) {
        throw new WindowingException(sprintf("Unknown OutputFormat Class %s", spec.getOutputFormatClass()), t);
    }
    // ensure user has not specified a FormatClass
    if (spec.getRecordWriterClass() != null) {
        throw new WindowingException("Illegal Output Spec: RecordWriter class not valid in MR mode");
    }
}

Example 8 with QueryTranslationInfo

use of com.sap.hadoop.windowing.query2.translate.QueryTranslationInfo in project SQLWindowing by hbutani.

the class OutputTranslation method translateSelectExprs.

public static void translateSelectExprs(QueryDef qDef) throws WindowingException {
    QueryTranslationInfo tInfo = qDef.getTranslationInfo();
    QueryInputDef iDef = qDef.getInput();
    InputInfo iInfo = tInfo.getInputInfo(iDef);
    SelectDef selectDef = qDef.getSelectList();
    SelectSpec selectSpec = qDef.getSpec().getSelectList();
    Iterator<Object> selectExprsAndAliases = selectSpec.getColumnListAndAlias();
    int i = 0;
    ColumnDef cDef = null;
    while (selectExprsAndAliases.hasNext()) {
        Object[] o = (Object[]) selectExprsAndAliases.next();
        boolean isWnFn = ((Boolean) o[0]).booleanValue();
        if (isWnFn) {
            cDef = translateWindowFnAlias(qDef, iInfo, i++, (String) o[1]);
        } else {
            cDef = translateSelectExpr(qDef, iInfo, i++, (String) o[1], (ASTNode) o[2]);
        }
        selectDef.addColumn(cDef);
    }
    TranslateUtils.setupSelectOI(selectDef);
}

Example 9 with QueryTranslationInfo

use of com.sap.hadoop.windowing.query2.translate.QueryTranslationInfo in project SQLWindowing by hbutani.

the class WindowFunctionTranslation method translate.

public static WindowFunctionDef translate(QueryDef qDef, TableFuncDef windowTableFnDef, WindowFunctionSpec wFnSpec) throws WindowingException {
    QueryTranslationInfo tInfo = qDef.getTranslationInfo();
    InputInfo iInfo = tInfo.getInputInfo(windowTableFnDef.getInput());
    WindowFunctionDef wFnDef = new WindowFunctionDef();
    wFnDef.setSpec(wFnSpec);
    /*
		 * translate args
		 */
    ArrayList<ASTNode> args = wFnSpec.getArgs();
    if (args != null) {
        for (ASTNode expr : args) {
            ArgDef argDef = translateWindowFunctionArg(qDef, windowTableFnDef, iInfo, expr);
            wFnDef.addArg(argDef);
        }
    }
    if (RANKING_FUNCS.contains(wFnSpec.getName())) {
        setupRankingArgs(qDef, windowTableFnDef, wFnDef, wFnSpec);
    }
    WindowDef wDef = translateWindowSpec(qDef, iInfo, wFnSpec);
    wFnDef.setWindow(wDef);
    validateWindowDefForWFn(windowTableFnDef, wFnDef);
    setupEvaluator(wFnDef);
    return wFnDef;
}

Example 10 with QueryTranslationInfo

use of com.sap.hadoop.windowing.query2.translate.QueryTranslationInfo in project SQLWindowing by hbutani.

the class LocalExecutor method execute.

public void execute(QueryDef qDef, WindowingShell wShell) throws WindowingException {
    QueryTranslationInfo tInfo = qDef.getTranslationInfo();
    HiveTableSpec hvTblSpec = qDef.getInput().getHiveTableSpec();
    WindowingInput wIn = IOUtils.createTableWindowingInput(hvTblSpec.getDbName(), hvTblSpec.getTableName(), tInfo.getHiveCfg());
    // Partition p = IOUtils.createPartition(partClassName, partMemSize, wIn);
    PartitionsIterator partsItr = new PartitionsIterator(wIn, qDef);
    while (partsItr.hasNext()) {
        Partition p = partsItr.next();
        Partition oP = executeChain(qDef, p);
        // IOUtils.dumpPartition(oP, System.out);
        executeSelectList(qDef, oP, new SysOutRS(out));
    }
}