Examples with ScriptOperator org.apache.hadoop.hive.ql.exec.ScriptOperator used on opensource projects

Example 1 with ScriptOperator

use of org.apache.hadoop.hive.ql.exec.ScriptOperator in project hive by apache.

the class OperatorHealthCheckerHook method checkOperator.

public static void checkOperator(Operator<?> op) {
    OperatorDesc conf = op.getConf();
    Map<String, ExprNodeDesc> exprMap = conf.getColumnExprMap();
    RowSchema schema = op.getSchema();
    checkSchema(schema);
    if (op instanceof SelectOperator) {
        checkSelectOperator((SelectOperator) op);
    }
    if (schema != null && exprMap != null) {
        for (Entry<String, ExprNodeDesc> c : exprMap.entrySet()) {
            if (c.getValue() instanceof ExprNodeConstantDesc) {
                continue;
            }
            ColumnInfo ci = schema.getColumnInfo(c.getKey());
            if (c.getKey().startsWith(Utilities.ReduceField.KEY + ".reducesinkkey")) {
                continue;
            }
            if (ci == null && conf.getComputedFields().contains(c.getKey())) {
                continue;
            }
            if (ci == null) {
                throw new RuntimeException("schema not found for " + c + " in " + schema);
            }
        }
        for (ColumnInfo sig : schema.getSignature()) {
            if (op instanceof ScriptOperator) {
                continue;
            }
            String iName = sig.getInternalName();
            ExprNodeDesc e = exprMap.get(iName);
            if (isSemiJoinRS(op)) {
                continue;
            }
            if (op.getConf() instanceof GroupByDesc) {
                continue;
            }
            if (e == null) {
                throw new RuntimeException("expr not found for " + iName + " in " + exprMap);
            }
        }
    }
}

Example 2 with ScriptOperator

use of org.apache.hadoop.hive.ql.exec.ScriptOperator in project hive by apache.

the class SparkTask method getOperatorCounters.

private Map<String, List<String>> getOperatorCounters() {
    String groupName = HiveConf.getVar(conf, HiveConf.ConfVars.HIVECOUNTERGROUP);
    Map<String, List<String>> counters = new HashMap<String, List<String>>();
    List<String> hiveCounters = new LinkedList<String>();
    counters.put(groupName, hiveCounters);
    hiveCounters.add(Operator.HIVE_COUNTER_CREATED_FILES);
    hiveCounters.add(FileSinkOperator.TOTAL_TABLE_ROWS_WRITTEN);
    // Spark transformation and Hive operators in SparkWork.
    for (MapOperator.Counter counter : MapOperator.Counter.values()) {
        hiveCounters.add(counter.toString());
    }
    SparkWork sparkWork = this.getWork();
    for (BaseWork work : sparkWork.getAllWork()) {
        for (Operator<? extends OperatorDesc> operator : work.getAllOperators()) {
            if (operator instanceof FileSinkOperator) {
                for (FileSinkOperator.Counter counter : FileSinkOperator.Counter.values()) {
                    hiveCounters.add(((FileSinkOperator) operator).getCounterName(counter));
                }
            } else if (operator instanceof ReduceSinkOperator) {
                final String contextName = conf.get(Operator.CONTEXT_NAME_KEY, "");
                for (ReduceSinkOperator.Counter counter : ReduceSinkOperator.Counter.values()) {
                    hiveCounters.add(Utilities.getVertexCounterName(counter.name(), contextName));
                }
            } else if (operator instanceof ScriptOperator) {
                for (ScriptOperator.Counter counter : ScriptOperator.Counter.values()) {
                    hiveCounters.add(counter.toString());
                }
            } else if (operator instanceof JoinOperator) {
                for (JoinOperator.SkewkeyTableCounter counter : JoinOperator.SkewkeyTableCounter.values()) {
                    hiveCounters.add(counter.toString());
                }
            }
        }
    }
    return counters;
}