Examples with TezCounters org.apache.tez.common.counters.TezCounters used on opensource projects

Example 11 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project hive by apache.

the class OperatorStatsReaderHook method run.

@Override
public void run(HookContext hookContext) throws Exception {
    if (hookContext.getHookType() == HookType.PRE_EXEC_HOOK) {
        return;
    }
    if (hookContext.getHookType() == HookType.POST_EXEC_HOOK && !isCollectOnSuccess()) {
        return;
    }
    HiveConf conf = hookContext.getConf();
    QueryPlan plan = hookContext.getQueryPlan();
    List<TezTask> rootTasks = Utilities.getTezTasks(plan.getRootTasks());
    for (TezTask tezTask : rootTasks) {
        List<BaseWork> baseWorks = tezTask.getWork().getAllWork();
        for (BaseWork baseWork : baseWorks) {
            String vertexName = baseWork.getName();
            LOG.debug("Reading runtime statistics for tez vertex task: {}", vertexName);
            TezCounters counters = tezTask.getTezCounters();
            if (counters != null) {
                String groupName = HiveConf.getVar(conf, HiveConf.ConfVars.HIVECOUNTERGROUP);
                for (Operator<? extends OperatorDesc> op : baseWork.getAllOperators()) {
                    String operatorId = op.getOperatorId();
                    OperatorStats operatorStats = null;
                    String counterName = Operator.Counter.RECORDS_OUT_OPERATOR.toString() + "_" + operatorId;
                    TezCounter tezCounter = counters.getGroup(groupName).findCounter(counterName, false);
                    if (tezCounter != null) {
                        if (operatorStats == null) {
                            operatorStats = new OperatorStats(operatorId);
                        }
                        operatorStats.setOutputRecords(tezCounter.getValue());
                    }
                    if (operatorStats != null) {
                        ((PrivateHookContext) hookContext).getContext().getPlanMapper().link(op, operatorStats);
                    } else {
                        LOG.debug("Unable to get statistics for vertex: {} opId: {} groupName: {}", vertexName, operatorId, groupName);
                    }
                }
            }
        }
    }
}