Examples with TezCounter org.apache.tez.common.counters.TezCounter used on opensource projects

Example 1 with TezCounter

use of org.apache.tez.common.counters.TezCounter in project hive by apache.

the class PostExecTezSummaryPrinter method run.

@Override
public void run(HookContext hookContext) throws Exception {
    assert (hookContext.getHookType() == HookContext.HookType.POST_EXEC_HOOK);
    HiveConf conf = hookContext.getConf();
    if (!"tez".equals(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE))) {
        return;
    }
    LOG.info("Executing post execution hook to print tez summary..");
    SessionState ss = SessionState.get();
    SessionState.LogHelper console = ss.getConsole();
    QueryPlan plan = hookContext.getQueryPlan();
    if (plan == null) {
        return;
    }
    List<TezTask> rootTasks = Utilities.getTezTasks(plan.getRootTasks());
    for (TezTask tezTask : rootTasks) {
        LOG.info("Printing summary for tez task: " + tezTask.getName());
        TezCounters counters = tezTask.getTezCounters();
        if (counters != null) {
            String hiveCountersGroup = HiveConf.getVar(conf, HiveConf.ConfVars.HIVECOUNTERGROUP);
            for (CounterGroup group : counters) {
                if (hiveCountersGroup.equals(group.getDisplayName())) {
                    console.printError(tezTask.getId() + " HIVE COUNTERS:");
                    for (TezCounter counter : group) {
                        console.printError("   " + counter.getDisplayName() + ": " + counter.getValue());
                    }
                } else if (group.getName().equals(FileSystemCounter.class.getName())) {
                    console.printError(tezTask.getId() + " FILE SYSTEM COUNTERS:");
                    for (TezCounter counter : group) {
                        // local file system counters
                        if (counter.getName().contains("HDFS")) {
                            console.printError("   " + counter.getDisplayName() + ": " + counter.getValue());
                        }
                    }
                } else if (group.getName().equals(LlapIOCounters.class.getName())) {
                    console.printError(tezTask.getId() + " LLAP IO COUNTERS:");
                    List<String> testSafeCounters = LlapIOCounters.testSafeCounterNames();
                    for (TezCounter counter : group) {
                        if (testSafeCounters.contains(counter.getDisplayName())) {
                            console.printError("   " + counter.getDisplayName() + ": " + counter.getValue());
                        }
                    }
                }
            }
        }
    }
}

Example 2 with TezCounter

use of org.apache.tez.common.counters.TezCounter in project hive by apache.

the class TezTask method execute.

@Override
public int execute(DriverContext driverContext) {
    int rc = 1;
    boolean cleanContext = false;
    Context ctx = null;
    TezSessionState session = null;
    try {
        // Get or create Context object. If we create it we have to clean it later as well.
        ctx = driverContext.getCtx();
        if (ctx == null) {
            ctx = new Context(conf);
            cleanContext = true;
        }
        // Need to remove this static hack. But this is the way currently to get a session.
        SessionState ss = SessionState.get();
        session = ss.getTezSession();
        if (session != null && !session.isOpen()) {
            LOG.warn("The session: " + session + " has not been opened");
        }
        session = TezSessionPoolManager.getInstance().getSession(session, conf, false, getWork().getLlapMode());
        ss.setTezSession(session);
        try {
            // jobConf will hold all the configuration for hadoop, tez, and hive
            JobConf jobConf = utils.createConfiguration(conf);
            // Get all user jars from work (e.g. input format stuff).
            String[] inputOutputJars = work.configureJobConfAndExtractJars(jobConf);
            // we will localize all the files (jars, plans, hashtables) to the
            // scratch dir. let's create this and tmp first.
            Path scratchDir = ctx.getMRScratchDir();
            // create the tez tmp dir
            scratchDir = utils.createTezDir(scratchDir, conf);
            Map<String, LocalResource> inputOutputLocalResources = getExtraLocalResources(jobConf, scratchDir, inputOutputJars);
            // Ensure the session is open and has the necessary local resources
            updateSession(session, jobConf, scratchDir, inputOutputJars, inputOutputLocalResources);
            List<LocalResource> additionalLr = session.getLocalizedResources();
            logResources(additionalLr);
            // unless already installed on all the cluster nodes, we'll have to
            // localize hive-exec.jar as well.
            LocalResource appJarLr = session.getAppJarLr();
            // next we translate the TezWork to a Tez DAG
            DAG dag = build(jobConf, work, scratchDir, appJarLr, additionalLr, ctx);
            CallerContext callerContext = CallerContext.create("HIVE", queryPlan.getQueryId(), "HIVE_QUERY_ID", queryPlan.getQueryStr());
            dag.setCallerContext(callerContext);
            // Add the extra resources to the dag
            addExtraResourcesToDag(session, dag, inputOutputJars, inputOutputLocalResources);
            // submit will send the job to the cluster and start executing
            dagClient = submit(jobConf, dag, scratchDir, appJarLr, session, additionalLr, inputOutputJars, inputOutputLocalResources);
            // finally monitor will print progress until the job is done
            TezJobMonitor monitor = new TezJobMonitor(work.getWorkMap(), dagClient, conf, dag, ctx);
            rc = monitor.monitorExecution();
            if (rc != 0) {
                this.setException(new HiveException(monitor.getDiagnostics()));
            }
            // fetch the counters
            try {
                Set<StatusGetOpts> statusGetOpts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
                counters = dagClient.getDAGStatus(statusGetOpts).getDAGCounters();
            } catch (Exception err) {
                // Don't fail execution due to counters - just don't print summary info
                LOG.warn("Failed to get counters. Ignoring, summary info will be incomplete. " + err, err);
                counters = null;
            }
        } finally {
            // We return this to the pool even if it's unusable; reopen is supposed to handle this.
            try {
                TezSessionPoolManager.getInstance().returnSession(session, getWork().getLlapMode());
            } catch (Exception e) {
                LOG.error("Failed to return session: {} to pool", session, e);
                throw e;
            }
        }
        if (LOG.isInfoEnabled() && counters != null && (HiveConf.getBoolVar(conf, HiveConf.ConfVars.TEZ_EXEC_SUMMARY) || Utilities.isPerfOrAboveLogging(conf))) {
            for (CounterGroup group : counters) {
                LOG.info(group.getDisplayName() + ":");
                for (TezCounter counter : group) {
                    LOG.info("   " + counter.getDisplayName() + ": " + counter.getValue());
                }
            }
        }
    } catch (Exception e) {
        LOG.error("Failed to execute tez graph.", e);
    // rc will be 1 at this point indicating failure.
    } finally {
        Utilities.clearWork(conf);
        // Clear gWorkMap
        for (BaseWork w : work.getAllWork()) {
            JobConf workCfg = workToConf.get(w);
            if (workCfg != null) {
                Utilities.clearWorkMapForConf(workCfg);
            }
        }
        if (cleanContext) {
            try {
                ctx.clear();
            } catch (Exception e) {
                /*best effort*/
                LOG.warn("Failed to clean up after tez job", e);
            }
        }
        // need to either move tmp files or remove them
        if (dagClient != null) {
            // rc will only be overwritten if close errors out
            rc = close(work, rc);
        }
    }
    return rc;
}

Example 3 with TezCounter

use of org.apache.tez.common.counters.TezCounter in project hive by apache.

the class PostExecOrcRowGroupCountPrinter method run.

@Override
public void run(HookContext hookContext) throws Exception {
    assert (hookContext.getHookType() == HookContext.HookType.POST_EXEC_HOOK);
    HiveConf conf = hookContext.getConf();
    if (!"tez".equals(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE))) {
        return;
    }
    LOG.info("Executing post execution hook to print ORC row groups read counter..");
    SessionState ss = SessionState.get();
    SessionState.LogHelper console = ss.getConsole();
    QueryPlan plan = hookContext.getQueryPlan();
    if (plan == null) {
        return;
    }
    List<TezTask> rootTasks = Utilities.getTezTasks(plan.getRootTasks());
    for (TezTask tezTask : rootTasks) {
        LOG.info("Printing ORC row group counter for tez task: " + tezTask.getName());
        TezCounters counters = tezTask.getTezCounters();
        if (counters != null) {
            for (CounterGroup group : counters) {
                if (group.getName().equals(LlapIOCounters.class.getName())) {
                    console.printError(tezTask.getId() + " LLAP IO COUNTERS:");
                    for (TezCounter counter : group) {
                        if (counter.getDisplayName().equals(LlapIOCounters.SELECTED_ROWGROUPS.name())) {
                            console.printError("   " + counter.getDisplayName() + ": " + counter.getValue());
                        }
                    }
                }
            }
        }
    }
}