Examples with CounterGroup org.apache.tez.common.counters.CounterGroup used on opensource projects

Example 11 with CounterGroup

use of org.apache.tez.common.counters.CounterGroup in project tez by apache.

the class AMWebController method constructCounterMapInfo.

Map<String, Map<String, Long>> constructCounterMapInfo(TezCounters counters, Map<String, Set<String>> counterNames) {
    if (counterNames == null || counterNames.isEmpty()) {
        return null;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Requested counter names=" + counterNames.entrySet());
        LOG.debug("actual counters=" + counters);
    }
    Map<String, Map<String, Long>> counterInfo = new TreeMap<String, Map<String, Long>>();
    if (counterNames.containsKey("*")) {
        for (CounterGroup grpCounters : counters) {
            Map<String, Long> matchedCounters = new HashMap<String, Long>();
            for (TezCounter counter : grpCounters) {
                matchedCounters.put(counter.getName(), counter.getValue());
            }
            counterInfo.put(grpCounters.getName(), matchedCounters);
        }
    } else {
        for (Entry<String, Set<String>> entry : counterNames.entrySet()) {
            Map<String, Long> matchedCounters = new HashMap<String, Long>();
            CounterGroup grpCounters = counters.getGroup(entry.getKey());
            for (TezCounter counter : grpCounters) {
                if (entry.getValue().isEmpty() || entry.getValue().contains(counter.getName())) {
                    matchedCounters.put(counter.getName(), counter.getValue());
                }
            }
            counterInfo.put(entry.getKey(), matchedCounters);
        }
    }
    return counterInfo;
}

Example 12 with CounterGroup

use of org.apache.tez.common.counters.CounterGroup in project tez by apache.

the class DagTypeConverters method convertTezCountersFromProto.

public static TezCounters convertTezCountersFromProto(TezCountersProto proto) {
    TezCounters counters = new TezCounters();
    for (TezCounterGroupProto counterGroupProto : proto.getCounterGroupsList()) {
        CounterGroup group = counters.addGroup(counterGroupProto.getName(), counterGroupProto.getDisplayName());
        for (TezCounterProto counterProto : counterGroupProto.getCountersList()) {
            TezCounter counter = group.findCounter(counterProto.getName(), counterProto.getDisplayName());
            counter.setValue(counterProto.getValue());
        }
    }
    return counters;
}

Example 13 with CounterGroup

use of org.apache.tez.common.counters.CounterGroup in project tez by apache.

the class TestTezJobs method testPerIOCounterAggregation.

@Test(timeout = 60000)
public void testPerIOCounterAggregation() throws Exception {
    String baseDir = "/tmp/perIOCounterAgg/";
    Path inPath1 = new Path(baseDir + "inPath1");
    Path inPath2 = new Path(baseDir + "inPath2");
    Path outPath = new Path(baseDir + "outPath");
    final Set<String> expectedResults = generateSortMergeJoinInput(inPath1, inPath2);
    Path stagingDirPath = new Path("/tmp/tez-staging-dir");
    remoteFs.mkdirs(stagingDirPath);
    TezConfiguration conf = new TezConfiguration(mrrTezCluster.getConfig());
    conf.setBoolean(TezConfiguration.TEZ_TASK_GENERATE_COUNTERS_PER_IO, true);
    TezClient tezClient = TezClient.create(SortMergeJoinHelper.class.getSimpleName(), conf);
    tezClient.start();
    SortMergeJoinHelper sortMergeJoinHelper = new SortMergeJoinHelper(tezClient);
    sortMergeJoinHelper.setConf(conf);
    String[] args = new String[] { "-D" + TezConfiguration.TEZ_AM_STAGING_DIR + "=" + stagingDirPath.toString(), "-counter", inPath1.toString(), inPath2.toString(), "1", outPath.toString() };
    assertEquals(0, sortMergeJoinHelper.run(conf, args, tezClient));
    verifySortMergeJoinInput(outPath, expectedResults);
    String joinerVertexName = "joiner";
    String input1Name = "input1";
    String input2Name = "input2";
    String joinOutputName = "joinOutput";
    Set<StatusGetOpts> statusOpts = new HashSet<StatusGetOpts>();
    statusOpts.add(StatusGetOpts.GET_COUNTERS);
    VertexStatus joinerVertexStatus = sortMergeJoinHelper.dagClient.getVertexStatus(joinerVertexName, statusOpts);
    final TezCounters joinerCounters = joinerVertexStatus.getVertexCounters();
    final CounterGroup aggregatedGroup = joinerCounters.getGroup(TaskCounter.class.getCanonicalName());
    final CounterGroup input1Group = joinerCounters.getGroup(TaskCounter.class.getSimpleName() + "_" + joinerVertexName + "_INPUT_" + input1Name);
    final CounterGroup input2Group = joinerCounters.getGroup(TaskCounter.class.getSimpleName() + "_" + joinerVertexName + "_INPUT_" + input2Name);
    assertTrue("aggregated counter group cannot be empty", aggregatedGroup.size() > 0);
    assertTrue("per io group for input1 cannot be empty", input1Group.size() > 0);
    assertTrue("per io group for input1 cannot be empty", input2Group.size() > 0);
    List<TaskCounter> countersToVerifyAgg = Arrays.asList(TaskCounter.ADDITIONAL_SPILLS_BYTES_READ, TaskCounter.ADDITIONAL_SPILLS_BYTES_WRITTEN, TaskCounter.COMBINE_INPUT_RECORDS, TaskCounter.MERGED_MAP_OUTPUTS, TaskCounter.NUM_DISK_TO_DISK_MERGES, TaskCounter.NUM_FAILED_SHUFFLE_INPUTS, TaskCounter.NUM_MEM_TO_DISK_MERGES, TaskCounter.NUM_SHUFFLED_INPUTS, TaskCounter.NUM_SKIPPED_INPUTS, TaskCounter.REDUCE_INPUT_GROUPS, TaskCounter.REDUCE_INPUT_RECORDS, TaskCounter.SHUFFLE_BYTES, TaskCounter.SHUFFLE_BYTES_DECOMPRESSED, TaskCounter.SHUFFLE_BYTES_DISK_DIRECT, TaskCounter.SHUFFLE_BYTES_TO_DISK, TaskCounter.SHUFFLE_BYTES_TO_MEM, TaskCounter.SPILLED_RECORDS);
    int nonZeroCounters = 0;
    // verify that the sum of the counter values for edges add up to the aggregated counter value.
    for (TaskCounter c : countersToVerifyAgg) {
        TezCounter aggregatedCounter = aggregatedGroup.findCounter(c.name(), false);
        TezCounter input1Counter = input1Group.findCounter(c.name(), false);
        TezCounter input2Counter = input2Group.findCounter(c.name(), false);
        assertNotNull("aggregated counter cannot be null " + c.name(), aggregatedCounter);
        assertNotNull("input1 counter cannot be null " + c.name(), input1Counter);
        assertNotNull("input2 counter cannot be null " + c.name(), input2Counter);
        assertEquals("aggregated counter does not match sum of input counters " + c.name(), aggregatedCounter.getValue(), input1Counter.getValue() + input2Counter.getValue());
        if (aggregatedCounter.getValue() > 0) {
            nonZeroCounters++;
        }
    }
    // ensure that at least one of the counters tested above were non-zero.
    assertTrue("At least one of the counter should be non-zero. invalid test ", nonZeroCounters > 0);
    CounterGroup joinerOutputGroup = joinerCounters.getGroup(TaskCounter.class.getSimpleName() + "_" + joinerVertexName + "_OUTPUT_" + joinOutputName);
    String outputCounterName = TaskCounter.OUTPUT_RECORDS.name();
    TezCounter aggregateCounter = aggregatedGroup.findCounter(outputCounterName, false);
    TezCounter joinerOutputCounter = joinerOutputGroup.findCounter(outputCounterName, false);
    assertNotNull("aggregated counter cannot be null " + outputCounterName, aggregateCounter);
    assertNotNull("output counter cannot be null " + outputCounterName, joinerOutputCounter);
    assertTrue("counter value is zero. test is invalid", aggregateCounter.getValue() > 0);
    assertEquals("aggregated counter does not match sum of output counters " + outputCounterName, aggregateCounter.getValue(), joinerOutputCounter.getValue());
}

Example 14 with CounterGroup

use of org.apache.tez.common.counters.CounterGroup in project hive by apache.

the class WmFragmentCounters method dumpToTezCounters.

public void dumpToTezCounters(TezCounters tezCounters, boolean isLast) {
    if (isLast) {
        // Record the final counters.
        changeStateDone();
    }
    for (int i = 0; i < fixedCounters.length(); ++i) {
        tezCounters.findCounter(LlapWmCounters.values()[i]).setValue(fixedCounters.get(i));
    }
    // to the Tez counters.
    if (addTaskTimeCounters) {
        String hostName = MetricsUtils.getHostName();
        long queued = fixedCounters.get(LlapWmCounters.GUARANTEED_QUEUED_NS.ordinal()) + fixedCounters.get(LlapWmCounters.SPECULATIVE_QUEUED_NS.ordinal());
        long running = fixedCounters.get(LlapWmCounters.GUARANTEED_RUNNING_NS.ordinal()) + fixedCounters.get(LlapWmCounters.SPECULATIVE_RUNNING_NS.ordinal());
        CounterGroup cg = tezCounters.getGroup("LlapTaskRuntimeAgg by daemon");
        cg.findCounter("QueueTime-" + hostName).setValue(queued);
        cg.findCounter("RunTime-" + hostName).setValue(running);
        cg.findCounter("Count-" + hostName).setValue(1);
    }
}

Example 15 with CounterGroup

use of org.apache.tez.common.counters.CounterGroup in project hive by apache.

the class PostExecOrcRowGroupCountPrinter method run.

@Override
public void run(HookContext hookContext) throws Exception {
    assert (hookContext.getHookType() == HookContext.HookType.POST_EXEC_HOOK);
    HiveConf conf = hookContext.getConf();
    if (!"tez".equals(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE))) {
        return;
    }
    LOG.info("Executing post execution hook to print ORC row groups read counter..");
    SessionState ss = SessionState.get();
    SessionState.LogHelper console = ss.getConsole();
    QueryPlan plan = hookContext.getQueryPlan();
    if (plan == null) {
        return;
    }
    List<TezTask> rootTasks = Utilities.getTezTasks(plan.getRootTasks());
    for (TezTask tezTask : rootTasks) {
        LOG.info("Printing ORC row group counter for tez task: " + tezTask.getName());
        TezCounters counters = tezTask.getTezCounters();
        if (counters != null) {
            for (CounterGroup group : counters) {
                if (group.getName().equals(LlapIOCounters.class.getName())) {
                    console.printInfo(tezTask.getId() + " LLAP IO COUNTERS:", false);
                    for (TezCounter counter : group) {
                        if (counter.getDisplayName().equals(LlapIOCounters.SELECTED_ROWGROUPS.name())) {
                            console.printInfo("   " + counter.getDisplayName() + ": " + counter.getValue(), false);
                        }
                    }
                }
            }
        }
    }
}