Examples with TezCounters org.apache.tez.common.counters.TezCounters used on opensource projects

Example 46 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project hive by apache.

the class DAGSummary method hiveInputRecordsFromTezCounters.

private long hiveInputRecordsFromTezCounters(String vertexName, String inputVertexName) {
    // Get the counters for the input vertex.
    Set<StatusGetOpts> statusOptions = Collections.singleton(StatusGetOpts.GET_COUNTERS);
    VertexStatus inputVertexStatus = vertexStatus(statusOptions, inputVertexName);
    final TezCounters inputVertexCounters = inputVertexStatus.getVertexCounters();
    // eg, group name TaskCounter_Map_7_OUTPUT_Reducer_8, counter name OUTPUT_RECORDS
    String groupName = formattedName("TaskCounter", inputVertexName, vertexName);
    String counterName = "OUTPUT_RECORDS";
    // Do not create counter if it does not exist -
    // instead fall back to default behavior for determining input records.
    TezCounter tezCounter = inputVertexCounters.getGroup(groupName).findCounter(counterName, false);
    if (tezCounter == null) {
        return -1;
    } else {
        return tezCounter.getValue();
    }
}

Example 47 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project hive by apache.

the class DAGSummary method vertexSummary.

private String vertexSummary(String vertexName, Progress progress, VertexStatus vertexStatus) {
    /*
   * Get the CPU & GC
   *
   * counters org.apache.tez.common.counters.TaskCounter
   *  GC_TIME_MILLIS=37712
   *  CPU_MILLISECONDS=2774230
   */
    final TezCounters vertexCounters = vertexStatus.getVertexCounters();
    final double cpuTimeMillis = getCounterValueByGroupName(vertexCounters, TaskCounter.class.getName(), TaskCounter.CPU_MILLISECONDS.name());
    final double gcTimeMillis = getCounterValueByGroupName(vertexCounters, TaskCounter.class.getName(), TaskCounter.GC_TIME_MILLIS.name());
    /*
     * Get the HIVE counters
     *
     * HIVE
     *  CREATED_FILES=1
     *  DESERIALIZE_ERRORS=0
     *  RECORDS_IN_Map_1=550076554
     *  RECORDS_OUT_INTERMEDIATE_Map_1=854987
     *  RECORDS_OUT_Reducer_2=1
     */
    final long hiveInputRecords = hiveCounterValue(formattedName(MapOperator.Counter.RECORDS_IN.toString(), vertexName)) + hiveInputRecordsFromOtherVertices(vertexName);
    final long hiveOutputRecords = hiveCounterValue(formattedName(FileSinkOperator.Counter.RECORDS_OUT.toString(), vertexName)) + hiveCounterValue(formattedName(ReduceSinkOperator.Counter.RECORDS_OUT_INTERMEDIATE.toString(), vertexName));
    final double duration = perfLogger.getDuration(PerfLogger.TEZ_RUN_VERTEX + vertexName);
    return String.format(FORMATTING_PATTERN, vertexName, secondsFormatter.format((duration)), commaFormatter.format(cpuTimeMillis), commaFormatter.format(gcTimeMillis), commaFormatter.format(hiveInputRecords), commaFormatter.format(hiveOutputRecords));
}

Example 48 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project hive by apache.

the class LLAPioSummary method print.

@Override
public void print(SessionState.LogHelper console) {
    console.printInfo("");
    console.printInfo(LLAP_IO_SUMMARY_HEADER);
    SortedSet<String> keys = new TreeSet<>(progressMap.keySet());
    Set<StatusGetOpts> statusOptions = Collections.singleton(StatusGetOpts.GET_COUNTERS);
    String counterGroup = LlapIOCounters.class.getName();
    for (String vertexName : keys) {
        // Reducers do not benefit from LLAP IO so no point in printing
        if (vertexName.startsWith("Reducer")) {
            continue;
        }
        TezCounters vertexCounters = vertexCounter(statusOptions, vertexName);
        if (vertexCounters != null) {
            if (!first) {
                console.printInfo(SEPARATOR);
                console.printInfo(LLAP_SUMMARY_HEADER);
                console.printInfo(SEPARATOR);
                first = true;
            }
            console.printInfo(vertexSummary(vertexName, counterGroup, vertexCounters));
        }
    }
    console.printInfo(SEPARATOR);
    console.printInfo("");
}

Example 49 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project hive by apache.

the class FSCountersSummary method print.

@Override
public void print(SessionState.LogHelper console) {
    console.printInfo("FileSystem Counters Summary");
    SortedSet<String> keys = new TreeSet<>(progressMap.keySet());
    Set<StatusGetOpts> statusOptions = Collections.singleton(StatusGetOpts.GET_COUNTERS);
    // as well. If not, we need a way to get all the schemes that are accessed by the tez task/llap.
    for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) {
        final String scheme = statistics.getScheme().toUpperCase();
        console.printInfo("");
        console.printInfo("Scheme: " + scheme);
        console.printInfo(SEPARATOR);
        console.printInfo(HEADER);
        console.printInfo(SEPARATOR);
        for (String vertexName : keys) {
            TezCounters vertexCounters = vertexCounters(statusOptions, vertexName);
            if (vertexCounters != null) {
                console.printInfo(summary(scheme, vertexName, vertexCounters));
            }
        }
        console.printInfo(SEPARATOR);
    }
}

Example 50 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestVertexImpl method testCounterLimits.

@Test(timeout = 5000)
public void testCounterLimits() {
    initAllVertices(VertexState.INITED);
    VertexImpl v = vertices.get("vertex2");
    startVertex(v);
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
    for (int i = 0; i < 2; ++i) {
        TezCounters ctrs = new TezCounters();
        for (int j = 0; j < 75; ++j) {
            ctrs.findCounter("g", "c" + i + "_" + j).increment(1);
        }
        Task t = v.getTask(i);
        ((TaskImpl) t).setCounters(ctrs);
    }
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, v.getState());
    Assert.assertEquals(1, v.getCompletedTasks());
    Assert.assertTrue((0.5f) == v.getCompletedTaskProgress());
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v.getState());
    Assert.assertEquals(2, v.getCompletedTasks());
    System.out.println(v.getDiagnostics());
    Assert.assertTrue("Diagnostics should contain counter limits error message", StringUtils.join(v.getDiagnostics(), ",").contains("Counters limit exceeded"));
}