Examples with TaskCounter org.apache.hadoop.mapreduce.TaskCounter used on opensource projects

Example 1 with TaskCounter

use of org.apache.hadoop.mapreduce.TaskCounter in project hadoop by apache.

the class MockJobs method newCounters.

public static Counters newCounters() {
    Counters hc = new Counters();
    for (JobCounter c : JobCounter.values()) {
        hc.findCounter(c).setValue((long) (Math.random() * 1000));
    }
    for (TaskCounter c : TaskCounter.values()) {
        hc.findCounter(c).setValue((long) (Math.random() * 1000));
    }
    int nc = FileSystemCounter.values().length * 4;
    for (int i = 0; i < nc; ++i) {
        for (FileSystemCounter c : FileSystemCounter.values()) {
            hc.findCounter(FS_SCHEMES.next(), c).setValue((long) (Math.random() * DT));
        }
    }
    for (int i = 0; i < 2 * 3; ++i) {
        hc.findCounter(USER_COUNTER_GROUPS.next(), USER_COUNTERS.next()).setValue((long) (Math.random() * 100000));
    }
    return hc;
}

Example 2 with TaskCounter

use of org.apache.hadoop.mapreduce.TaskCounter in project hadoop by apache.

the class Task method updateResourceCounters.

/**
   * Update resource information counters
   */
void updateResourceCounters() {
    // Update generic resource counters
    updateHeapUsageCounter();
    // Updating resources specified in ResourceCalculatorProcessTree
    if (pTree == null) {
        return;
    }
    pTree.updateProcessTree();
    long cpuTime = pTree.getCumulativeCpuTime();
    long pMem = pTree.getRssMemorySize();
    long vMem = pTree.getVirtualMemorySize();
    // Remove the CPU time consumed previously by JVM reuse
    if (cpuTime != ResourceCalculatorProcessTree.UNAVAILABLE && initCpuCumulativeTime != ResourceCalculatorProcessTree.UNAVAILABLE) {
        cpuTime -= initCpuCumulativeTime;
    }
    if (cpuTime != ResourceCalculatorProcessTree.UNAVAILABLE) {
        counters.findCounter(TaskCounter.CPU_MILLISECONDS).setValue(cpuTime);
    }
    if (pMem != ResourceCalculatorProcessTree.UNAVAILABLE) {
        counters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).setValue(pMem);
    }
    if (vMem != ResourceCalculatorProcessTree.UNAVAILABLE) {
        counters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).setValue(vMem);
    }
    if (pMem != ResourceCalculatorProcessTree.UNAVAILABLE) {
        TaskCounter counter = isMapTask() ? TaskCounter.MAP_PHYSICAL_MEMORY_BYTES_MAX : TaskCounter.REDUCE_PHYSICAL_MEMORY_BYTES_MAX;
        Counters.Counter pMemCounter = counters.findCounter(counter);
        pMemCounter.setValue(Math.max(pMemCounter.getValue(), pMem));
    }
    if (vMem != ResourceCalculatorProcessTree.UNAVAILABLE) {
        TaskCounter counter = isMapTask() ? TaskCounter.MAP_VIRTUAL_MEMORY_BYTES_MAX : TaskCounter.REDUCE_VIRTUAL_MEMORY_BYTES_MAX;
        Counters.Counter vMemCounter = counters.findCounter(counter);
        vMemCounter.setValue(Math.max(vMemCounter.getValue(), vMem));
    }
}