Examples with TezCounters org.apache.tez.common.counters.TezCounters used on opensource projects

Example 56 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class AMWebController method getAttemptsInfo.

/**
 * Renders the response JSON for attemptsInfo API
 * The JSON will have an array of attempt objects under the key attempts.
 */
public void getAttemptsInfo() {
    if (!setupResponse()) {
        return;
    }
    DAG dag = checkAndGetDAGFromRequest();
    if (dag == null) {
        return;
    }
    int limit = MAX_QUERIED;
    try {
        limit = getQueryParamInt(WebUIService.LIMIT);
    } catch (NumberFormatException e) {
    // Ignore
    }
    List<TaskAttempt> attempts = getRequestedAttempts(dag, limit);
    if (attempts == null) {
        return;
    }
    Map<String, Set<String>> counterNames = getCounterListFromRequest();
    ArrayList<Map<String, Object>> attemptsInfo = new ArrayList<Map<String, Object>>();
    for (TaskAttempt a : attempts) {
        Map<String, Object> attemptInfo = new HashMap<String, Object>();
        attemptInfo.put("id", a.getID().toString());
        attemptInfo.put("progress", Float.toString(a.getProgress()));
        attemptInfo.put("status", a.getState().toString());
        try {
            TezCounters counters = a.getCounters();
            Map<String, Map<String, Long>> counterMap = constructCounterMapInfo(counters, counterNames);
            if (counterMap != null && !counterMap.isEmpty()) {
                attemptInfo.put("counters", counterMap);
            }
        } catch (LimitExceededException e) {
        // Ignore
        // TODO: add an error message instead for counter key
        }
        attemptsInfo.add(attemptInfo);
    }
    renderJSON(ImmutableMap.of("attempts", attemptsInfo));
}

Example 57 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class AMWebController method getTasksInfo.

/**
 * Renders the response JSON for tasksInfo API
 * The JSON will have an array of task objects under the key tasks.
 */
public void getTasksInfo() {
    if (!setupResponse()) {
        return;
    }
    DAG dag = checkAndGetDAGFromRequest();
    if (dag == null) {
        return;
    }
    int limit = MAX_QUERIED;
    try {
        limit = getQueryParamInt(WebUIService.LIMIT);
    } catch (NumberFormatException e) {
    // Ignore
    }
    List<Task> tasks = getRequestedTasks(dag, limit);
    if (tasks == null) {
        return;
    }
    Map<String, Set<String>> counterNames = getCounterListFromRequest();
    ArrayList<Map<String, Object>> tasksInfo = new ArrayList<Map<String, Object>>();
    for (Task t : tasks) {
        Map<String, Object> taskInfo = new HashMap<String, Object>();
        taskInfo.put("id", t.getTaskId().toString());
        taskInfo.put("progress", Float.toString(t.getProgress()));
        taskInfo.put("status", t.getState().toString());
        try {
            TezCounters counters = t.getCounters();
            Map<String, Map<String, Long>> counterMap = constructCounterMapInfo(counters, counterNames);
            if (counterMap != null && !counterMap.isEmpty()) {
                taskInfo.put("counters", counterMap);
            }
        } catch (LimitExceededException e) {
        // Ignore
        // TODO: add an error message instead for counter key
        }
        tasksInfo.add(taskInfo);
    }
    renderJSON(ImmutableMap.of("tasks", tasksInfo));
}

Example 58 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestTezClient method testTezClientCounterLimits.

@Test(timeout = 5000)
public void testTezClientCounterLimits() throws YarnException, IOException, ServiceException {
    Limits.reset();
    int defaultCounterLimit = TezConfiguration.TEZ_COUNTERS_MAX_DEFAULT;
    int newCounterLimit = defaultCounterLimit + 500;
    TezConfiguration conf = new TezConfiguration();
    conf.setInt(TezConfiguration.TEZ_COUNTERS_MAX, newCounterLimit);
    configureAndCreateTezClient(conf);
    TezCounters counters = new TezCounters();
    for (int i = 0; i < newCounterLimit; i++) {
        counters.findCounter("GroupName", "TestCounter" + i).setValue(i);
    }
    try {
        counters.findCounter("GroupName", "TestCounterFail").setValue(1);
        fail("Expecting a LimitExceedException - too many counters");
    } catch (LimitExceededException e) {
    }
}

Example 59 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class DAGClientImpl method printDAGStatus.

private void printDAGStatus(Set<String> vertexNames, Set<StatusGetOpts> opts, DAGStatus dagStatus, Progress dagProgress) throws IOException, TezException {
    double vProgressFloat = 0.0f;
    log("DAG: State: " + dagStatus.getState() + " Progress: " + formatter.format(getProgress(dagProgress)) + " " + dagProgress);
    boolean displayCounter = opts != null && opts.contains(StatusGetOpts.GET_COUNTERS);
    if (displayCounter) {
        TezCounters counters = dagStatus.getDAGCounters();
        if (counters != null) {
            log("DAG Counters:\n" + counters);
        }
    }
    for (String vertex : vertexNames) {
        VertexStatus vStatus = getVertexStatus(vertex, opts);
        if (vStatus == null) {
            log("Could not retrieve status for vertex: " + vertex);
            continue;
        }
        Progress vProgress = vStatus.getProgress();
        if (vProgress != null) {
            vProgressFloat = 0.0f;
            if (vProgress.getTotalTaskCount() == 0) {
                vProgressFloat = 1.0f;
            } else if (vProgress.getTotalTaskCount() > 0) {
                vProgressFloat = getProgress(vProgress);
            }
            log("\tVertexStatus:" + " VertexName: " + vertex + " Progress: " + formatter.format(vProgressFloat) + " " + vProgress);
        }
        if (displayCounter) {
            TezCounters counters = vStatus.getVertexCounters();
            if (counters != null) {
                log("Vertex Counters for " + vertex + ":\n" + counters);
            }
        }
    }
// end of for loop
}

Example 60 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class DagTypeConverters method convertTezCountersFromProto.

public static TezCounters convertTezCountersFromProto(TezCountersProto proto) {
    TezCounters counters = new TezCounters();
    for (TezCounterGroupProto counterGroupProto : proto.getCounterGroupsList()) {
        CounterGroup group = counters.addGroup(counterGroupProto.getName(), counterGroupProto.getDisplayName());
        for (TezCounterProto counterProto : counterGroupProto.getCountersList()) {
            TezCounter counter = group.findCounter(counterProto.getName(), counterProto.getDisplayName());
            counter.setValue(counterProto.getValue());
        }
    }
    return counters;
}