Examples with LimitExceededException org.apache.tez.common.counters.LimitExceededException used on opensource projects

Example 1 with LimitExceededException

use of org.apache.tez.common.counters.LimitExceededException in project tez by apache.

the class DAGImpl method finished.

private DAGState finished(DAGState finalState) {
    if (finishTime == 0) {
        setFinishTime();
    }
    entityUpdateTracker.stop();
    boolean recoveryError = false;
    // update cpu time counters before finishing the dag
    updateCpuCounters();
    TezCounters counters = null;
    try {
        counters = getAllCounters();
    } catch (LimitExceededException e) {
        addDiagnostic("Counters limit exceeded: " + e.getMessage());
        finalState = DAGState.FAILED;
    }
    try {
        if (finalState == DAGState.SUCCEEDED) {
            logJobHistoryFinishedEvent(counters);
        } else {
            logJobHistoryUnsuccesfulEvent(finalState, counters);
        }
    } catch (IOException e) {
        LOG.warn("Failed to persist recovery event for DAG completion" + ", dagId=" + dagId + ", finalState=" + finalState);
        recoveryError = true;
    }
    if (finalState != DAGState.SUCCEEDED) {
        abortOutputs();
    }
    if (recoveryError) {
        eventHandler.handle(new DAGAppMasterEventDAGFinished(getID(), DAGState.ERROR));
    } else {
        eventHandler.handle(new DAGAppMasterEventDAGFinished(getID(), finalState));
    }
    LOG.info("DAG: " + getID() + " finished with state: " + finalState);
    return finalState;
}

Example 2 with LimitExceededException

use of org.apache.tez.common.counters.LimitExceededException in project tez by apache.

the class AMWebController method getDagInfo.

public void getDagInfo() {
    if (!setupResponse()) {
        return;
    }
    DAG dag = checkAndGetDAGFromRequest();
    if (dag == null) {
        return;
    }
    Map<String, Set<String>> counterNames = getCounterListFromRequest();
    Map<String, Object> dagInfo = new HashMap<String, Object>();
    dagInfo.put("id", dag.getID().toString());
    dagInfo.put("progress", Float.toString(dag.getCompletedTaskProgress()));
    dagInfo.put("status", dag.getState().toString());
    try {
        if (counterNames != null && !counterNames.isEmpty()) {
            TezCounters counters = dag.getCachedCounters();
            Map<String, Map<String, Long>> counterMap = constructCounterMapInfo(counters, counterNames);
            if (counterMap != null && !counterMap.isEmpty()) {
                dagInfo.put("counters", counterMap);
            }
        }
    } catch (LimitExceededException e) {
    // Ignore
    // TODO: add an error message instead for counter key
    }
    renderJSON(ImmutableMap.of("dag", dagInfo));
}

Example 3 with LimitExceededException

use of org.apache.tez.common.counters.LimitExceededException in project tez by apache.

the class AMWebController method getVertexInfoMap.

private Map<String, Object> getVertexInfoMap(Vertex vertex, Map<String, Set<String>> counterNames) {
    Map<String, Object> vertexInfo = new HashMap<String, Object>();
    vertexInfo.put("id", vertex.getVertexId().toString());
    vertexInfo.put("status", vertex.getState().toString());
    vertexInfo.put("progress", Float.toString(vertex.getCompletedTaskProgress()));
    vertexInfo.put("initTime", Long.toString(vertex.getInitTime()));
    vertexInfo.put("startTime", Long.toString(vertex.getStartTime()));
    vertexInfo.put("finishTime", Long.toString(vertex.getFinishTime()));
    vertexInfo.put("firstTaskStartTime", Long.toString(vertex.getFirstTaskStartTime()));
    vertexInfo.put("lastTaskFinishTime", Long.toString(vertex.getLastTaskFinishTime()));
    ProgressBuilder vertexProgress = vertex.getVertexProgress();
    vertexInfo.put("totalTasks", Integer.toString(vertexProgress.getTotalTaskCount()));
    vertexInfo.put("runningTasks", Integer.toString(vertexProgress.getRunningTaskCount()));
    vertexInfo.put("succeededTasks", Integer.toString(vertexProgress.getSucceededTaskCount()));
    vertexInfo.put("failedTaskAttempts", Integer.toString(vertexProgress.getFailedTaskAttemptCount()));
    vertexInfo.put("killedTaskAttempts", Integer.toString(vertexProgress.getKilledTaskAttemptCount()));
    try {
        if (counterNames != null && !counterNames.isEmpty()) {
            TezCounters counters = vertex.getCachedCounters();
            Map<String, Map<String, Long>> counterMap = constructCounterMapInfo(counters, counterNames);
            if (counterMap != null && !counterMap.isEmpty()) {
                vertexInfo.put("counters", counterMap);
            }
        }
    } catch (LimitExceededException e) {
    // Ignore
    // TODO: add an error message instead for counter key
    }
    return vertexInfo;
}

Example 4 with LimitExceededException

use of org.apache.tez.common.counters.LimitExceededException in project tez by apache.

the class AMWebController method getAttemptsInfo.

/**
 * Renders the response JSON for attemptsInfo API
 * The JSON will have an array of attempt objects under the key attempts.
 */
public void getAttemptsInfo() {
    if (!setupResponse()) {
        return;
    }
    DAG dag = checkAndGetDAGFromRequest();
    if (dag == null) {
        return;
    }
    int limit = MAX_QUERIED;
    try {
        limit = getQueryParamInt(WebUIService.LIMIT);
    } catch (NumberFormatException e) {
    // Ignore
    }
    List<TaskAttempt> attempts = getRequestedAttempts(dag, limit);
    if (attempts == null) {
        return;
    }
    Map<String, Set<String>> counterNames = getCounterListFromRequest();
    ArrayList<Map<String, Object>> attemptsInfo = new ArrayList<Map<String, Object>>();
    for (TaskAttempt a : attempts) {
        Map<String, Object> attemptInfo = new HashMap<String, Object>();
        attemptInfo.put("id", a.getID().toString());
        attemptInfo.put("progress", Float.toString(a.getProgress()));
        attemptInfo.put("status", a.getState().toString());
        try {
            TezCounters counters = a.getCounters();
            Map<String, Map<String, Long>> counterMap = constructCounterMapInfo(counters, counterNames);
            if (counterMap != null && !counterMap.isEmpty()) {
                attemptInfo.put("counters", counterMap);
            }
        } catch (LimitExceededException e) {
        // Ignore
        // TODO: add an error message instead for counter key
        }
        attemptsInfo.add(attemptInfo);
    }
    renderJSON(ImmutableMap.of("attempts", attemptsInfo));
}

Example 5 with LimitExceededException

use of org.apache.tez.common.counters.LimitExceededException in project tez by apache.

the class AMWebController method getTasksInfo.

/**
 * Renders the response JSON for tasksInfo API
 * The JSON will have an array of task objects under the key tasks.
 */
public void getTasksInfo() {
    if (!setupResponse()) {
        return;
    }
    DAG dag = checkAndGetDAGFromRequest();
    if (dag == null) {
        return;
    }
    int limit = MAX_QUERIED;
    try {
        limit = getQueryParamInt(WebUIService.LIMIT);
    } catch (NumberFormatException e) {
    // Ignore
    }
    List<Task> tasks = getRequestedTasks(dag, limit);
    if (tasks == null) {
        return;
    }
    Map<String, Set<String>> counterNames = getCounterListFromRequest();
    ArrayList<Map<String, Object>> tasksInfo = new ArrayList<Map<String, Object>>();
    for (Task t : tasks) {
        Map<String, Object> taskInfo = new HashMap<String, Object>();
        taskInfo.put("id", t.getTaskId().toString());
        taskInfo.put("progress", Float.toString(t.getProgress()));
        taskInfo.put("status", t.getState().toString());
        try {
            TezCounters counters = t.getCounters();
            Map<String, Map<String, Long>> counterMap = constructCounterMapInfo(counters, counterNames);
            if (counterMap != null && !counterMap.isEmpty()) {
                taskInfo.put("counters", counterMap);
            }
        } catch (LimitExceededException e) {
        // Ignore
        // TODO: add an error message instead for counter key
        }
        tasksInfo.add(taskInfo);
    }
    renderJSON(ImmutableMap.of("tasks", tasksInfo));
}