Examples with ProgressBuilder org.apache.tez.dag.api.client.ProgressBuilder used on opensource projects

Example 1 with ProgressBuilder

use of org.apache.tez.dag.api.client.ProgressBuilder in project tez by apache.

the class DAGImpl method getDAGStatus.

// monitoring apis
@Override
public DAGStatusBuilder getDAGStatus(Set<StatusGetOpts> statusOptions) {
    DAGStatusBuilder status = new DAGStatusBuilder();
    int totalTaskCount = 0;
    int totalSucceededTaskCount = 0;
    int totalRunningTaskCount = 0;
    int totalFailedTaskCount = 0;
    int totalKilledTaskCount = 0;
    int totalFailedTaskAttemptCount = 0;
    int totalKilledTaskAttemptCount = 0;
    int totalRejectedTaskAttemptCount = 0;
    readLock.lock();
    try {
        for (Map.Entry<String, Vertex> entry : vertexMap.entrySet()) {
            ProgressBuilder progress = entry.getValue().getVertexProgress();
            status.addVertexProgress(entry.getKey(), progress);
            totalTaskCount += progress.getTotalTaskCount();
            totalSucceededTaskCount += progress.getSucceededTaskCount();
            totalRunningTaskCount += progress.getRunningTaskCount();
            totalFailedTaskCount += progress.getFailedTaskCount();
            totalKilledTaskCount += progress.getKilledTaskCount();
            totalFailedTaskAttemptCount += progress.getFailedTaskAttemptCount();
            totalKilledTaskAttemptCount += progress.getKilledTaskAttemptCount();
            totalRejectedTaskAttemptCount += progress.getRejectedTaskAttemptCount();
        }
        ProgressBuilder dagProgress = new ProgressBuilder();
        dagProgress.setTotalTaskCount(totalTaskCount);
        dagProgress.setSucceededTaskCount(totalSucceededTaskCount);
        dagProgress.setRunningTaskCount(totalRunningTaskCount);
        dagProgress.setFailedTaskCount(totalFailedTaskCount);
        dagProgress.setKilledTaskCount(totalKilledTaskCount);
        dagProgress.setFailedTaskAttemptCount(totalFailedTaskAttemptCount);
        dagProgress.setKilledTaskAttemptCount(totalKilledTaskAttemptCount);
        dagProgress.setRejectedTaskAttemptCount(totalRejectedTaskAttemptCount);
        status.setState(getState());
        status.setDiagnostics(diagnostics);
        status.setDAGProgress(dagProgress);
        if (statusOptions.contains(StatusGetOpts.GET_COUNTERS)) {
            status.setDAGCounters(getAllCounters());
        }
        return status;
    } finally {
        readLock.unlock();
    }
}

Example 2 with ProgressBuilder

use of org.apache.tez.dag.api.client.ProgressBuilder in project tez by apache.

the class DAGImpl method getDAGProgress.

private ProgressBuilder getDAGProgress() {
    int totalTaskCount = 0;
    int totalSucceededTaskCount = 0;
    int totalRunningTaskCount = 0;
    int totalFailedTaskCount = 0;
    int totalKilledTaskCount = 0;
    int totalFailedTaskAttemptCount = 0;
    int totalKilledTaskAttemptCount = 0;
    int totalRejectedTaskAttemptCount = 0;
    readLock.lock();
    try {
        for (Map.Entry<String, Vertex> entry : vertexMap.entrySet()) {
            ProgressBuilder progress = entry.getValue().getVertexProgress();
            totalTaskCount += progress.getTotalTaskCount();
            totalSucceededTaskCount += progress.getSucceededTaskCount();
            totalRunningTaskCount += progress.getRunningTaskCount();
            totalFailedTaskCount += progress.getFailedTaskCount();
            totalKilledTaskCount += progress.getKilledTaskCount();
            totalFailedTaskAttemptCount += progress.getFailedTaskAttemptCount();
            totalKilledTaskAttemptCount += progress.getKilledTaskAttemptCount();
            totalRejectedTaskAttemptCount += progress.getRejectedTaskAttemptCount();
        }
        ProgressBuilder dagProgress = new ProgressBuilder();
        dagProgress.setTotalTaskCount(totalTaskCount);
        dagProgress.setSucceededTaskCount(totalSucceededTaskCount);
        dagProgress.setRunningTaskCount(totalRunningTaskCount);
        dagProgress.setFailedTaskCount(totalFailedTaskCount);
        dagProgress.setKilledTaskCount(totalKilledTaskCount);
        dagProgress.setFailedTaskAttemptCount(totalFailedTaskAttemptCount);
        dagProgress.setKilledTaskAttemptCount(totalKilledTaskAttemptCount);
        dagProgress.setRejectedTaskAttemptCount(totalRejectedTaskAttemptCount);
        return dagProgress;
    } finally {
        readLock.unlock();
    }
}

Example 3 with ProgressBuilder

use of org.apache.tez.dag.api.client.ProgressBuilder in project tez by apache.

the class TestAMWebController method verifySingleVertexResult.

private void verifySingleVertexResult(Vertex mockVertex2, Map<String, String> vertex2Result) {
    ProgressBuilder progress;
    Assert.assertEquals(mockVertex2.getVertexId().toString(), vertex2Result.get("id"));
    Assert.assertEquals(mockVertex2.getState().toString(), vertex2Result.get("status"));
    Assert.assertEquals(Float.toString(mockVertex2.getCompletedTaskProgress()), vertex2Result.get("progress"));
    progress = mockVertex2.getVertexProgress();
    Assert.assertEquals(Integer.toString(progress.getTotalTaskCount()), vertex2Result.get("totalTasks"));
    Assert.assertEquals(Integer.toString(progress.getRunningTaskCount()), vertex2Result.get("runningTasks"));
    Assert.assertEquals(Integer.toString(progress.getSucceededTaskCount()), vertex2Result.get("succeededTasks"));
    Assert.assertEquals(Integer.toString(progress.getKilledTaskAttemptCount()), vertex2Result.get("killedTaskAttempts"));
    Assert.assertEquals(Integer.toString(progress.getFailedTaskAttemptCount()), vertex2Result.get("failedTaskAttempts"));
    String str0 = Long.toString(mockVertex2.getInitTime());
    String str1 = vertex2Result.get("initTime");
    Assert.assertEquals(Long.toString(mockVertex2.getInitTime()), vertex2Result.get("initTime"));
    Assert.assertEquals(Long.toString(mockVertex2.getStartTime()), vertex2Result.get("startTime"));
    Assert.assertEquals(Long.toString(mockVertex2.getFinishTime()), vertex2Result.get("finishTime"));
    Assert.assertEquals(Long.toString(mockVertex2.getFirstTaskStartTime()), vertex2Result.get("firstTaskStartTime"));
    Assert.assertEquals(Long.toString(mockVertex2.getLastTaskFinishTime()), vertex2Result.get("lastTaskFinishTime"));
}

Example 4 with ProgressBuilder

use of org.apache.tez.dag.api.client.ProgressBuilder in project tez by apache.

the class AMWebController method getVertexInfoMap.

private Map<String, Object> getVertexInfoMap(Vertex vertex, Map<String, Set<String>> counterNames) {
    Map<String, Object> vertexInfo = new HashMap<String, Object>();
    vertexInfo.put("id", vertex.getVertexId().toString());
    vertexInfo.put("status", vertex.getState().toString());
    vertexInfo.put("progress", Float.toString(vertex.getCompletedTaskProgress()));
    vertexInfo.put("initTime", Long.toString(vertex.getInitTime()));
    vertexInfo.put("startTime", Long.toString(vertex.getStartTime()));
    vertexInfo.put("finishTime", Long.toString(vertex.getFinishTime()));
    vertexInfo.put("firstTaskStartTime", Long.toString(vertex.getFirstTaskStartTime()));
    vertexInfo.put("lastTaskFinishTime", Long.toString(vertex.getLastTaskFinishTime()));
    ProgressBuilder vertexProgress = vertex.getVertexProgress();
    vertexInfo.put("totalTasks", Integer.toString(vertexProgress.getTotalTaskCount()));
    vertexInfo.put("runningTasks", Integer.toString(vertexProgress.getRunningTaskCount()));
    vertexInfo.put("succeededTasks", Integer.toString(vertexProgress.getSucceededTaskCount()));
    vertexInfo.put("failedTaskAttempts", Integer.toString(vertexProgress.getFailedTaskAttemptCount()));
    vertexInfo.put("killedTaskAttempts", Integer.toString(vertexProgress.getKilledTaskAttemptCount()));
    try {
        if (counterNames != null && !counterNames.isEmpty()) {
            TezCounters counters = vertex.getCachedCounters();
            Map<String, Map<String, Long>> counterMap = constructCounterMapInfo(counters, counterNames);
            if (counterMap != null && !counterMap.isEmpty()) {
                vertexInfo.put("counters", counterMap);
            }
        }
    } catch (LimitExceededException e) {
    // Ignore
    // TODO: add an error message instead for counter key
    }
    return vertexInfo;
}

Example 5 with ProgressBuilder

use of org.apache.tez.dag.api.client.ProgressBuilder in project tez by apache.

the class TestAMWebController method createMockVertex.

private Vertex createMockVertex(String vertexIDStr, VertexState status, float progress, int taskCounts) {
    ProgressBuilder pb = new ProgressBuilder();
    pb.setTotalTaskCount(taskCounts);
    pb.setSucceededTaskCount(taskCounts * 2);
    pb.setFailedTaskAttemptCount(taskCounts * 3);
    pb.setKilledTaskAttemptCount(taskCounts * 4);
    pb.setRunningTaskCount(taskCounts * 5);
    Vertex mockVertex = mock(Vertex.class);
    doReturn(TezVertexID.fromString(vertexIDStr)).when(mockVertex).getVertexId();
    doReturn(status).when(mockVertex).getState();
    doReturn(progress).when(mockVertex).getProgress();
    doReturn(pb).when(mockVertex).getVertexProgress();
    doReturn(1L).when(mockVertex).getInitTime();
    doReturn(1L).when(mockVertex).getStartTime();
    doReturn(2L).when(mockVertex).getFinishTime();
    doReturn(1L).when(mockVertex).getFirstTaskStartTime();
    doReturn(2L).when(mockVertex).getLastTaskFinishTime();
    TezCounters counters = new TezCounters();
    counters.addGroup("g1", "g1");
    counters.addGroup("g2", "g2");
    counters.addGroup("g3", "g3");
    counters.addGroup("g4", "g4");
    counters.findCounter("g1", "g1_c1").setValue(100);
    counters.findCounter("g1", "g1_c2").setValue(100);
    counters.findCounter("g2", "g2_c3").setValue(100);
    counters.findCounter("g2", "g2_c4").setValue(100);
    counters.findCounter("g3", "g3_c5").setValue(100);
    counters.findCounter("g3", "g3_c6").setValue(100);
    doReturn(counters).when(mockVertex).getAllCounters();
    doReturn(counters).when(mockVertex).getCachedCounters();
    return mockVertex;
}