Examples with TezCounters org.apache.tez.common.counters.TezCounters used on opensource projects

Example 61 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestTaskReporter method testStatusUpdateAfterInitializationAndCounterFlag.

@Test(timeout = 5000)
public void testStatusUpdateAfterInitializationAndCounterFlag() {
    TezTaskAttemptID mockTaskAttemptId = mock(TezTaskAttemptID.class);
    LogicalIOProcessorRuntimeTask mockTask = mock(LogicalIOProcessorRuntimeTask.class);
    doReturn("vertexName").when(mockTask).getVertexName();
    doReturn(mockTaskAttemptId).when(mockTask).getTaskAttemptID();
    boolean progressNotified = false;
    doReturn(progressNotified).when(mockTask).getAndClearProgressNotification();
    TezTaskUmbilicalProtocol mockUmbilical = mock(TezTaskUmbilicalProtocol.class);
    float progress = 0.5f;
    TaskStatistics stats = new TaskStatistics();
    TezCounters counters = new TezCounters();
    doReturn(progress).when(mockTask).getProgress();
    doReturn(stats).when(mockTask).getTaskStatistics();
    doReturn(counters).when(mockTask).getCounters();
    // Setup the sleep time to be way higher than the test timeout
    TaskReporter.HeartbeatCallable heartbeatCallable = new TaskReporter.HeartbeatCallable(mockTask, mockUmbilical, 100000, 100000, 5, new AtomicLong(0), "containerIdStr");
    // task not initialized - nothing obtained from task
    doReturn(false).when(mockTask).hasInitialized();
    TaskStatusUpdateEvent event = heartbeatCallable.getStatusUpdateEvent(true);
    verify(mockTask, times(1)).hasInitialized();
    verify(mockTask, times(0)).getProgress();
    verify(mockTask, times(0)).getAndClearProgressNotification();
    verify(mockTask, times(0)).getTaskStatistics();
    verify(mockTask, times(0)).getCounters();
    Assert.assertEquals(0, event.getProgress(), 0);
    Assert.assertEquals(false, event.getProgressNotified());
    Assert.assertNull(event.getCounters());
    Assert.assertNull(event.getStatistics());
    // task is initialized - progress obtained but not counters since flag is false
    doReturn(true).when(mockTask).hasInitialized();
    event = heartbeatCallable.getStatusUpdateEvent(false);
    verify(mockTask, times(2)).hasInitialized();
    verify(mockTask, times(1)).getProgress();
    verify(mockTask, times(1)).getAndClearProgressNotification();
    verify(mockTask, times(0)).getTaskStatistics();
    verify(mockTask, times(0)).getCounters();
    Assert.assertEquals(progress, event.getProgress(), 0);
    Assert.assertEquals(progressNotified, event.getProgressNotified());
    Assert.assertNull(event.getCounters());
    Assert.assertNull(event.getStatistics());
    // task is initialized - progress obtained and also counters since flag is true
    progressNotified = true;
    doReturn(progressNotified).when(mockTask).getAndClearProgressNotification();
    doReturn(true).when(mockTask).hasInitialized();
    event = heartbeatCallable.getStatusUpdateEvent(true);
    verify(mockTask, times(3)).hasInitialized();
    verify(mockTask, times(2)).getProgress();
    verify(mockTask, times(2)).getAndClearProgressNotification();
    verify(mockTask, times(1)).getTaskStatistics();
    verify(mockTask, times(1)).getCounters();
    Assert.assertEquals(progress, event.getProgress(), 0);
    Assert.assertEquals(progressNotified, event.getProgressNotified());
    Assert.assertEquals(counters, event.getCounters());
    Assert.assertEquals(stats, event.getStatistics());
}

Example 62 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class RuntimeTask method getCounters.

public TezCounters getCounters() {
    TezCounters fullCounters = new TezCounters();
    fullCounters.incrAllCounters(tezCounters);
    for (TezCounters counter : counterMap.values()) {
        fullCounters.incrAllCounters(counter);
    }
    return fullCounters;
}

Example 63 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class RuntimeTask method addAndGetTezCounter.

public TezCounters addAndGetTezCounter(String name) {
    TezCounters counter = new TezCounters();
    counterMap.put(name, counter);
    return counter;
}

Example 64 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TaskImpl method getCounters.

@Override
public TezCounters getCounters() {
    TezCounters counters = new TezCounters();
    counters.incrAllCounters(this.counters);
    readLock.lock();
    try {
        TaskAttempt bestAttempt = selectBestAttempt();
        if (bestAttempt != null) {
            counters.incrAllCounters(bestAttempt.getCounters());
        }
        return counters;
    } finally {
        readLock.unlock();
    }
}

Example 65 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class VertexImpl method logJobHistoryVertexFailedEvent.

void logJobHistoryVertexFailedEvent(VertexState state) throws IOException {
    if (recoveryData == null || !recoveryData.isVertexFinished()) {
        TezCounters counters = null;
        try {
            counters = getAllCounters();
        } catch (LimitExceededException e) {
            // Ignore as failed vertex
            addDiagnostic("Counters limit exceeded: " + e.getMessage());
        }
        logJobHistoryVertexCompletedHelper(state, clock.getTime(), StringUtils.join(getDiagnostics(), LINE_SEPARATOR), counters);
    }
}