Examples with TezCounters org.apache.tez.common.counters.TezCounters used on opensource projects

Example 66 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class VertexImpl method constructFinalFullcounters.

@Private
public void constructFinalFullcounters() {
    this.fullCounters = new TezCounters();
    this.fullCounters.incrAllCounters(counters);
    this.vertexStats = new VertexStats();
    for (Task t : this.tasks.values()) {
        vertexStats.updateStats(t.getReport());
        TezCounters counters = t.getCounters();
        this.fullCounters.incrAllCounters(counters);
    }
}

Example 67 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class VertexImpl method getCachedCounters.

@Override
public TezCounters getCachedCounters() {
    readLock.lock();
    try {
        // FIXME a better lightweight approach for counters is needed
        if (fullCounters == null && cachedCounters != null && ((cachedCountersTimestamp + 10000) > System.currentTimeMillis())) {
            LOG.info("Asked for counters" + ", cachedCountersTimestamp=" + cachedCountersTimestamp + ", currentTime=" + System.currentTimeMillis());
            return cachedCounters;
        }
        cachedCountersTimestamp = System.currentTimeMillis();
        if (inTerminalState()) {
            this.mayBeConstructFinalFullCounters();
            return fullCounters;
        }
        TezCounters counters = new TezCounters();
        counters.incrAllCounters(this.counters);
        cachedCounters = incrTaskCounters(counters, tasks.values());
        return cachedCounters;
    } finally {
        readLock.unlock();
    }
}

Example 68 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class DAGImpl method constructFinalFullcounters.

@Private
public void constructFinalFullcounters() {
    this.fullCounters = new TezCounters();
    this.fullCounters.incrAllCounters(dagCounters);
    for (Vertex v : this.vertices.values()) {
        this.fullCounters.incrAllCounters(v.getAllCounters());
    }
}

Example 69 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestAMRecovery method testVertexPartiallyFinished_ScatterGather.

/**
 * Fine-grained recovery task-level, In a vertex (v1), task 0 is done task 1
 * is not started. History flush happens. AM dies. Once AM is recovered, task 0 is
 * not re-run. Task 1 is re-run. (SCATTER_GATHER)
 *
 * @throws Exception
 */
@Test(timeout = 120000)
public void testVertexPartiallyFinished_ScatterGather() throws Exception {
    DAG dag = createDAG("VertexPartiallyFinished_ScatterGather", ControlledShuffleVertexManager.class, DataMovementType.SCATTER_GATHER, true);
    TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
    assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
    assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());
    List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
    List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
    printHistoryEvents(historyEvents1, 1);
    printHistoryEvents(historyEvents1, 2);
    // task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
    // attempt 1
    assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
    assertEquals(0, findTaskAttemptFinishedEvent(historyEvents1, 0, 1).size());
    // task_0 of v1 is finished in attempt 1 and not rerun, task_1 of v1 is
    // finished in attempt 2
    assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 0).size());
    assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 1).size());
}

Example 70 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestAMRecovery method testVertexPartialFinished_One2One.

/**
 * Fine-grained recovery task-level, In a vertex (v1), task 0 is done task 1
 * is not started. History flush happens. AM dies. Once AM is recovered, task 0 is
 * not re-run. Task 1 is re-run. (ONE_TO_ONE)
 *
 * @throws Exception
 */
@Test(timeout = 120000)
public void testVertexPartialFinished_One2One() throws Exception {
    DAG dag = createDAG("VertexPartialFinished_One2One", ControlledInputReadyVertexManager.class, DataMovementType.ONE_TO_ONE, true);
    TezCounters counters = runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
    assertEquals(4, counters.findCounter(DAGCounter.NUM_SUCCEEDED_TASKS).getValue());
    assertEquals(2, counters.findCounter(TestCounter.Counter_1).getValue());
    List<HistoryEvent> historyEvents1 = readRecoveryLog(1);
    List<HistoryEvent> historyEvents2 = readRecoveryLog(2);
    printHistoryEvents(historyEvents1, 1);
    printHistoryEvents(historyEvents1, 2);
    // task_0 of v1 is finished in attempt 1, task_1 of v1 is not finished in
    // attempt 1
    assertEquals(1, findTaskAttemptFinishedEvent(historyEvents1, 0, 0).size());
    assertEquals(0, findTaskAttemptFinishedEvent(historyEvents1, 0, 1).size());
    // task_0 of v1 is finished in attempt 1 and not rerun, task_1 of v1 is
    // finished in attempt 2
    assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 0).size());
    assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 1).size());
}