Examples with TezCounters org.apache.tez.common.counters.TezCounters used on opensource projects

Example 51 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestDAGImpl method testCounterLimits.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testCounterLimits() {
    initDAG(mrrDag);
    dispatcher.await();
    startDAG(mrrDag);
    dispatcher.await();
    for (int i = 0; i < 3; ++i) {
        Vertex v = mrrDag.getVertex("vertex" + (i + 1));
        TezCounters ctrs = new TezCounters();
        for (int j = 0; j < 50; ++j) {
            ctrs.findCounter("g", "c" + i + "_" + j).increment(1);
        }
        ((VertexImpl) v).setCounters(ctrs);
        dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
        dispatcher.await();
        Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
        Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices());
    }
    Assert.assertEquals(3, mrrDag.getSuccessfulVertices());
    Assert.assertEquals(DAGState.FAILED, mrrDag.getState());
    Assert.assertTrue("Diagnostics should contain counter limits error message", StringUtils.join(mrrDag.getDiagnostics(), ",").contains("Counters limit exceeded"));
}

Example 52 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestMockDAGAppMaster method testBasicCounterMemory.

@Ignore
@Test(timeout = 60000)
public void testBasicCounterMemory() throws Exception {
    Logger.getRootLogger().setLevel(Level.WARN);
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
    tezClient.start();
    final String vAName = "A";
    DAG dag = DAG.create("testBasicCounterMemory");
    Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 10000);
    dag.addVertex(vA);
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.countersDelegate = new CountersDelegate() {

        @Override
        public TezCounters getCounters(TaskSpec taskSpec) {
            TezCounters counters = new TezCounters();
            final String longName = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz";
            final String shortName = "abcdefghijklmnopqrstuvwxyz";
            for (int i = 0; i < 6; ++i) {
                for (int j = 0; j < 15; ++j) {
                    counters.findCounter((i + longName), (i + (shortName))).increment(1);
                }
            }
            return counters;
        }
    };
    mockApp.doSleep = false;
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    DAGStatus status = dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
    TezCounters counters = dagImpl.getAllCounters();
    Assert.assertNotNull(counters);
    checkMemory(dag.getName(), mockApp);
    tezClient.stop();
}

Example 53 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestAMWebController method createMockAttempt.

private TaskAttempt createMockAttempt(String attemptIDStr, TaskAttemptState status, float progress) {
    TaskAttempt mockAttempt = mock(TaskAttempt.class);
    doReturn(TezTaskAttemptID.fromString(attemptIDStr)).when(mockAttempt).getID();
    doReturn(status).when(mockAttempt).getState();
    doReturn(progress).when(mockAttempt).getProgress();
    TezCounters counters = new TezCounters();
    counters.addGroup("g1", "g1");
    counters.addGroup("g2", "g2");
    counters.addGroup("g3", "g3");
    counters.addGroup("g4", "g4");
    counters.findCounter("g1", "g1_c1").setValue(101);
    counters.findCounter("g1", "g1_c2").setValue(102);
    counters.findCounter("g2", "g2_c3").setValue(103);
    counters.findCounter("g2", "g2_c4").setValue(104);
    counters.findCounter("g3", "g3_c5").setValue(105);
    counters.findCounter("g3", "g3_c6").setValue(106);
    doReturn(counters).when(mockAttempt).getCounters();
    return mockAttempt;
}

Example 54 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestAMWebController method createMockTask.

private Task createMockTask(String taskIDStr, TaskState status, float progress) {
    Task mockTask = mock(Task.class);
    doReturn(TezTaskID.fromString(taskIDStr)).when(mockTask).getTaskId();
    doReturn(status).when(mockTask).getState();
    doReturn(progress).when(mockTask).getProgress();
    TezCounters counters = new TezCounters();
    counters.addGroup("g1", "g1");
    counters.addGroup("g2", "g2");
    counters.addGroup("g3", "g3");
    counters.addGroup("g4", "g4");
    counters.findCounter("g1", "g1_c1").setValue(101);
    counters.findCounter("g1", "g1_c2").setValue(102);
    counters.findCounter("g2", "g2_c3").setValue(103);
    counters.findCounter("g2", "g2_c4").setValue(104);
    counters.findCounter("g3", "g3_c5").setValue(105);
    counters.findCounter("g3", "g3_c6").setValue(106);
    doReturn(counters).when(mockTask).getCounters();
    return mockTask;
}

Example 55 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestAMWebController method createMockVertex.

private Vertex createMockVertex(String vertexIDStr, VertexState status, float progress, int taskCounts) {
    ProgressBuilder pb = new ProgressBuilder();
    pb.setTotalTaskCount(taskCounts);
    pb.setSucceededTaskCount(taskCounts * 2);
    pb.setFailedTaskAttemptCount(taskCounts * 3);
    pb.setKilledTaskAttemptCount(taskCounts * 4);
    pb.setRunningTaskCount(taskCounts * 5);
    Vertex mockVertex = mock(Vertex.class);
    doReturn(TezVertexID.fromString(vertexIDStr)).when(mockVertex).getVertexId();
    doReturn(status).when(mockVertex).getState();
    doReturn(progress).when(mockVertex).getProgress();
    doReturn(pb).when(mockVertex).getVertexProgress();
    doReturn(1L).when(mockVertex).getInitTime();
    doReturn(1L).when(mockVertex).getStartTime();
    doReturn(2L).when(mockVertex).getFinishTime();
    doReturn(1L).when(mockVertex).getFirstTaskStartTime();
    doReturn(2L).when(mockVertex).getLastTaskFinishTime();
    TezCounters counters = new TezCounters();
    counters.addGroup("g1", "g1");
    counters.addGroup("g2", "g2");
    counters.addGroup("g3", "g3");
    counters.addGroup("g4", "g4");
    counters.findCounter("g1", "g1_c1").setValue(100);
    counters.findCounter("g1", "g1_c2").setValue(100);
    counters.findCounter("g2", "g2_c3").setValue(100);
    counters.findCounter("g2", "g2_c4").setValue(100);
    counters.findCounter("g3", "g3_c5").setValue(100);
    counters.findCounter("g3", "g3_c6").setValue(100);
    doReturn(counters).when(mockVertex).getAllCounters();
    doReturn(counters).when(mockVertex).getCachedCounters();
    return mockVertex;
}