Examples with VertexStatistics org.apache.tez.runtime.api.VertexStatistics used on opensource projects

Example 1 with VertexStatistics

use of org.apache.tez.runtime.api.VertexStatistics in project tez by apache.

the class TestVertexImpl method testCompletedStatsCache.

@Test(timeout = 5000)
public void testCompletedStatsCache() {
    initAllVertices(VertexState.INITED);
    VertexImpl v = vertices.get("vertex2");
    startVertex(v);
    TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
    dispatcher.getEventHandler().handle(new TaskEventTALaunched(TezTaskAttemptID.getInstance(t1, 0)));
    dispatcher.getEventHandler().handle(new TaskEventTASucceeded(TezTaskAttemptID.getInstance(t1, 0)));
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
    dispatcher.await();
    VertexStatistics stats = v.getStatistics();
    // Ensure that task 0 is available in completed stats cache
    Assert.assertTrue(v.completedTasksStatsCache.taskSet.get(0));
    // Reschedule task 0
    dispatcher.getEventHandler().handle(new VertexEventTaskReschedule(t1));
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, v.getState());
    // cache should be cleared
    Assert.assertTrue(v.completedTasksStatsCache.taskSet.cardinality() == 0);
}

Example 2 with VertexStatistics

use of org.apache.tez.runtime.api.VertexStatistics in project tez by apache.

the class TestMockDAGAppMaster method testBasicStatistics.

@Test(timeout = 10000)
public void testBasicStatistics() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
    tezClient.start();
    final String vAName = "A";
    final String vBName = "B";
    final String sourceName = "In";
    final String sinkName = "Out";
    DAG dag = DAG.create("testBasisStatistics");
    Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 3);
    Vertex vB = Vertex.create(vBName, ProcessorDescriptor.create("Proc.class"), 2);
    vA.addDataSource(sourceName, DataSourceDescriptor.create(InputDescriptor.create("In"), null, null));
    vB.addDataSink(sinkName, DataSinkDescriptor.create(OutputDescriptor.create("Out"), null, null));
    dag.addVertex(vA).addVertex(vB).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
    IOStatistics ioStats = new IOStatistics();
    ioStats.setDataSize(1);
    ioStats.setItemsProcessed(1);
    TaskStatistics vAStats = new TaskStatistics();
    vAStats.addIO(vBName, ioStats);
    vAStats.addIO(sourceName, ioStats);
    TaskStatistics vBStats = new TaskStatistics();
    vBStats.addIO(vAName, ioStats);
    vBStats.addIO(sinkName, ioStats);
    ByteArrayOutputStream bosA = new ByteArrayOutputStream();
    DataOutput outA = new DataOutputStream(bosA);
    vAStats.write(outA);
    final byte[] payloadA = bosA.toByteArray();
    ByteArrayOutputStream bosB = new ByteArrayOutputStream();
    DataOutput outB = new DataOutputStream(bosB);
    vBStats.write(outB);
    final byte[] payloadB = bosB.toByteArray();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.statsDelegate = new StatisticsDelegate() {

        @Override
        public TaskStatistics getStatistics(TaskSpec taskSpec) {
            byte[] payload = payloadA;
            TaskStatistics stats = new TaskStatistics();
            if (taskSpec.getVertexName().equals(vBName)) {
                payload = payloadB;
            }
            final DataInputByteBuffer in = new DataInputByteBuffer();
            in.reset(ByteBuffer.wrap(payload));
            try {
                // this ensures that the serde code path is covered.
                stats.readFields(in);
            } catch (IOException e) {
                Assert.fail(e.getMessage());
            }
            return stats;
        }
    };
    mockApp.doSleep = false;
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    DAGStatus status = dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
    // verify that the values have been correct aggregated
    for (org.apache.tez.dag.app.dag.Vertex v : dagImpl.getVertices().values()) {
        VertexStatistics vStats = v.getStatistics();
        if (v.getName().equals(vAName)) {
            Assert.assertEquals(3, vStats.getOutputStatistics(vBName).getDataSize());
            Assert.assertEquals(3, vStats.getInputStatistics(sourceName).getDataSize());
            Assert.assertEquals(3, vStats.getOutputStatistics(vBName).getItemsProcessed());
            Assert.assertEquals(3, vStats.getInputStatistics(sourceName).getItemsProcessed());
        } else {
            Assert.assertEquals(2, vStats.getInputStatistics(vAName).getDataSize());
            Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getDataSize());
            Assert.assertEquals(2, vStats.getInputStatistics(vAName).getItemsProcessed());
            Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getItemsProcessed());
        }
    }
    tezClient.stop();
}