Search in sources :

Example 1 with TaskStatistics

use of org.apache.tez.runtime.api.impl.TaskStatistics in project tez by apache.

the class TaskStatusUpdateEvent method readFields.

@Override
public void readFields(DataInput in) throws IOException {
    progress = in.readFloat();
    progressNotified = in.readBoolean();
    if (in.readBoolean()) {
        tezCounters = new TezCounters();
        tezCounters.readFields(in);
    }
    if (in.readBoolean()) {
        statistics = new TaskStatistics();
        statistics.readFields(in);
    }
}
Also used : TaskStatistics(org.apache.tez.runtime.api.impl.TaskStatistics) TezCounters(org.apache.tez.common.counters.TezCounters)

Example 2 with TaskStatistics

use of org.apache.tez.runtime.api.impl.TaskStatistics in project tez by apache.

the class TestMockDAGAppMaster method testBasicStatisticsMemory.

@Ignore
@Test(timeout = 60000)
public void testBasicStatisticsMemory() throws Exception {
    Logger.getRootLogger().setLevel(Level.WARN);
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
    tezClient.start();
    final String vAName = "abcdefghijklmnopqrstuvwxyz";
    int numTasks = 10000;
    int numSources = 10;
    IOStatistics ioStats = new IOStatistics();
    ioStats.setDataSize(1);
    ioStats.setItemsProcessed(1);
    TaskStatistics vAStats = new TaskStatistics();
    DAG dag = DAG.create("testBasicStatisticsMemory");
    Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), numTasks);
    for (int i = 0; i < numSources; ++i) {
        final String sourceName = i + vAName;
        vA.addDataSource(sourceName, DataSourceDescriptor.create(InputDescriptor.create(sourceName), null, null));
        vAStats.addIO(sourceName, ioStats);
    }
    dag.addVertex(vA);
    ByteArrayOutputStream bosA = new ByteArrayOutputStream();
    DataOutput outA = new DataOutputStream(bosA);
    vAStats.write(outA);
    final byte[] payloadA = bosA.toByteArray();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.statsDelegate = new StatisticsDelegate() {

        @Override
        public TaskStatistics getStatistics(TaskSpec taskSpec) {
            byte[] payload = payloadA;
            TaskStatistics stats = new TaskStatistics();
            final DataInputByteBuffer in = new DataInputByteBuffer();
            in.reset(ByteBuffer.wrap(payload));
            try {
                // this ensures that the serde code path is covered.
                stats.readFields(in);
            } catch (IOException e) {
                Assert.fail(e.getMessage());
            }
            return stats;
        }
    };
    mockApp.doSleep = false;
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    DAGStatus status = dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
    Assert.assertEquals(numTasks, dagImpl.getVertex(vAName).getStatistics().getInputStatistics(0 + vAName).getDataSize());
    Assert.assertEquals(numTasks, dagImpl.getVertex(vAName).getStatistics().getInputStatistics(0 + vAName).getItemsProcessed());
    checkMemory(dag.getName(), mockApp);
    tezClient.stop();
}
Also used : IOStatistics(org.apache.tez.runtime.api.impl.IOStatistics) Vertex(org.apache.tez.dag.api.Vertex) DataOutput(java.io.DataOutput) StatisticsDelegate(org.apache.tez.dag.app.MockDAGAppMaster.StatisticsDelegate) DataOutputStream(java.io.DataOutputStream) DataInputByteBuffer(org.apache.hadoop.io.DataInputByteBuffer) TaskSpec(org.apache.tez.runtime.api.impl.TaskSpec) DAG(org.apache.tez.dag.api.DAG) ByteArrayOutputStream(java.io.ByteArrayOutputStream) IOException(java.io.IOException) MockContainerLauncher(org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher) DAGImpl(org.apache.tez.dag.app.dag.impl.DAGImpl) TaskStatistics(org.apache.tez.runtime.api.impl.TaskStatistics) DAGClient(org.apache.tez.dag.api.client.DAGClient) DAGStatus(org.apache.tez.dag.api.client.DAGStatus) TezConfiguration(org.apache.tez.dag.api.TezConfiguration) Ignore(org.junit.Ignore) Test(org.junit.Test)

Example 3 with TaskStatistics

use of org.apache.tez.runtime.api.impl.TaskStatistics in project tez by apache.

the class TestMockDAGAppMaster method testBasicStatistics.

@Test(timeout = 10000)
public void testBasicStatistics() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
    tezClient.start();
    final String vAName = "A";
    final String vBName = "B";
    final String sourceName = "In";
    final String sinkName = "Out";
    DAG dag = DAG.create("testBasisStatistics");
    Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 3);
    Vertex vB = Vertex.create(vBName, ProcessorDescriptor.create("Proc.class"), 2);
    vA.addDataSource(sourceName, DataSourceDescriptor.create(InputDescriptor.create("In"), null, null));
    vB.addDataSink(sinkName, DataSinkDescriptor.create(OutputDescriptor.create("Out"), null, null));
    dag.addVertex(vA).addVertex(vB).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
    IOStatistics ioStats = new IOStatistics();
    ioStats.setDataSize(1);
    ioStats.setItemsProcessed(1);
    TaskStatistics vAStats = new TaskStatistics();
    vAStats.addIO(vBName, ioStats);
    vAStats.addIO(sourceName, ioStats);
    TaskStatistics vBStats = new TaskStatistics();
    vBStats.addIO(vAName, ioStats);
    vBStats.addIO(sinkName, ioStats);
    ByteArrayOutputStream bosA = new ByteArrayOutputStream();
    DataOutput outA = new DataOutputStream(bosA);
    vAStats.write(outA);
    final byte[] payloadA = bosA.toByteArray();
    ByteArrayOutputStream bosB = new ByteArrayOutputStream();
    DataOutput outB = new DataOutputStream(bosB);
    vBStats.write(outB);
    final byte[] payloadB = bosB.toByteArray();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.statsDelegate = new StatisticsDelegate() {

        @Override
        public TaskStatistics getStatistics(TaskSpec taskSpec) {
            byte[] payload = payloadA;
            TaskStatistics stats = new TaskStatistics();
            if (taskSpec.getVertexName().equals(vBName)) {
                payload = payloadB;
            }
            final DataInputByteBuffer in = new DataInputByteBuffer();
            in.reset(ByteBuffer.wrap(payload));
            try {
                // this ensures that the serde code path is covered.
                stats.readFields(in);
            } catch (IOException e) {
                Assert.fail(e.getMessage());
            }
            return stats;
        }
    };
    mockApp.doSleep = false;
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    DAGStatus status = dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
    // verify that the values have been correct aggregated
    for (org.apache.tez.dag.app.dag.Vertex v : dagImpl.getVertices().values()) {
        VertexStatistics vStats = v.getStatistics();
        if (v.getName().equals(vAName)) {
            Assert.assertEquals(3, vStats.getOutputStatistics(vBName).getDataSize());
            Assert.assertEquals(3, vStats.getInputStatistics(sourceName).getDataSize());
            Assert.assertEquals(3, vStats.getOutputStatistics(vBName).getItemsProcessed());
            Assert.assertEquals(3, vStats.getInputStatistics(sourceName).getItemsProcessed());
        } else {
            Assert.assertEquals(2, vStats.getInputStatistics(vAName).getDataSize());
            Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getDataSize());
            Assert.assertEquals(2, vStats.getInputStatistics(vAName).getItemsProcessed());
            Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getItemsProcessed());
        }
    }
    tezClient.stop();
}
Also used : IOStatistics(org.apache.tez.runtime.api.impl.IOStatistics) Vertex(org.apache.tez.dag.api.Vertex) DataOutput(java.io.DataOutput) StatisticsDelegate(org.apache.tez.dag.app.MockDAGAppMaster.StatisticsDelegate) DataOutputStream(java.io.DataOutputStream) DataInputByteBuffer(org.apache.hadoop.io.DataInputByteBuffer) MockContainerLauncher(org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher) TaskStatistics(org.apache.tez.runtime.api.impl.TaskStatistics) DAGStatus(org.apache.tez.dag.api.client.DAGStatus) TezConfiguration(org.apache.tez.dag.api.TezConfiguration) TaskSpec(org.apache.tez.runtime.api.impl.TaskSpec) DAG(org.apache.tez.dag.api.DAG) ByteArrayOutputStream(java.io.ByteArrayOutputStream) IOException(java.io.IOException) VertexStatistics(org.apache.tez.runtime.api.VertexStatistics) DAGImpl(org.apache.tez.dag.app.dag.impl.DAGImpl) DAGClient(org.apache.tez.dag.api.client.DAGClient) Test(org.junit.Test)

Example 4 with TaskStatistics

use of org.apache.tez.runtime.api.impl.TaskStatistics in project tez by apache.

the class TestTaskReporter method testStatusUpdateAfterInitializationAndCounterFlag.

@Test(timeout = 5000)
public void testStatusUpdateAfterInitializationAndCounterFlag() {
    TezTaskAttemptID mockTaskAttemptId = mock(TezTaskAttemptID.class);
    LogicalIOProcessorRuntimeTask mockTask = mock(LogicalIOProcessorRuntimeTask.class);
    doReturn("vertexName").when(mockTask).getVertexName();
    doReturn(mockTaskAttemptId).when(mockTask).getTaskAttemptID();
    boolean progressNotified = false;
    doReturn(progressNotified).when(mockTask).getAndClearProgressNotification();
    TezTaskUmbilicalProtocol mockUmbilical = mock(TezTaskUmbilicalProtocol.class);
    float progress = 0.5f;
    TaskStatistics stats = new TaskStatistics();
    TezCounters counters = new TezCounters();
    doReturn(progress).when(mockTask).getProgress();
    doReturn(stats).when(mockTask).getTaskStatistics();
    doReturn(counters).when(mockTask).getCounters();
    // Setup the sleep time to be way higher than the test timeout
    TaskReporter.HeartbeatCallable heartbeatCallable = new TaskReporter.HeartbeatCallable(mockTask, mockUmbilical, 100000, 100000, 5, new AtomicLong(0), "containerIdStr");
    // task not initialized - nothing obtained from task
    doReturn(false).when(mockTask).hasInitialized();
    TaskStatusUpdateEvent event = heartbeatCallable.getStatusUpdateEvent(true);
    verify(mockTask, times(1)).hasInitialized();
    verify(mockTask, times(0)).getProgress();
    verify(mockTask, times(0)).getAndClearProgressNotification();
    verify(mockTask, times(0)).getTaskStatistics();
    verify(mockTask, times(0)).getCounters();
    Assert.assertEquals(0, event.getProgress(), 0);
    Assert.assertEquals(false, event.getProgressNotified());
    Assert.assertNull(event.getCounters());
    Assert.assertNull(event.getStatistics());
    // task is initialized - progress obtained but not counters since flag is false
    doReturn(true).when(mockTask).hasInitialized();
    event = heartbeatCallable.getStatusUpdateEvent(false);
    verify(mockTask, times(2)).hasInitialized();
    verify(mockTask, times(1)).getProgress();
    verify(mockTask, times(1)).getAndClearProgressNotification();
    verify(mockTask, times(0)).getTaskStatistics();
    verify(mockTask, times(0)).getCounters();
    Assert.assertEquals(progress, event.getProgress(), 0);
    Assert.assertEquals(progressNotified, event.getProgressNotified());
    Assert.assertNull(event.getCounters());
    Assert.assertNull(event.getStatistics());
    // task is initialized - progress obtained and also counters since flag is true
    progressNotified = true;
    doReturn(progressNotified).when(mockTask).getAndClearProgressNotification();
    doReturn(true).when(mockTask).hasInitialized();
    event = heartbeatCallable.getStatusUpdateEvent(true);
    verify(mockTask, times(3)).hasInitialized();
    verify(mockTask, times(2)).getProgress();
    verify(mockTask, times(2)).getAndClearProgressNotification();
    verify(mockTask, times(1)).getTaskStatistics();
    verify(mockTask, times(1)).getCounters();
    Assert.assertEquals(progress, event.getProgress(), 0);
    Assert.assertEquals(progressNotified, event.getProgressNotified());
    Assert.assertEquals(counters, event.getCounters());
    Assert.assertEquals(stats, event.getStatistics());
}
Also used : LogicalIOProcessorRuntimeTask(org.apache.tez.runtime.LogicalIOProcessorRuntimeTask) AtomicLong(java.util.concurrent.atomic.AtomicLong) TezTaskUmbilicalProtocol(org.apache.tez.common.TezTaskUmbilicalProtocol) TaskStatistics(org.apache.tez.runtime.api.impl.TaskStatistics) TaskStatusUpdateEvent(org.apache.tez.runtime.api.events.TaskStatusUpdateEvent) TezCounters(org.apache.tez.common.counters.TezCounters) TezTaskAttemptID(org.apache.tez.dag.records.TezTaskAttemptID) Test(org.junit.Test)

Aggregations

TaskStatistics (org.apache.tez.runtime.api.impl.TaskStatistics)4 Test (org.junit.Test)3 ByteArrayOutputStream (java.io.ByteArrayOutputStream)2 DataOutput (java.io.DataOutput)2 DataOutputStream (java.io.DataOutputStream)2 IOException (java.io.IOException)2 DataInputByteBuffer (org.apache.hadoop.io.DataInputByteBuffer)2 TezCounters (org.apache.tez.common.counters.TezCounters)2 DAG (org.apache.tez.dag.api.DAG)2 TezConfiguration (org.apache.tez.dag.api.TezConfiguration)2 Vertex (org.apache.tez.dag.api.Vertex)2 DAGClient (org.apache.tez.dag.api.client.DAGClient)2 DAGStatus (org.apache.tez.dag.api.client.DAGStatus)2 MockContainerLauncher (org.apache.tez.dag.app.MockDAGAppMaster.MockContainerLauncher)2 StatisticsDelegate (org.apache.tez.dag.app.MockDAGAppMaster.StatisticsDelegate)2 DAGImpl (org.apache.tez.dag.app.dag.impl.DAGImpl)2 IOStatistics (org.apache.tez.runtime.api.impl.IOStatistics)2 TaskSpec (org.apache.tez.runtime.api.impl.TaskSpec)2 AtomicLong (java.util.concurrent.atomic.AtomicLong)1 TezTaskUmbilicalProtocol (org.apache.tez.common.TezTaskUmbilicalProtocol)1