use of org.apache.tez.runtime.api.impl.TaskStatistics in project tez by apache.
the class TaskStatusUpdateEvent method readFields.
@Override
public void readFields(DataInput in) throws IOException {
progress = in.readFloat();
progressNotified = in.readBoolean();
if (in.readBoolean()) {
tezCounters = new TezCounters();
tezCounters.readFields(in);
}
if (in.readBoolean()) {
statistics = new TaskStatistics();
statistics.readFields(in);
}
}
use of org.apache.tez.runtime.api.impl.TaskStatistics in project tez by apache.
the class TestMockDAGAppMaster method testBasicStatisticsMemory.
@Ignore
@Test(timeout = 60000)
public void testBasicStatisticsMemory() throws Exception {
Logger.getRootLogger().setLevel(Level.WARN);
TezConfiguration tezconf = new TezConfiguration(defaultConf);
MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
tezClient.start();
final String vAName = "abcdefghijklmnopqrstuvwxyz";
int numTasks = 10000;
int numSources = 10;
IOStatistics ioStats = new IOStatistics();
ioStats.setDataSize(1);
ioStats.setItemsProcessed(1);
TaskStatistics vAStats = new TaskStatistics();
DAG dag = DAG.create("testBasicStatisticsMemory");
Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), numTasks);
for (int i = 0; i < numSources; ++i) {
final String sourceName = i + vAName;
vA.addDataSource(sourceName, DataSourceDescriptor.create(InputDescriptor.create(sourceName), null, null));
vAStats.addIO(sourceName, ioStats);
}
dag.addVertex(vA);
ByteArrayOutputStream bosA = new ByteArrayOutputStream();
DataOutput outA = new DataOutputStream(bosA);
vAStats.write(outA);
final byte[] payloadA = bosA.toByteArray();
MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
mockLauncher.startScheduling(false);
mockApp.statsDelegate = new StatisticsDelegate() {
@Override
public TaskStatistics getStatistics(TaskSpec taskSpec) {
byte[] payload = payloadA;
TaskStatistics stats = new TaskStatistics();
final DataInputByteBuffer in = new DataInputByteBuffer();
in.reset(ByteBuffer.wrap(payload));
try {
// this ensures that the serde code path is covered.
stats.readFields(in);
} catch (IOException e) {
Assert.fail(e.getMessage());
}
return stats;
}
};
mockApp.doSleep = false;
DAGClient dagClient = tezClient.submitDAG(dag);
mockLauncher.waitTillContainersLaunched();
DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
mockLauncher.startScheduling(true);
DAGStatus status = dagClient.waitForCompletion();
Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
Assert.assertEquals(numTasks, dagImpl.getVertex(vAName).getStatistics().getInputStatistics(0 + vAName).getDataSize());
Assert.assertEquals(numTasks, dagImpl.getVertex(vAName).getStatistics().getInputStatistics(0 + vAName).getItemsProcessed());
checkMemory(dag.getName(), mockApp);
tezClient.stop();
}
use of org.apache.tez.runtime.api.impl.TaskStatistics in project tez by apache.
the class TestMockDAGAppMaster method testBasicStatistics.
@Test(timeout = 10000)
public void testBasicStatistics() throws Exception {
TezConfiguration tezconf = new TezConfiguration(defaultConf);
MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
tezClient.start();
final String vAName = "A";
final String vBName = "B";
final String sourceName = "In";
final String sinkName = "Out";
DAG dag = DAG.create("testBasisStatistics");
Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 3);
Vertex vB = Vertex.create(vBName, ProcessorDescriptor.create("Proc.class"), 2);
vA.addDataSource(sourceName, DataSourceDescriptor.create(InputDescriptor.create("In"), null, null));
vB.addDataSink(sinkName, DataSinkDescriptor.create(OutputDescriptor.create("Out"), null, null));
dag.addVertex(vA).addVertex(vB).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
IOStatistics ioStats = new IOStatistics();
ioStats.setDataSize(1);
ioStats.setItemsProcessed(1);
TaskStatistics vAStats = new TaskStatistics();
vAStats.addIO(vBName, ioStats);
vAStats.addIO(sourceName, ioStats);
TaskStatistics vBStats = new TaskStatistics();
vBStats.addIO(vAName, ioStats);
vBStats.addIO(sinkName, ioStats);
ByteArrayOutputStream bosA = new ByteArrayOutputStream();
DataOutput outA = new DataOutputStream(bosA);
vAStats.write(outA);
final byte[] payloadA = bosA.toByteArray();
ByteArrayOutputStream bosB = new ByteArrayOutputStream();
DataOutput outB = new DataOutputStream(bosB);
vBStats.write(outB);
final byte[] payloadB = bosB.toByteArray();
MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
mockLauncher.startScheduling(false);
mockApp.statsDelegate = new StatisticsDelegate() {
@Override
public TaskStatistics getStatistics(TaskSpec taskSpec) {
byte[] payload = payloadA;
TaskStatistics stats = new TaskStatistics();
if (taskSpec.getVertexName().equals(vBName)) {
payload = payloadB;
}
final DataInputByteBuffer in = new DataInputByteBuffer();
in.reset(ByteBuffer.wrap(payload));
try {
// this ensures that the serde code path is covered.
stats.readFields(in);
} catch (IOException e) {
Assert.fail(e.getMessage());
}
return stats;
}
};
mockApp.doSleep = false;
DAGClient dagClient = tezClient.submitDAG(dag);
mockLauncher.waitTillContainersLaunched();
DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
mockLauncher.startScheduling(true);
DAGStatus status = dagClient.waitForCompletion();
Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
// verify that the values have been correct aggregated
for (org.apache.tez.dag.app.dag.Vertex v : dagImpl.getVertices().values()) {
VertexStatistics vStats = v.getStatistics();
if (v.getName().equals(vAName)) {
Assert.assertEquals(3, vStats.getOutputStatistics(vBName).getDataSize());
Assert.assertEquals(3, vStats.getInputStatistics(sourceName).getDataSize());
Assert.assertEquals(3, vStats.getOutputStatistics(vBName).getItemsProcessed());
Assert.assertEquals(3, vStats.getInputStatistics(sourceName).getItemsProcessed());
} else {
Assert.assertEquals(2, vStats.getInputStatistics(vAName).getDataSize());
Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getDataSize());
Assert.assertEquals(2, vStats.getInputStatistics(vAName).getItemsProcessed());
Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getItemsProcessed());
}
}
tezClient.stop();
}
use of org.apache.tez.runtime.api.impl.TaskStatistics in project tez by apache.
the class TestTaskReporter method testStatusUpdateAfterInitializationAndCounterFlag.
@Test(timeout = 5000)
public void testStatusUpdateAfterInitializationAndCounterFlag() {
TezTaskAttemptID mockTaskAttemptId = mock(TezTaskAttemptID.class);
LogicalIOProcessorRuntimeTask mockTask = mock(LogicalIOProcessorRuntimeTask.class);
doReturn("vertexName").when(mockTask).getVertexName();
doReturn(mockTaskAttemptId).when(mockTask).getTaskAttemptID();
boolean progressNotified = false;
doReturn(progressNotified).when(mockTask).getAndClearProgressNotification();
TezTaskUmbilicalProtocol mockUmbilical = mock(TezTaskUmbilicalProtocol.class);
float progress = 0.5f;
TaskStatistics stats = new TaskStatistics();
TezCounters counters = new TezCounters();
doReturn(progress).when(mockTask).getProgress();
doReturn(stats).when(mockTask).getTaskStatistics();
doReturn(counters).when(mockTask).getCounters();
// Setup the sleep time to be way higher than the test timeout
TaskReporter.HeartbeatCallable heartbeatCallable = new TaskReporter.HeartbeatCallable(mockTask, mockUmbilical, 100000, 100000, 5, new AtomicLong(0), "containerIdStr");
// task not initialized - nothing obtained from task
doReturn(false).when(mockTask).hasInitialized();
TaskStatusUpdateEvent event = heartbeatCallable.getStatusUpdateEvent(true);
verify(mockTask, times(1)).hasInitialized();
verify(mockTask, times(0)).getProgress();
verify(mockTask, times(0)).getAndClearProgressNotification();
verify(mockTask, times(0)).getTaskStatistics();
verify(mockTask, times(0)).getCounters();
Assert.assertEquals(0, event.getProgress(), 0);
Assert.assertEquals(false, event.getProgressNotified());
Assert.assertNull(event.getCounters());
Assert.assertNull(event.getStatistics());
// task is initialized - progress obtained but not counters since flag is false
doReturn(true).when(mockTask).hasInitialized();
event = heartbeatCallable.getStatusUpdateEvent(false);
verify(mockTask, times(2)).hasInitialized();
verify(mockTask, times(1)).getProgress();
verify(mockTask, times(1)).getAndClearProgressNotification();
verify(mockTask, times(0)).getTaskStatistics();
verify(mockTask, times(0)).getCounters();
Assert.assertEquals(progress, event.getProgress(), 0);
Assert.assertEquals(progressNotified, event.getProgressNotified());
Assert.assertNull(event.getCounters());
Assert.assertNull(event.getStatistics());
// task is initialized - progress obtained and also counters since flag is true
progressNotified = true;
doReturn(progressNotified).when(mockTask).getAndClearProgressNotification();
doReturn(true).when(mockTask).hasInitialized();
event = heartbeatCallable.getStatusUpdateEvent(true);
verify(mockTask, times(3)).hasInitialized();
verify(mockTask, times(2)).getProgress();
verify(mockTask, times(2)).getAndClearProgressNotification();
verify(mockTask, times(1)).getTaskStatistics();
verify(mockTask, times(1)).getCounters();
Assert.assertEquals(progress, event.getProgress(), 0);
Assert.assertEquals(progressNotified, event.getProgressNotified());
Assert.assertEquals(counters, event.getCounters());
Assert.assertEquals(stats, event.getStatistics());
}
Aggregations