use of org.apache.tez.dag.records.TezVertexID in project tez by apache.
the class TestEdge method testInvalidPhysicalInputCount.
@Test(timeout = 5000)
public void testInvalidPhysicalInputCount() throws Exception {
EventHandler mockEventHandler = mock(EventHandler.class);
Edge edge = new Edge(EdgeProperty.create(EdgeManagerPluginDescriptor.create(CustomEdgeManagerWithInvalidReturnValue.class.getName()).setUserPayload(new CustomEdgeManagerWithInvalidReturnValue.EdgeManagerConfig(-1, 1, 1, 1).toUserPayload()), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create(""), InputDescriptor.create("")), mockEventHandler, new TezConfiguration());
TezVertexID v1Id = createVertexID(1);
TezVertexID v2Id = createVertexID(2);
edge.setSourceVertex(mockVertex("v1", v1Id, new LinkedHashMap<TezTaskID, Task>()));
edge.setDestinationVertex(mockVertex("v2", v2Id, new LinkedHashMap<TezTaskID, Task>()));
edge.initialize();
try {
edge.getDestinationSpec(0);
Assert.fail();
} catch (AMUserCodeException e) {
e.printStackTrace();
assertTrue(e.getCause().getMessage().contains("PhysicalInputCount should not be negative"));
}
}
use of org.apache.tez.dag.records.TezVertexID in project tez by apache.
the class TestEdge method testInvalidPhysicalOutputCount.
@Test(timeout = 5000)
public void testInvalidPhysicalOutputCount() throws Exception {
EventHandler mockEventHandler = mock(EventHandler.class);
Edge edge = new Edge(EdgeProperty.create(EdgeManagerPluginDescriptor.create(CustomEdgeManagerWithInvalidReturnValue.class.getName()).setUserPayload(new CustomEdgeManagerWithInvalidReturnValue.EdgeManagerConfig(1, -1, 1, 1).toUserPayload()), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create(""), InputDescriptor.create("")), mockEventHandler, new TezConfiguration());
TezVertexID v1Id = createVertexID(1);
TezVertexID v2Id = createVertexID(2);
edge.setSourceVertex(mockVertex("v1", v1Id, new LinkedHashMap<TezTaskID, Task>()));
edge.setDestinationVertex(mockVertex("v2", v2Id, new LinkedHashMap<TezTaskID, Task>()));
edge.initialize();
try {
edge.getSourceSpec(0);
Assert.fail();
} catch (AMUserCodeException e) {
e.printStackTrace();
assertTrue(e.getCause().getMessage().contains("PhysicalOutputCount should not be negative"));
}
}
use of org.apache.tez.dag.records.TezVertexID in project tez by apache.
the class TestEdge method testInvalidConsumerNumber.
@Test(timeout = 5000)
public void testInvalidConsumerNumber() throws Exception {
EventHandler mockEventHandler = mock(EventHandler.class);
Edge edge = new Edge(EdgeProperty.create(EdgeManagerPluginDescriptor.create(CustomEdgeManagerWithInvalidReturnValue.class.getName()).setUserPayload(new CustomEdgeManagerWithInvalidReturnValue.EdgeManagerConfig(1, 1, 0, 1).toUserPayload()), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create(""), InputDescriptor.create("")), mockEventHandler, new TezConfiguration());
TezVertexID v1Id = createVertexID(1);
TezVertexID v2Id = createVertexID(2);
edge.setSourceVertex(mockVertex("v1", v1Id, new LinkedHashMap<TezTaskID, Task>()));
edge.setDestinationVertex(mockVertex("v2", v2Id, new LinkedHashMap<TezTaskID, Task>()));
edge.initialize();
try {
TezEvent ireEvent = new TezEvent(InputReadErrorEvent.create("diag", 0, 1), new EventMetaData(EventProducerConsumerType.INPUT, "v2", "v1", TezTaskAttemptID.getInstance(TezTaskID.getInstance(v2Id, 1), 1)));
edge.sendTezEventToSourceTasks(ireEvent);
Assert.fail();
} catch (AMUserCodeException e) {
e.printStackTrace();
assertTrue(e.getCause().getMessage().contains("ConsumerTaskNum must be positive"));
}
}
use of org.apache.tez.dag.records.TezVertexID in project tez by apache.
the class TestVertexStats method testBasicStats.
@Test(timeout = 5000)
public void testBasicStats() {
VertexStats stats = new VertexStats();
Assert.assertEquals(-1, stats.firstTaskStartTime);
Assert.assertEquals(-1, stats.lastTaskFinishTime);
Assert.assertEquals(-1, stats.minTaskDuration);
Assert.assertEquals(-1, stats.maxTaskDuration);
Assert.assertTrue(-1 == stats.avgTaskDuration);
Assert.assertEquals(0, stats.getFirstTasksToStart().size());
Assert.assertEquals(0, stats.getLastTasksToFinish().size());
Assert.assertEquals(0, stats.getShortestDurationTasks().size());
Assert.assertEquals(0, stats.getLongestDurationTasks().size());
TezVertexID tezVertexID = TezVertexID.getInstance(TezDAGID.getInstance(ApplicationId.newInstance(100l, 1), 1), 1);
TezTaskID tezTaskID1 = TezTaskID.getInstance(tezVertexID, 1);
TezTaskID tezTaskID2 = TezTaskID.getInstance(tezVertexID, 2);
TezTaskID tezTaskID3 = TezTaskID.getInstance(tezVertexID, 3);
TezTaskID tezTaskID4 = TezTaskID.getInstance(tezVertexID, 4);
TezTaskID tezTaskID5 = TezTaskID.getInstance(tezVertexID, 5);
TezTaskID tezTaskID6 = TezTaskID.getInstance(tezVertexID, 6);
stats.updateStats(new TaskReportImpl(tezTaskID1, TaskState.SUCCEEDED, 1, 100, 200));
Assert.assertEquals(100, stats.firstTaskStartTime);
Assert.assertEquals(200, stats.lastTaskFinishTime);
Assert.assertEquals(100, stats.minTaskDuration);
Assert.assertEquals(100, stats.maxTaskDuration);
Assert.assertTrue(100 == stats.avgTaskDuration);
Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID1));
Assert.assertTrue(stats.lastTasksToFinish.contains(tezTaskID1));
Assert.assertTrue(stats.shortestDurationTasks.contains(tezTaskID1));
Assert.assertTrue(stats.longestDurationTasks.contains(tezTaskID1));
Assert.assertEquals(1, stats.firstTasksToStart.size());
Assert.assertEquals(1, stats.lastTasksToFinish.size());
Assert.assertEquals(1, stats.shortestDurationTasks.size());
Assert.assertEquals(1, stats.longestDurationTasks.size());
stats.updateStats(new TaskReportImpl(tezTaskID2, TaskState.FAILED, 1, 150, 300));
Assert.assertEquals(100, stats.firstTaskStartTime);
Assert.assertEquals(300, stats.lastTaskFinishTime);
Assert.assertEquals(100, stats.minTaskDuration);
Assert.assertEquals(100, stats.maxTaskDuration);
Assert.assertTrue(100 == stats.avgTaskDuration);
Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID1));
Assert.assertTrue(stats.lastTasksToFinish.contains(tezTaskID2));
Assert.assertTrue(stats.shortestDurationTasks.contains(tezTaskID1));
Assert.assertTrue(stats.longestDurationTasks.contains(tezTaskID1));
Assert.assertEquals(1, stats.firstTasksToStart.size());
Assert.assertEquals(1, stats.lastTasksToFinish.size());
Assert.assertEquals(1, stats.shortestDurationTasks.size());
Assert.assertEquals(1, stats.longestDurationTasks.size());
stats.updateStats(new TaskReportImpl(tezTaskID3, TaskState.RUNNING, 1, 50, 550));
Assert.assertEquals(50, stats.firstTaskStartTime);
Assert.assertEquals(550, stats.lastTaskFinishTime);
Assert.assertEquals(100, stats.minTaskDuration);
Assert.assertEquals(100, stats.maxTaskDuration);
Assert.assertTrue(100 == stats.avgTaskDuration);
Assert.assertTrue(stats.shortestDurationTasks.contains(tezTaskID1));
Assert.assertTrue(stats.longestDurationTasks.contains(tezTaskID1));
Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID3));
Assert.assertTrue(stats.lastTasksToFinish.contains(tezTaskID3));
Assert.assertEquals(1, stats.firstTasksToStart.size());
Assert.assertEquals(1, stats.lastTasksToFinish.size());
Assert.assertEquals(1, stats.shortestDurationTasks.size());
Assert.assertEquals(1, stats.longestDurationTasks.size());
stats.updateStats(new TaskReportImpl(tezTaskID4, TaskState.SUCCEEDED, 1, 50, 450));
Assert.assertEquals(50, stats.firstTaskStartTime);
Assert.assertEquals(550, stats.lastTaskFinishTime);
Assert.assertEquals(100, stats.minTaskDuration);
Assert.assertEquals(400, stats.maxTaskDuration);
Assert.assertTrue(250 == stats.avgTaskDuration);
Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID4));
Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID3));
Assert.assertTrue(stats.lastTasksToFinish.contains(tezTaskID3));
Assert.assertTrue(stats.shortestDurationTasks.contains(tezTaskID1));
Assert.assertTrue(stats.longestDurationTasks.contains(tezTaskID4));
Assert.assertEquals(2, stats.firstTasksToStart.size());
Assert.assertEquals(1, stats.lastTasksToFinish.size());
Assert.assertEquals(1, stats.shortestDurationTasks.size());
Assert.assertEquals(1, stats.longestDurationTasks.size());
stats.updateStats(new TaskReportImpl(tezTaskID5, TaskState.SUCCEEDED, 1, 50, 450));
Assert.assertEquals(50, stats.firstTaskStartTime);
Assert.assertEquals(550, stats.lastTaskFinishTime);
Assert.assertEquals(100, stats.minTaskDuration);
Assert.assertEquals(400, stats.maxTaskDuration);
Assert.assertTrue(300 == stats.avgTaskDuration);
Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID5));
Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID4));
Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID3));
Assert.assertTrue(stats.lastTasksToFinish.contains(tezTaskID3));
Assert.assertTrue(stats.shortestDurationTasks.contains(tezTaskID1));
Assert.assertTrue(stats.longestDurationTasks.contains(tezTaskID4));
Assert.assertTrue(stats.longestDurationTasks.contains(tezTaskID5));
Assert.assertEquals(3, stats.firstTasksToStart.size());
Assert.assertEquals(1, stats.lastTasksToFinish.size());
Assert.assertEquals(1, stats.shortestDurationTasks.size());
Assert.assertEquals(2, stats.longestDurationTasks.size());
stats.updateStats(new TaskReportImpl(tezTaskID6, TaskState.SUCCEEDED, 1, 450, 550));
Assert.assertEquals(50, stats.firstTaskStartTime);
Assert.assertEquals(550, stats.lastTaskFinishTime);
Assert.assertEquals(100, stats.minTaskDuration);
Assert.assertEquals(400, stats.maxTaskDuration);
Assert.assertTrue(250 == stats.avgTaskDuration);
Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID5));
Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID4));
Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID3));
Assert.assertTrue(stats.lastTasksToFinish.contains(tezTaskID3));
Assert.assertTrue(stats.lastTasksToFinish.contains(tezTaskID6));
Assert.assertTrue(stats.shortestDurationTasks.contains(tezTaskID1));
Assert.assertTrue(stats.shortestDurationTasks.contains(tezTaskID6));
Assert.assertTrue(stats.longestDurationTasks.contains(tezTaskID4));
Assert.assertTrue(stats.longestDurationTasks.contains(tezTaskID5));
Assert.assertEquals(3, stats.firstTasksToStart.size());
Assert.assertEquals(2, stats.lastTasksToFinish.size());
Assert.assertEquals(2, stats.shortestDurationTasks.size());
Assert.assertEquals(2, stats.longestDurationTasks.size());
}
use of org.apache.tez.dag.records.TezVertexID in project tez by apache.
the class TestContainerReuse method testReuseWithTaskSpecificLaunchCmdOption.
@Test(timeout = 10000l)
public void testReuseWithTaskSpecificLaunchCmdOption() throws IOException, InterruptedException, ExecutionException {
LOG.info("Test testReuseWithTaskSpecificLaunchCmdOption");
Configuration tezConf = new Configuration(new YarnConfiguration());
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true);
tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true);
tezConf.setLong(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 0);
tezConf.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, 0);
tezConf.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, 0);
// Profile 3 tasks
tezConf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, "v1[1,3,4]");
tezConf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, "dir=/tmp/__VERTEX_NAME__/__TASK_INDEX__");
TaskSpecificLaunchCmdOption taskSpecificLaunchCmdOption = new TaskSpecificLaunchCmdOption(tezConf);
CapturingEventHandler eventHandler = new CapturingEventHandler();
TezDAGID dagID = TezDAGID.getInstance("0", 0, 0);
AMRMClient<CookieContainerRequest> rmClientCore = new AMRMClientForTest();
TezAMRMClientAsync<CookieContainerRequest> rmClient = spy(new AMRMClientAsyncForTest(rmClientCore, 100));
AppContext appContext = mock(AppContext.class);
doReturn(new Configuration(false)).when(appContext).getAMConf();
AMContainerMap amContainerMap = new AMContainerMap(mock(ContainerHeartbeatHandler.class), mock(TaskCommunicatorManagerInterface.class), new ContainerContextMatcher(), appContext);
AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
doReturn(amContainerMap).when(appContext).getAllContainers();
doReturn(amNodeTracker).when(appContext).getNodeTracker();
doReturn(DAGAppMasterState.RUNNING).when(appContext).getAMState();
doReturn(dagID).when(appContext).getCurrentDAGID();
doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo();
// Use ContainerContextMatcher here. Otherwise it would not match the JVM options
TaskSchedulerManager taskSchedulerManagerReal = new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient, new ContainerContextMatcher(), TezUtils.createUserPayloadFromConf(tezConf));
TaskSchedulerManager taskSchedulerManager = spy(taskSchedulerManagerReal);
taskSchedulerManager.init(tezConf);
taskSchedulerManager.start();
TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext) ((TaskSchedulerManagerForTest) taskSchedulerManager).getSpyTaskScheduler();
TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback();
AtomicBoolean drainNotifier = new AtomicBoolean(false);
taskScheduler.delayedContainerManager.drainedDelayedContainersForTest = drainNotifier;
Resource resource1 = Resource.newInstance(1024, 1);
String[] host1 = { "host1" };
String[] host2 = { "host2" };
String[] host3 = { "host3" };
String[] racks = { "/default-rack" };
Priority priority1 = Priority.newInstance(1);
TezVertexID vertexID1 = TezVertexID.getInstance(dagID, 1);
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
String tsLaunchCmdOpts = taskSpecificLaunchCmdOption.getTaskSpecificOption("", "v1", 1);
/**
* Schedule 2 tasks (1 with additional launch-cmd option and another in normal mode).
* Container should not be reused in this case.
*/
// Vertex 1, Task 1, Attempt 1, host1
TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 1), 1);
TaskAttempt ta11 = mock(TaskAttempt.class);
AMSchedulerEventTALaunchRequest lrEvent1 = createLaunchRequestEvent(taID11, ta11, resource1, host1, racks, priority1, localResources, tsLaunchCmdOpts);
// Vertex 1, Task 2, Attempt 1, host1
TezTaskAttemptID taID12 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 2), 1);
TaskAttempt ta12 = mock(TaskAttempt.class);
AMSchedulerEventTALaunchRequest lrEvent2 = createLaunchRequestEvent(taID12, ta12, resource1, host1, racks, priority1);
taskSchedulerManager.handleEvent(lrEvent1);
taskSchedulerManager.handleEvent(lrEvent2);
Container container1 = createContainer(1, "host1", resource1, priority1);
// One container allocated.
taskScheduler.onContainersAllocated(Collections.singletonList(container1));
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainableAppCallback.drain();
verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta11), any(Object.class), eq(container1));
// First task had profiling on. This container can not be reused further.
taskSchedulerManager.handleEvent(new AMSchedulerEventTAEnded(ta11, container1.getId(), TaskAttemptState.SUCCEEDED, null, null, 0));
drainableAppCallback.drain();
verifyDeAllocateTask(taskScheduler, ta11, true, null, null);
verify(taskSchedulerManager, times(0)).taskAllocated(eq(0), eq(ta12), any(Object.class), eq(container1));
verify(rmClient, times(1)).releaseAssignedContainer(eq(container1.getId()));
eventHandler.verifyInvocation(AMContainerEventStopRequest.class);
eventHandler.reset();
/**
* Schedule 2 tasks (both having different task specific JVM option).
* Container should not be reused.
*/
// Vertex 1, Task 3, Attempt 1, host2
tsLaunchCmdOpts = taskSpecificLaunchCmdOption.getTaskSpecificOption("", "v1", 3);
TezTaskAttemptID taID13 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 3), 1);
TaskAttempt ta13 = mock(TaskAttempt.class);
AMSchedulerEventTALaunchRequest lrEvent3 = createLaunchRequestEvent(taID13, ta13, resource1, host2, racks, priority1, localResources, tsLaunchCmdOpts);
// Vertex 1, Task 4, Attempt 1, host2
tsLaunchCmdOpts = taskSpecificLaunchCmdOption.getTaskSpecificOption("", "v1", 4);
TezTaskAttemptID taID14 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 4), 1);
TaskAttempt ta14 = mock(TaskAttempt.class);
AMSchedulerEventTALaunchRequest lrEvent4 = createLaunchRequestEvent(taID14, ta14, resource1, host2, racks, priority1, localResources, tsLaunchCmdOpts);
taskSchedulerManager.handleEvent(lrEvent3);
taskSchedulerManager.handleEvent(lrEvent4);
// Container started
Container container2 = createContainer(2, "host2", resource1, priority1);
taskScheduler.onContainersAllocated(Collections.singletonList(container2));
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainableAppCallback.drain();
verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta13), any(Object.class), eq(container2));
// Verify that the container can not be reused when profiling option is turned on
// Even for 2 tasks having same profiling option can have container reusability.
taskSchedulerManager.handleEvent(new AMSchedulerEventTAEnded(ta13, container2.getId(), TaskAttemptState.SUCCEEDED, null, null, 0));
drainableAppCallback.drain();
verifyDeAllocateTask(taskScheduler, ta13, true, null, null);
verify(taskSchedulerManager, times(0)).taskAllocated(eq(0), eq(ta14), any(Object.class), eq(container2));
verify(rmClient, times(1)).releaseAssignedContainer(eq(container2.getId()));
eventHandler.verifyInvocation(AMContainerEventStopRequest.class);
eventHandler.reset();
/**
* Schedule 2 tasks with same jvm profiling option.
* Container should be reused.
*/
tezConf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, "v1[1,2,3,5,6]");
tezConf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, "dummyOpts");
taskSpecificLaunchCmdOption = new TaskSpecificLaunchCmdOption(tezConf);
// Vertex 1, Task 5, Attempt 1, host3
TezTaskAttemptID taID15 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 3), 1);
TaskAttempt ta15 = mock(TaskAttempt.class);
AMSchedulerEventTALaunchRequest lrEvent5 = createLaunchRequestEvent(taID15, ta15, resource1, host3, racks, priority1, localResources, taskSpecificLaunchCmdOption.getTaskSpecificOption("", "v1", 5));
// Vertex 1, Task 6, Attempt 1, host3
tsLaunchCmdOpts = taskSpecificLaunchCmdOption.getTaskSpecificOption("", "v1", 4);
TezTaskAttemptID taID16 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 4), 1);
TaskAttempt ta16 = mock(TaskAttempt.class);
AMSchedulerEventTALaunchRequest lrEvent6 = createLaunchRequestEvent(taID16, ta16, resource1, host3, racks, priority1, localResources, taskSpecificLaunchCmdOption.getTaskSpecificOption("", "v1", 6));
// Container started
Container container3 = createContainer(3, "host3", resource1, priority1);
taskSchedulerManager.handleEvent(lrEvent5);
taskSchedulerManager.handleEvent(lrEvent6);
taskScheduler.onContainersAllocated(Collections.singletonList(container3));
TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier);
drainableAppCallback.drain();
verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta15), any(Object.class), eq(container3));
// Ensure task 6 (of vertex 1) is allocated to same container
taskSchedulerManager.handleEvent(new AMSchedulerEventTAEnded(ta15, container3.getId(), TaskAttemptState.SUCCEEDED, null, null, 0));
drainableAppCallback.drain();
verifyDeAllocateTask(taskScheduler, ta15, true, null, null);
verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta16), any(Object.class), eq(container3));
eventHandler.reset();
taskScheduler.shutdown();
taskSchedulerManager.close();
}
Aggregations