use of org.apache.tez.dag.app.rm.TaskSchedulerManager in project tez by apache.
the class TestVertexImpl method setupPostDagCreation.
@SuppressWarnings({ "unchecked", "rawtypes" })
public void setupPostDagCreation() throws TezException {
String dagName = "dag0";
// dispatcher may be created multiple times (setupPostDagCreation may be called multiples)
if (dispatcher != null) {
dispatcher.stop();
}
dispatcher = new DrainDispatcher();
appContext = mock(AppContext.class);
when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim());
when(appContext.getContainerLauncherName(anyInt())).thenReturn(TezConstants.getTezYarnServicePluginName());
thh = mock(TaskHeartbeatHandler.class);
historyEventHandler = mock(HistoryEventHandler.class);
TaskSchedulerManager taskScheduler = mock(TaskSchedulerManager.class);
UserGroupInformation ugi;
try {
ugi = UserGroupInformation.getCurrentUser();
} catch (IOException e) {
throw new RuntimeException(e);
}
DAG dag = mock(DAG.class);
doReturn(ugi).when(dag).getDagUGI();
doReturn(dagName).when(dag).getName();
Map<String, LocalResource> localResources = new HashMap<>();
for (PlanLocalResource planLR : dagPlan.getLocalResourceList()) {
localResources.put(planLR.getName(), DagTypeConverters.convertPlanLocalResourceToLocalResource(planLR));
}
when(dag.getLocalResources()).thenReturn(localResources);
doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
doReturn(appAttemptId.getApplicationId()).when(appContext).getApplicationID();
doReturn(dag).when(appContext).getCurrentDAG();
execService = mock(ListeningExecutorService.class);
final ListenableFuture<Void> mockFuture = mock(ListenableFuture.class);
Mockito.doAnswer(new Answer() {
public ListenableFuture<Void> answer(InvocationOnMock invocation) {
Object[] args = invocation.getArguments();
CallableEvent e = (CallableEvent) args[0];
dispatcher.getEventHandler().handle(e);
return mockFuture;
}
}).when(execService).submit((Callable<Void>) any());
MockClock clock = new MockClock();
doReturn(execService).when(appContext).getExecService();
doReturn(conf).when(appContext).getAMConf();
doReturn(new Credentials()).when(dag).getCredentials();
doReturn(DAGPlan.getDefaultInstance()).when(dag).getJobPlan();
doReturn(dagId).when(appContext).getCurrentDAGID();
doReturn(dagId).when(dag).getID();
doReturn(taskScheduler).when(appContext).getTaskScheduler();
doReturn(Resource.newInstance(102400, 60)).when(taskScheduler).getTotalResources(0);
doReturn(historyEventHandler).when(appContext).getHistoryHandler();
doReturn(dispatcher.getEventHandler()).when(appContext).getEventHandler();
doReturn(clock).when(appContext).getClock();
vertexGroups = Maps.newHashMap();
for (PlanVertexGroupInfo groupInfo : dagPlan.getVertexGroupsList()) {
vertexGroups.put(groupInfo.getGroupName(), new VertexGroupInfo(groupInfo));
}
// updateTracker may be created multiple times (setupPostDagCreation may be called multiples)
if (updateTracker != null) {
updateTracker.stop();
}
updateTracker = new StateChangeNotifierForTest(appContext.getCurrentDAG());
setupVertices();
when(dag.getVertex(any(TezVertexID.class))).thenAnswer(new Answer<Vertex>() {
@Override
public Vertex answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
if (args.length != 1) {
return null;
}
TezVertexID vId = (TezVertexID) args[0];
return vertexIdMap.get(vId);
}
});
when(dag.getVertex(any(String.class))).thenAnswer(new Answer<Vertex>() {
@Override
public Vertex answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
if (args.length != 1) {
return null;
}
String vId = (String) args[0];
return vertices.get(vId);
}
});
// TODO - this test logic is tightly linked to impl DAGImpl code.
edges = new HashMap<String, Edge>();
for (EdgePlan edgePlan : dagPlan.getEdgeList()) {
EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(edgePlan);
edges.put(edgePlan.getId(), new Edge(edgeProperty, dispatcher.getEventHandler(), conf));
}
parseVertexEdges();
for (Edge edge : edges.values()) {
edge.initialize();
}
dispatcher.register(CallableEventType.class, new CallableEventDispatcher());
taskAttemptEventDispatcher = new TaskAttemptEventDispatcher();
dispatcher.register(TaskAttemptEventType.class, taskAttemptEventDispatcher);
taskEventDispatcher = new TaskEventDispatcher();
dispatcher.register(TaskEventType.class, taskEventDispatcher);
vertexEventDispatcher = new VertexEventDispatcher();
dispatcher.register(VertexEventType.class, vertexEventDispatcher);
dagEventDispatcher = new DagEventDispatcher();
dispatcher.register(DAGEventType.class, dagEventDispatcher);
amSchedulerEventDispatcher = new AMSchedulerEventDispatcher();
dispatcher.register(AMSchedulerEventType.class, amSchedulerEventDispatcher);
dispatcher.init(conf);
dispatcher.start();
}
use of org.apache.tez.dag.app.rm.TaskSchedulerManager in project tez by apache.
the class TestAMNodeTracker method testSingleNodeNotBlacklistedAlternateScheduler.
@Test(timeout = 5000)
public void testSingleNodeNotBlacklistedAlternateScheduler() {
AppContext appContext = mock(AppContext.class);
Configuration conf = new Configuration(false);
conf.setInt(TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, 2);
conf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, true);
conf.setInt(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_IGNORE_THRESHOLD, 33);
TestEventHandler handler = new TestEventHandler();
AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
doReturn(amNodeTracker).when(appContext).getNodeTracker();
AMContainerMap amContainerMap = mock(AMContainerMap.class);
TaskSchedulerManager taskSchedulerManager = mock(TaskSchedulerManager.class);
dispatcher.register(AMNodeEventType.class, amNodeTracker);
dispatcher.register(AMContainerEventType.class, amContainerMap);
dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
amNodeTracker.init(conf);
amNodeTracker.start();
_testSingleNodeNotBlacklisted(amNodeTracker, handler, 1);
}
use of org.apache.tez.dag.app.rm.TaskSchedulerManager in project tez by apache.
the class TestAMNodeTracker method testNodeSelfBlacklistAlternateScheduler2.
@Test(timeout = 10000)
public void testNodeSelfBlacklistAlternateScheduler2() {
AppContext appContext = mock(AppContext.class);
Configuration conf = new Configuration(false);
conf.setInt(TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, 2);
TestEventHandler handler = new TestEventHandler();
AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
doReturn(amNodeTracker).when(appContext).getNodeTracker();
AMContainerMap amContainerMap = mock(AMContainerMap.class);
TaskSchedulerManager taskSchedulerManager = mock(TaskSchedulerManager.class);
dispatcher.register(AMNodeEventType.class, amNodeTracker);
dispatcher.register(AMContainerEventType.class, amContainerMap);
dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
amNodeTracker.init(conf);
amNodeTracker.start();
try {
// This should not affect the blacklisting behaviour
for (int i = 0; i < 100; i++) {
amNodeTracker.nodeSeen(NodeId.newInstance("fakenode" + i, 3333), 0);
}
_testNodeSelfBlacklist(amNodeTracker, handler, 1);
assertFalse(amNodeTracker.isBlacklistingIgnored(0));
} finally {
amNodeTracker.stop();
}
}
use of org.apache.tez.dag.app.rm.TaskSchedulerManager in project tez by apache.
the class TestAMNodeTracker method testSingleNodeNotBlacklisted.
@Test(timeout = 5000)
public void testSingleNodeNotBlacklisted() {
AppContext appContext = mock(AppContext.class);
Configuration conf = new Configuration(false);
conf.setInt(TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, 2);
conf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, true);
conf.setInt(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_IGNORE_THRESHOLD, 33);
TestEventHandler handler = new TestEventHandler();
AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
doReturn(amNodeTracker).when(appContext).getNodeTracker();
AMContainerMap amContainerMap = mock(AMContainerMap.class);
TaskSchedulerManager taskSchedulerManager = mock(TaskSchedulerManager.class);
dispatcher.register(AMNodeEventType.class, amNodeTracker);
dispatcher.register(AMContainerEventType.class, amContainerMap);
dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
amNodeTracker.init(conf);
amNodeTracker.start();
_testSingleNodeNotBlacklisted(amNodeTracker, handler, 0);
}
use of org.apache.tez.dag.app.rm.TaskSchedulerManager in project tez by apache.
the class TestAMNodeTracker method testNodeCompletedAndCleanup.
@Test(timeout = 10000L)
public void testNodeCompletedAndCleanup() {
AppContext appContext = mock(AppContext.class);
Configuration conf = new Configuration(false);
conf.setInt(TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, 2);
TestEventHandler handler = new TestEventHandler();
AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
doReturn(amNodeTracker).when(appContext).getNodeTracker();
AMContainerMap amContainerMap = mock(AMContainerMap.class);
TaskSchedulerManager taskSchedulerManager = mock(TaskSchedulerManager.class);
dispatcher.register(AMNodeEventType.class, amNodeTracker);
dispatcher.register(AMContainerEventType.class, amContainerMap);
dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
amNodeTracker.init(conf);
amNodeTracker.start();
try {
NodeId nodeId = NodeId.newInstance("fakenode", 3333);
amNodeTracker.nodeSeen(nodeId, 0);
AMNode amNode = amNodeTracker.get(nodeId, 0);
ContainerId[] containerIds = new ContainerId[7];
// Start 5 containers.
for (int i = 0; i < 5; i++) {
containerIds[i] = mock(ContainerId.class);
amNodeTracker.handle(new AMNodeEventContainerAllocated(nodeId, 0, containerIds[i]));
}
assertEquals(5, amNode.getContainers().size());
// Finnish 1st dag
amNodeTracker.dagComplete(mock(DAG.class));
assertEquals(5, amNode.getContainers().size());
// Mark 2 as complete. Finish 2nd dag.
for (int i = 0; i < 2; i++) {
amNodeTracker.handle(new AMNodeEventContainerCompleted(nodeId, 0, containerIds[i]));
}
amNodeTracker.dagComplete(mock(DAG.class));
assertEquals(3, amNode.getContainers().size());
// Add 2 more containers. Mark all as complete. Finish 3rd dag.
for (int i = 5; i < 7; i++) {
containerIds[i] = mock(ContainerId.class);
amNodeTracker.handle(new AMNodeEventContainerAllocated(nodeId, 0, containerIds[i]));
}
assertEquals(5, amNode.getContainers().size());
amNodeTracker.dagComplete(mock(DAG.class));
assertEquals(5, amNode.getContainers().size());
amNodeTracker.dagComplete(mock(DAG.class));
assertEquals(5, amNode.getContainers().size());
for (int i = 2; i < 7; i++) {
amNodeTracker.handle(new AMNodeEventContainerCompleted(nodeId, 0, containerIds[i]));
}
assertEquals(5, amNode.getContainers().size());
amNodeTracker.dagComplete(mock(DAG.class));
assertEquals(0, amNode.getContainers().size());
} finally {
amNodeTracker.stop();
}
}
Aggregations