use of org.apache.hadoop.yarn.event.InlineDispatcher in project hadoop by apache.
the class TestTaskImpl method setup.
@Before
@SuppressWarnings("unchecked")
public void setup() {
dispatcher = new InlineDispatcher();
++startCount;
conf = new JobConf();
taskAttemptListener = mock(TaskAttemptListener.class);
jobToken = (Token<JobTokenIdentifier>) mock(Token.class);
remoteJobConfFile = mock(Path.class);
credentials = null;
clock = SystemClock.getInstance();
metrics = mock(MRAppMetrics.class);
dataLocations = new String[1];
appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
jobId = Records.newRecord(JobId.class);
jobId.setId(1);
jobId.setAppId(appId);
appContext = mock(AppContext.class);
taskSplitMetaInfo = mock(TaskSplitMetaInfo.class);
when(taskSplitMetaInfo.getLocations()).thenReturn(dataLocations);
taskAttempts = new ArrayList<MockTaskAttemptImpl>();
taskAttemptEventHandler = new MockTaskAttemptEventHandler();
dispatcher.register(TaskAttemptEventType.class, taskAttemptEventHandler);
}
use of org.apache.hadoop.yarn.event.InlineDispatcher in project hadoop by apache.
the class TestLogAggregationService method testAppLogDirCreation.
@Test
public void testAppLogDirCreation() throws Exception {
final String logSuffix = "logs";
this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, logSuffix);
InlineDispatcher dispatcher = new InlineDispatcher();
dispatcher.init(this.conf);
dispatcher.start();
FileSystem fs = FileSystem.get(this.conf);
final FileSystem spyFs = spy(FileSystem.get(this.conf));
LogAggregationService aggSvc = new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler) {
@Override
protected FileSystem getFileSystem(Configuration conf) {
return spyFs;
}
};
aggSvc.init(this.conf);
aggSvc.start();
// start an application and verify user, suffix, and app dirs created
ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
Path userDir = fs.makeQualified(new Path(remoteRootLogDir.getAbsolutePath(), this.user));
Path suffixDir = new Path(userDir, logSuffix);
Path appDir = new Path(suffixDir, appId.toString());
LogAggregationContext contextWithAllContainers = Records.newRecord(LogAggregationContext.class);
contextWithAllContainers.setLogAggregationPolicyClassName(AllContainerLogAggregationPolicy.class.getName());
aggSvc.handle(new LogHandlerAppStartedEvent(appId, this.user, null, this.acls, contextWithAllContainers));
verify(spyFs).mkdirs(eq(userDir), isA(FsPermission.class));
verify(spyFs).mkdirs(eq(suffixDir), isA(FsPermission.class));
verify(spyFs).mkdirs(eq(appDir), isA(FsPermission.class));
// start another application and verify only app dir created
ApplicationId appId2 = BuilderUtils.newApplicationId(1, 2);
Path appDir2 = new Path(suffixDir, appId2.toString());
aggSvc.handle(new LogHandlerAppStartedEvent(appId2, this.user, null, this.acls, contextWithAllContainers));
verify(spyFs).mkdirs(eq(appDir2), isA(FsPermission.class));
// start another application with the app dir already created and verify
// we do not try to create it again
ApplicationId appId3 = BuilderUtils.newApplicationId(1, 3);
Path appDir3 = new Path(suffixDir, appId3.toString());
new File(appDir3.toUri().getPath()).mkdir();
aggSvc.handle(new LogHandlerAppStartedEvent(appId3, this.user, null, this.acls, contextWithAllContainers));
verify(spyFs, never()).mkdirs(eq(appDir3), isA(FsPermission.class));
aggSvc.stop();
aggSvc.close();
dispatcher.stop();
}
use of org.apache.hadoop.yarn.event.InlineDispatcher in project hadoop by apache.
the class TestRMNodeLabelsManager method testReplaceLabelsFromNode.
@Test
public void testReplaceLabelsFromNode() throws Exception {
RMContext rmContext = mock(RMContext.class);
Dispatcher syncDispatcher = new InlineDispatcher();
SchedulerEventHandler schedEventsHandler = new SchedulerEventHandler();
syncDispatcher.register(SchedulerEventType.class, schedEventsHandler);
when(rmContext.getDispatcher()).thenReturn(syncDispatcher);
mgr.setRMContext(rmContext);
mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3"));
mgr.activateNode(NodeId.newInstance("n1", 1), SMALL_RESOURCE);
mgr.activateNode(NodeId.newInstance("n2", 1), SMALL_RESOURCE);
mgr.activateNode(NodeId.newInstance("n3", 1), SMALL_RESOURCE);
mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"), toNodeId("n2:1"), toSet("p2"), toNodeId("n3"), toSet("p3")));
assertTrue("Event should be sent when there is change in labels", schedEventsHandler.receivedEvent);
assertEquals("3 node label mapping modified", 3, schedEventsHandler.updatedNodeToLabels.size());
ImmutableMap<NodeId, Set<String>> modifiedMap = ImmutableMap.of(toNodeId("n1:1"), toSet("p1"), toNodeId("n2:1"), toSet("p2"), toNodeId("n3:1"), toSet("p3"));
assertEquals("Node label mapping is not matching", modifiedMap, schedEventsHandler.updatedNodeToLabels);
schedEventsHandler.receivedEvent = false;
mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1")));
assertFalse("No event should be sent when there is no change in labels", schedEventsHandler.receivedEvent);
schedEventsHandler.receivedEvent = false;
mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n2:1"), toSet("p1"), toNodeId("n3"), toSet("p3")));
assertTrue("Event should be sent when there is change in labels", schedEventsHandler.receivedEvent);
assertEquals("Single node label mapping modified", 1, schedEventsHandler.updatedNodeToLabels.size());
assertCollectionEquals(toSet("p1"), schedEventsHandler.updatedNodeToLabels.get(toNodeId("n2:1")));
schedEventsHandler.receivedEvent = false;
mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n3"), toSet("p2")));
assertTrue("Event should be sent when there is change in labels @ HOST", schedEventsHandler.receivedEvent);
assertEquals("Single node label mapping modified", 1, schedEventsHandler.updatedNodeToLabels.size());
assertCollectionEquals(toSet("p2"), schedEventsHandler.updatedNodeToLabels.get(toNodeId("n3:1")));
schedEventsHandler.receivedEvent = false;
mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p2")));
assertTrue("Event should be sent when labels are modified at host though labels were set @ NM level", schedEventsHandler.receivedEvent);
assertEquals("Single node label mapping modified", 1, schedEventsHandler.updatedNodeToLabels.size());
assertCollectionEquals(toSet("p2"), schedEventsHandler.updatedNodeToLabels.get(toNodeId("n1:1")));
schedEventsHandler.receivedEvent = false;
}
use of org.apache.hadoop.yarn.event.InlineDispatcher in project hadoop by apache.
the class TestRMNMRPCResponseId method setUp.
@Before
public void setUp() {
Configuration conf = new Configuration();
// Dispatcher that processes events inline
Dispatcher dispatcher = new InlineDispatcher();
dispatcher.register(SchedulerEventType.class, new EventHandler<Event>() {
@Override
public void handle(Event event) {
// ignore
;
}
});
RMContext context = new RMContextImpl(dispatcher, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), null, null);
dispatcher.register(RMNodeEventType.class, new ResourceManager.NodeEventDispatcher(context));
NodesListManager nodesListManager = new NodesListManager(context);
nodesListManager.init(conf);
context.getContainerTokenSecretManager().rollMasterKey();
context.getNMTokenSecretManager().rollMasterKey();
resourceTrackerService = new ResourceTrackerService(context, nodesListManager, new NMLivelinessMonitor(dispatcher), context.getContainerTokenSecretManager(), context.getNMTokenSecretManager());
resourceTrackerService.init(conf);
}
use of org.apache.hadoop.yarn.event.InlineDispatcher in project hadoop by apache.
the class TestNMExpiry method setUp.
@Before
public void setUp() {
Configuration conf = new Configuration();
// Dispatcher that processes events inline
Dispatcher dispatcher = new InlineDispatcher();
RMContext context = new RMContextImpl(dispatcher, null, null, null, null, null, null, null, null, null);
dispatcher.register(SchedulerEventType.class, new InlineDispatcher.EmptyEventHandler());
dispatcher.register(RMNodeEventType.class, new NodeEventDispatcher(context));
NMLivelinessMonitor nmLivelinessMonitor = new TestNmLivelinessMonitor(dispatcher);
nmLivelinessMonitor.init(conf);
nmLivelinessMonitor.start();
NodesListManager nodesListManager = new NodesListManager(context);
nodesListManager.init(conf);
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf);
containerTokenSecretManager.start();
NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf);
nmTokenSecretManager.start();
resourceTrackerService = new ResourceTrackerService(context, nodesListManager, nmLivelinessMonitor, containerTokenSecretManager, nmTokenSecretManager);
resourceTrackerService.init(conf);
resourceTrackerService.start();
}
Aggregations