Search in sources :

Example 1 with InlineDispatcher

use of org.apache.hadoop.yarn.event.InlineDispatcher in project hadoop by apache.

the class TestTaskImpl method setup.

@Before
@SuppressWarnings("unchecked")
public void setup() {
    dispatcher = new InlineDispatcher();
    ++startCount;
    conf = new JobConf();
    taskAttemptListener = mock(TaskAttemptListener.class);
    jobToken = (Token<JobTokenIdentifier>) mock(Token.class);
    remoteJobConfFile = mock(Path.class);
    credentials = null;
    clock = SystemClock.getInstance();
    metrics = mock(MRAppMetrics.class);
    dataLocations = new String[1];
    appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    jobId = Records.newRecord(JobId.class);
    jobId.setId(1);
    jobId.setAppId(appId);
    appContext = mock(AppContext.class);
    taskSplitMetaInfo = mock(TaskSplitMetaInfo.class);
    when(taskSplitMetaInfo.getLocations()).thenReturn(dataLocations);
    taskAttempts = new ArrayList<MockTaskAttemptImpl>();
    taskAttemptEventHandler = new MockTaskAttemptEventHandler();
    dispatcher.register(TaskAttemptEventType.class, taskAttemptEventHandler);
}
Also used : Path(org.apache.hadoop.fs.Path) TaskAttemptListener(org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener) AppContext(org.apache.hadoop.mapreduce.v2.app.AppContext) JobTokenIdentifier(org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier) InlineDispatcher(org.apache.hadoop.yarn.event.InlineDispatcher) TaskSplitMetaInfo(org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo) MRAppMetrics(org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics) JobConf(org.apache.hadoop.mapred.JobConf) JobId(org.apache.hadoop.mapreduce.v2.api.records.JobId) Before(org.junit.Before)

Example 2 with InlineDispatcher

use of org.apache.hadoop.yarn.event.InlineDispatcher in project hadoop by apache.

the class TestLogAggregationService method testAppLogDirCreation.

@Test
public void testAppLogDirCreation() throws Exception {
    final String logSuffix = "logs";
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, logSuffix);
    InlineDispatcher dispatcher = new InlineDispatcher();
    dispatcher.init(this.conf);
    dispatcher.start();
    FileSystem fs = FileSystem.get(this.conf);
    final FileSystem spyFs = spy(FileSystem.get(this.conf));
    LogAggregationService aggSvc = new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler) {

        @Override
        protected FileSystem getFileSystem(Configuration conf) {
            return spyFs;
        }
    };
    aggSvc.init(this.conf);
    aggSvc.start();
    // start an application and verify user, suffix, and app dirs created
    ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
    Path userDir = fs.makeQualified(new Path(remoteRootLogDir.getAbsolutePath(), this.user));
    Path suffixDir = new Path(userDir, logSuffix);
    Path appDir = new Path(suffixDir, appId.toString());
    LogAggregationContext contextWithAllContainers = Records.newRecord(LogAggregationContext.class);
    contextWithAllContainers.setLogAggregationPolicyClassName(AllContainerLogAggregationPolicy.class.getName());
    aggSvc.handle(new LogHandlerAppStartedEvent(appId, this.user, null, this.acls, contextWithAllContainers));
    verify(spyFs).mkdirs(eq(userDir), isA(FsPermission.class));
    verify(spyFs).mkdirs(eq(suffixDir), isA(FsPermission.class));
    verify(spyFs).mkdirs(eq(appDir), isA(FsPermission.class));
    // start another application and verify only app dir created
    ApplicationId appId2 = BuilderUtils.newApplicationId(1, 2);
    Path appDir2 = new Path(suffixDir, appId2.toString());
    aggSvc.handle(new LogHandlerAppStartedEvent(appId2, this.user, null, this.acls, contextWithAllContainers));
    verify(spyFs).mkdirs(eq(appDir2), isA(FsPermission.class));
    // start another application with the app dir already created and verify
    // we do not try to create it again
    ApplicationId appId3 = BuilderUtils.newApplicationId(1, 3);
    Path appDir3 = new Path(suffixDir, appId3.toString());
    new File(appDir3.toUri().getPath()).mkdir();
    aggSvc.handle(new LogHandlerAppStartedEvent(appId3, this.user, null, this.acls, contextWithAllContainers));
    verify(spyFs, never()).mkdirs(eq(appDir3), isA(FsPermission.class));
    aggSvc.stop();
    aggSvc.close();
    dispatcher.stop();
}
Also used : Path(org.apache.hadoop.fs.Path) LogHandlerAppStartedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) FileSystem(org.apache.hadoop.fs.FileSystem) AbstractFileSystem(org.apache.hadoop.fs.AbstractFileSystem) InlineDispatcher(org.apache.hadoop.yarn.event.InlineDispatcher) FsPermission(org.apache.hadoop.fs.permission.FsPermission) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) File(java.io.File) LogAggregationContext(org.apache.hadoop.yarn.api.records.LogAggregationContext) BaseContainerManagerTest(org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest) Test(org.junit.Test)

Example 3 with InlineDispatcher

use of org.apache.hadoop.yarn.event.InlineDispatcher in project hadoop by apache.

the class TestRMNodeLabelsManager method testReplaceLabelsFromNode.

@Test
public void testReplaceLabelsFromNode() throws Exception {
    RMContext rmContext = mock(RMContext.class);
    Dispatcher syncDispatcher = new InlineDispatcher();
    SchedulerEventHandler schedEventsHandler = new SchedulerEventHandler();
    syncDispatcher.register(SchedulerEventType.class, schedEventsHandler);
    when(rmContext.getDispatcher()).thenReturn(syncDispatcher);
    mgr.setRMContext(rmContext);
    mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3"));
    mgr.activateNode(NodeId.newInstance("n1", 1), SMALL_RESOURCE);
    mgr.activateNode(NodeId.newInstance("n2", 1), SMALL_RESOURCE);
    mgr.activateNode(NodeId.newInstance("n3", 1), SMALL_RESOURCE);
    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"), toNodeId("n2:1"), toSet("p2"), toNodeId("n3"), toSet("p3")));
    assertTrue("Event should be sent when there is change in labels", schedEventsHandler.receivedEvent);
    assertEquals("3 node label mapping modified", 3, schedEventsHandler.updatedNodeToLabels.size());
    ImmutableMap<NodeId, Set<String>> modifiedMap = ImmutableMap.of(toNodeId("n1:1"), toSet("p1"), toNodeId("n2:1"), toSet("p2"), toNodeId("n3:1"), toSet("p3"));
    assertEquals("Node label mapping is not matching", modifiedMap, schedEventsHandler.updatedNodeToLabels);
    schedEventsHandler.receivedEvent = false;
    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1")));
    assertFalse("No event should be sent when there is no change in labels", schedEventsHandler.receivedEvent);
    schedEventsHandler.receivedEvent = false;
    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n2:1"), toSet("p1"), toNodeId("n3"), toSet("p3")));
    assertTrue("Event should be sent when there is change in labels", schedEventsHandler.receivedEvent);
    assertEquals("Single node label mapping modified", 1, schedEventsHandler.updatedNodeToLabels.size());
    assertCollectionEquals(toSet("p1"), schedEventsHandler.updatedNodeToLabels.get(toNodeId("n2:1")));
    schedEventsHandler.receivedEvent = false;
    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n3"), toSet("p2")));
    assertTrue("Event should be sent when there is change in labels @ HOST", schedEventsHandler.receivedEvent);
    assertEquals("Single node label mapping modified", 1, schedEventsHandler.updatedNodeToLabels.size());
    assertCollectionEquals(toSet("p2"), schedEventsHandler.updatedNodeToLabels.get(toNodeId("n3:1")));
    schedEventsHandler.receivedEvent = false;
    mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p2")));
    assertTrue("Event should be sent when labels are modified at host though labels were set @ NM level", schedEventsHandler.receivedEvent);
    assertEquals("Single node label mapping modified", 1, schedEventsHandler.updatedNodeToLabels.size());
    assertCollectionEquals(toSet("p2"), schedEventsHandler.updatedNodeToLabels.get(toNodeId("n1:1")));
    schedEventsHandler.receivedEvent = false;
}
Also used : RMContext(org.apache.hadoop.yarn.server.resourcemanager.RMContext) ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) NodeId(org.apache.hadoop.yarn.api.records.NodeId) InlineDispatcher(org.apache.hadoop.yarn.event.InlineDispatcher) Dispatcher(org.apache.hadoop.yarn.event.Dispatcher) InlineDispatcher(org.apache.hadoop.yarn.event.InlineDispatcher) Test(org.junit.Test)

Example 4 with InlineDispatcher

use of org.apache.hadoop.yarn.event.InlineDispatcher in project hadoop by apache.

the class TestRMNMRPCResponseId method setUp.

@Before
public void setUp() {
    Configuration conf = new Configuration();
    // Dispatcher that processes events inline
    Dispatcher dispatcher = new InlineDispatcher();
    dispatcher.register(SchedulerEventType.class, new EventHandler<Event>() {

        @Override
        public void handle(Event event) {
            // ignore
            ;
        }
    });
    RMContext context = new RMContextImpl(dispatcher, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), null, null);
    dispatcher.register(RMNodeEventType.class, new ResourceManager.NodeEventDispatcher(context));
    NodesListManager nodesListManager = new NodesListManager(context);
    nodesListManager.init(conf);
    context.getContainerTokenSecretManager().rollMasterKey();
    context.getNMTokenSecretManager().rollMasterKey();
    resourceTrackerService = new ResourceTrackerService(context, nodesListManager, new NMLivelinessMonitor(dispatcher), context.getContainerTokenSecretManager(), context.getNMTokenSecretManager());
    resourceTrackerService.init(conf);
}
Also used : RMContext(org.apache.hadoop.yarn.server.resourcemanager.RMContext) Configuration(org.apache.hadoop.conf.Configuration) InlineDispatcher(org.apache.hadoop.yarn.event.InlineDispatcher) ResourceTrackerService(org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService) ResourceManager(org.apache.hadoop.yarn.server.resourcemanager.ResourceManager) Dispatcher(org.apache.hadoop.yarn.event.Dispatcher) InlineDispatcher(org.apache.hadoop.yarn.event.InlineDispatcher) NMTokenSecretManagerInRM(org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM) Event(org.apache.hadoop.yarn.event.Event) RMContainerTokenSecretManager(org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager) NMLivelinessMonitor(org.apache.hadoop.yarn.server.resourcemanager.NMLivelinessMonitor) NodesListManager(org.apache.hadoop.yarn.server.resourcemanager.NodesListManager) RMContextImpl(org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl) Before(org.junit.Before)

Example 5 with InlineDispatcher

use of org.apache.hadoop.yarn.event.InlineDispatcher in project hadoop by apache.

the class TestNMExpiry method setUp.

@Before
public void setUp() {
    Configuration conf = new Configuration();
    // Dispatcher that processes events inline
    Dispatcher dispatcher = new InlineDispatcher();
    RMContext context = new RMContextImpl(dispatcher, null, null, null, null, null, null, null, null, null);
    dispatcher.register(SchedulerEventType.class, new InlineDispatcher.EmptyEventHandler());
    dispatcher.register(RMNodeEventType.class, new NodeEventDispatcher(context));
    NMLivelinessMonitor nmLivelinessMonitor = new TestNmLivelinessMonitor(dispatcher);
    nmLivelinessMonitor.init(conf);
    nmLivelinessMonitor.start();
    NodesListManager nodesListManager = new NodesListManager(context);
    nodesListManager.init(conf);
    RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf);
    containerTokenSecretManager.start();
    NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf);
    nmTokenSecretManager.start();
    resourceTrackerService = new ResourceTrackerService(context, nodesListManager, nmLivelinessMonitor, containerTokenSecretManager, nmTokenSecretManager);
    resourceTrackerService.init(conf);
    resourceTrackerService.start();
}
Also used : RMContext(org.apache.hadoop.yarn.server.resourcemanager.RMContext) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) Configuration(org.apache.hadoop.conf.Configuration) InlineDispatcher(org.apache.hadoop.yarn.event.InlineDispatcher) ResourceTrackerService(org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService) NodeEventDispatcher(org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.NodeEventDispatcher) Dispatcher(org.apache.hadoop.yarn.event.Dispatcher) InlineDispatcher(org.apache.hadoop.yarn.event.InlineDispatcher) NMTokenSecretManagerInRM(org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM) NodeEventDispatcher(org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.NodeEventDispatcher) NMLivelinessMonitor(org.apache.hadoop.yarn.server.resourcemanager.NMLivelinessMonitor) RMContainerTokenSecretManager(org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager) NodesListManager(org.apache.hadoop.yarn.server.resourcemanager.NodesListManager) RMContextImpl(org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl) Before(org.junit.Before)

Aggregations

InlineDispatcher (org.apache.hadoop.yarn.event.InlineDispatcher)14 Configuration (org.apache.hadoop.conf.Configuration)9 RMContextImpl (org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl)8 YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)7 Test (org.junit.Test)7 RMContext (org.apache.hadoop.yarn.server.resourcemanager.RMContext)6 Before (org.junit.Before)6 RMApplicationHistoryWriter (org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter)5 SystemMetricsPublisher (org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher)5 RMContainerTokenSecretManager (org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager)5 ApplicationAttemptId (org.apache.hadoop.yarn.api.records.ApplicationAttemptId)4 NMTokenSecretManagerInRM (org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM)4 ArrayList (java.util.ArrayList)3 ApplicationId (org.apache.hadoop.yarn.api.records.ApplicationId)3 Dispatcher (org.apache.hadoop.yarn.event.Dispatcher)3 NMLivelinessMonitor (org.apache.hadoop.yarn.server.resourcemanager.NMLivelinessMonitor)3 NodesListManager (org.apache.hadoop.yarn.server.resourcemanager.NodesListManager)3 ResourceTrackerService (org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService)3 NodeAddedSchedulerEvent (org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent)3 NodeUpdateSchedulerEvent (org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent)3