Search in sources :

Example 6 with InlineDispatcher

use of org.apache.hadoop.yarn.event.InlineDispatcher in project hadoop by apache.

the class TestNMReconnect method setUp.

@Before
public void setUp() {
    Configuration conf = new Configuration();
    // Dispatcher that processes events inline
    dispatcher = new InlineDispatcher();
    dispatcher.register(RMNodeEventType.class, new TestRMNodeEventDispatcher());
    context = new RMContextImpl(dispatcher, null, null, null, null, null, null, null, null, null);
    dispatcher.register(SchedulerEventType.class, new InlineDispatcher.EmptyEventHandler());
    dispatcher.register(RMNodeEventType.class, new NodeEventDispatcher(context));
    NMLivelinessMonitor nmLivelinessMonitor = new NMLivelinessMonitor(dispatcher);
    nmLivelinessMonitor.init(conf);
    nmLivelinessMonitor.start();
    NodesListManager nodesListManager = new NodesListManager(context);
    nodesListManager.init(conf);
    RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf);
    containerTokenSecretManager.start();
    NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf);
    nmTokenSecretManager.start();
    resourceTrackerService = new ResourceTrackerService(context, nodesListManager, nmLivelinessMonitor, containerTokenSecretManager, nmTokenSecretManager);
    resourceTrackerService.init(conf);
    resourceTrackerService.start();
}
Also used : YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) Configuration(org.apache.hadoop.conf.Configuration) InlineDispatcher(org.apache.hadoop.yarn.event.InlineDispatcher) ResourceTrackerService(org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService) NodeEventDispatcher(org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.NodeEventDispatcher) NMLivelinessMonitor(org.apache.hadoop.yarn.server.resourcemanager.NMLivelinessMonitor) RMContainerTokenSecretManager(org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager) NodesListManager(org.apache.hadoop.yarn.server.resourcemanager.NodesListManager) NMTokenSecretManagerInRM(org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM) RMContextImpl(org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl) Before(org.junit.Before)

Example 7 with InlineDispatcher

use of org.apache.hadoop.yarn.event.InlineDispatcher in project hadoop by apache.

the class TestRMAppAttemptTransitions method setUp.

@SuppressWarnings("deprecation")
@Before
public void setUp() throws Exception {
    AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
    if (isSecurityEnabled) {
        authMethod = AuthenticationMethod.KERBEROS;
    }
    SecurityUtil.setAuthenticationMethod(authMethod, conf);
    UserGroupInformation.setConfiguration(conf);
    InlineDispatcher rmDispatcher = new InlineDispatcher();
    ContainerAllocationExpirer containerAllocationExpirer = mock(ContainerAllocationExpirer.class);
    amLivelinessMonitor = mock(AMLivelinessMonitor.class);
    amFinishingMonitor = mock(AMLivelinessMonitor.class);
    writer = mock(RMApplicationHistoryWriter.class);
    MasterKeyData masterKeyData = amRMTokenManager.createNewMasterKey();
    when(amRMTokenManager.getMasterKey()).thenReturn(masterKeyData);
    rmContext = new RMContextImpl(rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, null, amRMTokenManager, new RMContainerTokenSecretManager(conf), nmTokenManager, clientToAMTokenManager);
    store = mock(RMStateStore.class);
    ((RMContextImpl) rmContext).setStateStore(store);
    publisher = mock(SystemMetricsPublisher.class);
    rmContext.setSystemMetricsPublisher(publisher);
    rmContext.setRMApplicationHistoryWriter(writer);
    scheduler = mock(YarnScheduler.class);
    masterService = mock(ApplicationMasterService.class);
    applicationMasterLauncher = mock(ApplicationMasterLauncher.class);
    rmDispatcher.register(RMAppAttemptEventType.class, new TestApplicationAttemptEventDispatcher());
    rmDispatcher.register(RMAppEventType.class, new TestApplicationEventDispatcher());
    rmDispatcher.register(SchedulerEventType.class, new TestSchedulerEventDispatcher());
    rmDispatcher.register(AMLauncherEventType.class, new TestAMLauncherEventDispatcher());
    rmnodeEventHandler = mock(RMNodeImpl.class);
    rmDispatcher.register(RMNodeEventType.class, rmnodeEventHandler);
    rmDispatcher.init(conf);
    rmDispatcher.start();
    ApplicationId applicationId = MockApps.newAppID(appId++);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
    resourceScheduler = mock(ResourceScheduler.class);
    ApplicationResourceUsageReport appResUsgRpt = mock(ApplicationResourceUsageReport.class);
    when(appResUsgRpt.getMemorySeconds()).thenReturn(0L);
    when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L);
    when(resourceScheduler.getAppResourceUsageReport((ApplicationAttemptId) Matchers.any())).thenReturn(appResUsgRpt);
    spyRMContext = spy(rmContext);
    Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler();
    final String user = MockApps.newUserName();
    final String queue = MockApps.newQueue();
    submissionContext = mock(ApplicationSubmissionContext.class);
    when(submissionContext.getQueue()).thenReturn(queue);
    Resource resource = BuilderUtils.newResource(1536, 1);
    ContainerLaunchContext amContainerSpec = BuilderUtils.newContainerLaunchContext(null, null, null, null, null, null);
    when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec);
    when(submissionContext.getResource()).thenReturn(resource);
    unmanagedAM = false;
    application = mock(RMAppImpl.class);
    applicationAttempt = new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler, masterService, submissionContext, new Configuration(), BuilderUtils.newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1), application);
    when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
    when(application.getApplicationId()).thenReturn(applicationId);
    spyRMContext.getRMApps().put(application.getApplicationId(), application);
    testAppAttemptNewState();
}
Also used : ApplicationMasterService(org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ContainerAllocationExpirer(org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer) ApplicationMasterLauncher(org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher) InlineDispatcher(org.apache.hadoop.yarn.event.InlineDispatcher) AuthenticationMethod(org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod) SystemMetricsPublisher(org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher) YarnScheduler(org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler) ApplicationSubmissionContext(org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext) ResourceScheduler(org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler) RMContainerTokenSecretManager(org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager) RMContextImpl(org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl) MasterKeyData(org.apache.hadoop.yarn.server.security.MasterKeyData) RMAppImpl(org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl) RMStateStore(org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore) Resource(org.apache.hadoop.yarn.api.records.Resource) RMApplicationHistoryWriter(org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) ApplicationResourceUsageReport(org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport) RMNodeImpl(org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) Before(org.junit.Before)

Example 8 with InlineDispatcher

use of org.apache.hadoop.yarn.event.InlineDispatcher in project hadoop by apache.

the class TestFifoScheduler method testNodeLocalAssignment.

@Test(timeout = 2000)
public void testNodeLocalAssignment() throws Exception {
    AsyncDispatcher dispatcher = new InlineDispatcher();
    Configuration conf = new Configuration();
    RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf);
    containerTokenSecretManager.rollMasterKey();
    NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf);
    nmTokenSecretManager.rollMasterKey();
    RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
    FifoScheduler scheduler = new FifoScheduler();
    RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
    rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
    rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
    ((RMContextImpl) rmContext).setYarnConfiguration(new YarnConfiguration());
    scheduler.setRMContext(rmContext);
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(new Configuration(), rmContext);
    RMNode node0 = MockNodes.newNodeInfo(1, Resources.createResource(1024 * 64), 1, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
    scheduler.handle(nodeEvent1);
    int _appId = 1;
    int _appAttemptId = 1;
    ApplicationAttemptId appAttemptId = createAppAttemptId(_appId, _appAttemptId);
    createMockRMApp(appAttemptId, rmContext);
    AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue1", "user1");
    scheduler.handle(appEvent);
    AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    scheduler.handle(attemptEvent);
    int memory = 64;
    int nConts = 3;
    int priority = 20;
    List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
    ResourceRequest nodeLocal = createResourceRequest(memory, node0.getHostName(), priority, nConts);
    ResourceRequest rackLocal = createResourceRequest(memory, node0.getRackName(), priority, nConts);
    ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority, nConts);
    ask.add(nodeLocal);
    ask.add(rackLocal);
    ask.add(any);
    scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
    NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
    // Before the node update event, there are 3 local requests outstanding
    Assert.assertEquals(3, nodeLocal.getNumContainers());
    scheduler.handle(node0Update);
    // After the node update event, check that there are no more local requests
    // outstanding
    Assert.assertEquals(0, nodeLocal.getNumContainers());
    //Also check that the containers were scheduled
    SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
    Assert.assertEquals(3, info.getLiveContainers().size());
    scheduler.stop();
}
Also used : RMContext(org.apache.hadoop.yarn.server.resourcemanager.RMContext) NodeUpdateSchedulerEvent(org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent) NodeAddedSchedulerEvent(org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent) CapacitySchedulerConfiguration(org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) AppAddedSchedulerEvent(org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent) ArrayList(java.util.ArrayList) InlineDispatcher(org.apache.hadoop.yarn.event.InlineDispatcher) RMApplicationHistoryWriter(org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) NMTokenSecretManagerInRM(org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM) RMNode(org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode) AsyncDispatcher(org.apache.hadoop.yarn.event.AsyncDispatcher) SystemMetricsPublisher(org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) RMContainerTokenSecretManager(org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager) AppAttemptAddedSchedulerEvent(org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent) UpdateNodeResourceRequest(org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest) ResourceRequest(org.apache.hadoop.yarn.api.records.ResourceRequest) RMContextImpl(org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl) SchedulerAppReport(org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport) Test(org.junit.Test)

Example 9 with InlineDispatcher

use of org.apache.hadoop.yarn.event.InlineDispatcher in project hadoop by apache.

the class TestRMAppLogAggregationStatus method setUp.

@Before
public void setUp() throws Exception {
    InlineDispatcher rmDispatcher = new InlineDispatcher();
    rmContext = new RMContextImpl(rmDispatcher, null, null, null, null, null, null, null, null);
    rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
    rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
    rmContext.setRMTimelineCollectorManager(mock(RMTimelineCollectorManager.class));
    scheduler = mock(YarnScheduler.class);
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            final SchedulerEvent event = (SchedulerEvent) (invocation.getArguments()[0]);
            eventType = event.getType();
            if (eventType == SchedulerEventType.NODE_UPDATE) {
            //DO NOTHING
            }
            return null;
        }
    }).when(scheduler).handle(any(SchedulerEvent.class));
    rmDispatcher.register(SchedulerEventType.class, new TestSchedulerEventDispatcher());
    appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
}
Also used : InlineDispatcher(org.apache.hadoop.yarn.event.InlineDispatcher) RMApplicationHistoryWriter(org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter) SchedulerEvent(org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent) RMTimelineCollectorManager(org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager) SystemMetricsPublisher(org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher) YarnScheduler(org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler) InvocationOnMock(org.mockito.invocation.InvocationOnMock) RMContextImpl(org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl) Before(org.junit.Before)

Example 10 with InlineDispatcher

use of org.apache.hadoop.yarn.event.InlineDispatcher in project hadoop by apache.

the class TestRMNodeTransitions method setUp.

@Before
public void setUp() throws Exception {
    InlineDispatcher rmDispatcher = new InlineDispatcher();
    rmContext = new RMContextImpl(rmDispatcher, mock(ContainerAllocationExpirer.class), null, null, mock(DelegationTokenRenewer.class), null, null, null, null, null);
    NodesListManager nodesListManager = mock(NodesListManager.class);
    HostsFileReader reader = mock(HostsFileReader.class);
    when(nodesListManager.getHostsReader()).thenReturn(reader);
    ((RMContextImpl) rmContext).setNodesListManager(nodesListManager);
    scheduler = mock(YarnScheduler.class);
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            final SchedulerEvent event = (SchedulerEvent) (invocation.getArguments()[0]);
            eventType = event.getType();
            if (eventType == SchedulerEventType.NODE_UPDATE) {
                List<UpdatedContainerInfo> lastestContainersInfoList = ((NodeUpdateSchedulerEvent) event).getRMNode().pullContainerUpdates();
                for (UpdatedContainerInfo lastestContainersInfo : lastestContainersInfoList) {
                    completedContainers.addAll(lastestContainersInfo.getCompletedContainers());
                }
            }
            return null;
        }
    }).when(scheduler).handle(any(SchedulerEvent.class));
    rmDispatcher.register(SchedulerEventType.class, new TestSchedulerEventDispatcher());
    rmDispatcher.register(NodesListManagerEventType.class, new TestNodeListManagerEventDispatcher());
    NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
    node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
    nodesListManagerEvent = null;
}
Also used : NodeUpdateSchedulerEvent(org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent) InlineDispatcher(org.apache.hadoop.yarn.event.InlineDispatcher) NodeAddedSchedulerEvent(org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent) SchedulerEvent(org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent) NodeUpdateSchedulerEvent(org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent) NodeRemovedSchedulerEvent(org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent) HostsFileReader(org.apache.hadoop.util.HostsFileReader) YarnScheduler(org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler) InvocationOnMock(org.mockito.invocation.InvocationOnMock) UpdatedContainerInfo(org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo) NodeId(org.apache.hadoop.yarn.api.records.NodeId) List(java.util.List) ArrayList(java.util.ArrayList) RMNodeImpl(org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl) Before(org.junit.Before)

Aggregations

InlineDispatcher (org.apache.hadoop.yarn.event.InlineDispatcher)14 Configuration (org.apache.hadoop.conf.Configuration)9 RMContextImpl (org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl)8 YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)7 Test (org.junit.Test)7 RMContext (org.apache.hadoop.yarn.server.resourcemanager.RMContext)6 Before (org.junit.Before)6 RMApplicationHistoryWriter (org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter)5 SystemMetricsPublisher (org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher)5 RMContainerTokenSecretManager (org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager)5 ApplicationAttemptId (org.apache.hadoop.yarn.api.records.ApplicationAttemptId)4 NMTokenSecretManagerInRM (org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM)4 ArrayList (java.util.ArrayList)3 ApplicationId (org.apache.hadoop.yarn.api.records.ApplicationId)3 Dispatcher (org.apache.hadoop.yarn.event.Dispatcher)3 NMLivelinessMonitor (org.apache.hadoop.yarn.server.resourcemanager.NMLivelinessMonitor)3 NodesListManager (org.apache.hadoop.yarn.server.resourcemanager.NodesListManager)3 ResourceTrackerService (org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService)3 NodeAddedSchedulerEvent (org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent)3 NodeUpdateSchedulerEvent (org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent)3