Examples with DrainDispatcher org.apache.hadoop.yarn.event.DrainDispatcher used on opensource projects

Example 26 with DrainDispatcher

use of org.apache.hadoop.yarn.event.DrainDispatcher in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method createDispatcher.

private DrainDispatcher createDispatcher(Configuration conf) {
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    return dispatcher;
}

Example 27 with DrainDispatcher

use of org.apache.hadoop.yarn.event.DrainDispatcher in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testStateStoreFailedLocalization.

@Test
@SuppressWarnings("unchecked")
public void testStateStoreFailedLocalization() throws Exception {
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(1, 1);
    // This is a random path. NO File creation will take place at this place.
    final Path localDir = new Path("/tmp");
    Configuration conf = new YarnConfiguration();
    DrainDispatcher dispatcher = null;
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);
    NMStateStoreService stateStore = mock(NMStateStoreService.class);
    try {
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore);
        // Container 1 needs lr1 resource
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        // Container 1 requests lr1 to be localized
        ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.APPLICATION, lc1);
        tracker.handle(reqEvent1);
        dispatcher.await();
        // Simulate the process of localization of lr1
        Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir, null);
        ArgumentCaptor<LocalResourceProto> localResourceCaptor = ArgumentCaptor.forClass(LocalResourceProto.class);
        ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
        verify(stateStore).startResourceLocalization(eq(user), eq(appId), localResourceCaptor.capture(), pathCaptor.capture());
        LocalResourceProto lrProto = localResourceCaptor.getValue();
        Path localizedPath1 = pathCaptor.getValue();
        Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
        Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());
        ResourceFailedLocalizationEvent rfe1 = new ResourceFailedLocalizationEvent(lr1, new Exception("Test").toString());
        tracker.handle(rfe1);
        dispatcher.await();
        verify(stateStore).removeLocalizedResource(eq(user), eq(appId), eq(localizedPath1));
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 28 with DrainDispatcher

use of org.apache.hadoop.yarn.event.DrainDispatcher in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testResourcePresentInGoodDir.

@SuppressWarnings("unchecked")
@Test
public void testResourcePresentInGoodDir() throws IOException {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        LocalResourceRequest req2 = createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
        LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
        LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        localrsrc.put(req1, lr1);
        localrsrc.put(req2, lr2);
        LocalDirsHandlerService dirsHandler = mock(LocalDirsHandlerService.class);
        List<String> goodDirs = new ArrayList<String>();
        // /tmp/somedir2 is bad
        goodDirs.add("/tmp/somedir1/");
        goodDirs.add("/tmp/somedir2");
        Mockito.when(dirsHandler.getLocalDirs()).thenReturn(goodDirs);
        Mockito.when(dirsHandler.getLocalDirsForRead()).thenReturn(goodDirs);
        LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, true, conf, new NMNullStateStoreService(), dirsHandler);
        ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
        ResourceEvent req21Event = new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);
        // Localize R1 for C1
        tracker.handle(req11Event);
        // Localize R2 for C1
        tracker.handle(req21Event);
        dispatcher.await();
        // Localize resource1
        Path p1 = tracker.getPathForLocalization(req1, new Path("/tmp/somedir1"), null);
        Path p2 = tracker.getPathForLocalization(req2, new Path("/tmp/somedir2"), null);
        ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(req1, p1, 1);
        tracker.handle(rle1);
        ResourceLocalizedEvent rle2 = new ResourceLocalizedEvent(req2, p2, 1);
        tracker.handle(rle2);
        dispatcher.await();
        // Remove somedir2 from gooddirs
        Assert.assertTrue(tracker.checkLocalResource(lr2));
        goodDirs.remove(1);
        Assert.assertFalse(tracker.checkLocalResource(lr2));
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 29 with DrainDispatcher

use of org.apache.hadoop.yarn.event.DrainDispatcher in project hadoop by apache.

the class TestLocalizedResource method testNotification.

@Test
// mocked generic
@SuppressWarnings("unchecked")
public void testNotification() throws Exception {
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(new Configuration());
    try {
        dispatcher.start();
        EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
        EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class);
        dispatcher.register(ContainerEventType.class, containerBus);
        dispatcher.register(LocalizerEventType.class, localizerBus);
        // mock resource
        LocalResource apiRsrc = createMockResource();
        final ContainerId container0 = getMockContainer(0L);
        final Credentials creds0 = new Credentials();
        final LocalResourceVisibility vis0 = LocalResourceVisibility.PRIVATE;
        final LocalizerContext ctxt0 = new LocalizerContext("yak", container0, creds0);
        LocalResourceRequest rsrcA = new LocalResourceRequest(apiRsrc);
        LocalizedResource local = new LocalizedResource(rsrcA, dispatcher);
        local.handle(new ResourceRequestEvent(rsrcA, vis0, ctxt0));
        dispatcher.await();
        // Register C0, verify request event
        LocalizerEventMatcher matchesL0Req = new LocalizerEventMatcher(container0, creds0, vis0, LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
        verify(localizerBus).handle(argThat(matchesL0Req));
        assertEquals(ResourceState.DOWNLOADING, local.getState());
        // Register C1, verify request event
        final Credentials creds1 = new Credentials();
        final ContainerId container1 = getMockContainer(1L);
        final LocalizerContext ctxt1 = new LocalizerContext("yak", container1, creds1);
        final LocalResourceVisibility vis1 = LocalResourceVisibility.PUBLIC;
        local.handle(new ResourceRequestEvent(rsrcA, vis1, ctxt1));
        dispatcher.await();
        LocalizerEventMatcher matchesL1Req = new LocalizerEventMatcher(container1, creds1, vis1, LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
        verify(localizerBus).handle(argThat(matchesL1Req));
        // Release C0 container localization, verify no notification
        local.handle(new ResourceReleaseEvent(rsrcA, container0));
        dispatcher.await();
        verify(containerBus, never()).handle(isA(ContainerEvent.class));
        assertEquals(ResourceState.DOWNLOADING, local.getState());
        // Release C1 container localization, verify no notification
        local.handle(new ResourceReleaseEvent(rsrcA, container1));
        dispatcher.await();
        verify(containerBus, never()).handle(isA(ContainerEvent.class));
        assertEquals(ResourceState.DOWNLOADING, local.getState());
        // Register C2, C3
        final ContainerId container2 = getMockContainer(2L);
        final LocalResourceVisibility vis2 = LocalResourceVisibility.PRIVATE;
        final Credentials creds2 = new Credentials();
        final LocalizerContext ctxt2 = new LocalizerContext("yak", container2, creds2);
        final ContainerId container3 = getMockContainer(3L);
        final LocalResourceVisibility vis3 = LocalResourceVisibility.PRIVATE;
        final Credentials creds3 = new Credentials();
        final LocalizerContext ctxt3 = new LocalizerContext("yak", container3, creds3);
        local.handle(new ResourceRequestEvent(rsrcA, vis2, ctxt2));
        local.handle(new ResourceRequestEvent(rsrcA, vis3, ctxt3));
        dispatcher.await();
        LocalizerEventMatcher matchesL2Req = new LocalizerEventMatcher(container2, creds2, vis2, LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
        verify(localizerBus).handle(argThat(matchesL2Req));
        LocalizerEventMatcher matchesL3Req = new LocalizerEventMatcher(container3, creds3, vis3, LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
        verify(localizerBus).handle(argThat(matchesL3Req));
        // Successful localization. verify notification C2, C3
        Path locA = new Path("file:///cache/rsrcA");
        local.handle(new ResourceLocalizedEvent(rsrcA, locA, 10));
        dispatcher.await();
        ContainerEventMatcher matchesC2Localized = new ContainerEventMatcher(container2, ContainerEventType.RESOURCE_LOCALIZED);
        ContainerEventMatcher matchesC3Localized = new ContainerEventMatcher(container3, ContainerEventType.RESOURCE_LOCALIZED);
        verify(containerBus).handle(argThat(matchesC2Localized));
        verify(containerBus).handle(argThat(matchesC3Localized));
        assertEquals(ResourceState.LOCALIZED, local.getState());
        // Register C4, verify notification
        final ContainerId container4 = getMockContainer(4L);
        final Credentials creds4 = new Credentials();
        final LocalizerContext ctxt4 = new LocalizerContext("yak", container4, creds4);
        final LocalResourceVisibility vis4 = LocalResourceVisibility.PRIVATE;
        local.handle(new ResourceRequestEvent(rsrcA, vis4, ctxt4));
        dispatcher.await();
        ContainerEventMatcher matchesC4Localized = new ContainerEventMatcher(container4, ContainerEventType.RESOURCE_LOCALIZED);
        verify(containerBus).handle(argThat(matchesC4Localized));
        assertEquals(ResourceState.LOCALIZED, local.getState());
    } finally {
        dispatcher.stop();
    }
}

Example 30 with DrainDispatcher

use of org.apache.hadoop.yarn.event.DrainDispatcher in project hadoop by apache.

the class TestRM method testApplicationKillAtAcceptedState.

/**
   * Validate killing an application when it is at accepted state.
   * @throws Exception exception
   */
@Test(timeout = 60000)
public void testApplicationKillAtAcceptedState() throws Exception {
    final Dispatcher dispatcher = new DrainDispatcher() {

        @Override
        public EventHandler<Event> getEventHandler() {
            class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {

                @Override
                public boolean matches(Object argument) {
                    if (argument instanceof RMAppAttemptEvent) {
                        if (((RMAppAttemptEvent) argument).getType().equals(RMAppAttemptEventType.KILL)) {
                            return true;
                        }
                    }
                    return false;
                }
            }
            EventHandler handler = spy(super.getEventHandler());
            doNothing().when(handler).handle(argThat(new EventArgMatcher()));
            return handler;
        }
    };
    MockRM rm = new MockRM(conf) {

        @Override
        protected Dispatcher createDispatcher() {
            return dispatcher;
        }
    };
    // test metrics
    QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
    int appsKilled = metrics.getAppsKilled();
    int appsSubmitted = metrics.getAppsSubmitted();
    rm.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
    nm1.registerNode();
    // a failed app
    RMApp application = rm.submitApp(200);
    MockAM am = MockRM.launchAM(application, rm, nm1);
    rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.LAUNCHED);
    nm1.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.RUNNING);
    rm.waitForState(application.getApplicationId(), RMAppState.ACCEPTED);
    // Now kill the application before new attempt is launched, the app report
    // returns the invalid AM host and port.
    KillApplicationRequest request = KillApplicationRequest.newInstance(application.getApplicationId());
    rm.getClientRMService().forceKillApplication(request);
    // Specific test for YARN-1689 follows
    // Now let's say a race causes AM to register now. This should not crash RM.
    am.registerAppAttempt(false);
    // We explicitly intercepted the kill-event to RMAppAttempt, so app should
    // still be in KILLING state.
    rm.waitForState(application.getApplicationId(), RMAppState.KILLING);
    // AM should now be in running
    rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
    // Simulate that appAttempt is killed.
    rm.getRMContext().getDispatcher().getEventHandler().handle(new RMAppEvent(application.getApplicationId(), RMAppEventType.ATTEMPT_KILLED));
    rm.waitForState(application.getApplicationId(), RMAppState.KILLED);
    // test metrics
    metrics = rm.getResourceScheduler().getRootQueueMetrics();
    Assert.assertEquals(appsKilled + 1, metrics.getAppsKilled());
    Assert.assertEquals(appsSubmitted + 1, metrics.getAppsSubmitted());
}