Examples with ResourceEvent org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent used on opensource projects

Example 1 with ResourceEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testLocalResourceCache.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void testLocalResourceCache() {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<ContainerEvent> containerEventHandler = mock(EventHandler.class);
        // Registering event handlers.
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, true, conf, new NMNullStateStoreService(), null);
        LocalResourceRequest lr = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        // Creating 2 containers for same application which will be requesting
        // same local resource.
        // Container 1 requesting local resource.
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        ResourceEvent reqEvent1 = new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc1);
        // No resource request is initially present in local cache
        Assert.assertEquals(0, localrsrc.size());
        // Container-1 requesting local resource.
        tracker.handle(reqEvent1);
        dispatcher.await();
        // New localized Resource should have been added to local resource map
        // and the requesting container will be added to its waiting queue.
        Assert.assertEquals(1, localrsrc.size());
        Assert.assertTrue(localrsrc.containsKey(lr));
        Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
        Assert.assertTrue(localrsrc.get(lr).ref.contains(cId1));
        Assert.assertEquals(ResourceState.DOWNLOADING, localrsrc.get(lr).getState());
        // Container 2 requesting the resource
        ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2);
        LocalizerContext lc2 = new LocalizerContext(user, cId2, null);
        ResourceEvent reqEvent2 = new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc2);
        tracker.handle(reqEvent2);
        dispatcher.await();
        // Container 2 should have been added to the waiting queue of the local
        // resource
        Assert.assertEquals(2, localrsrc.get(lr).getRefCount());
        Assert.assertTrue(localrsrc.get(lr).ref.contains(cId2));
        // Failing resource localization
        ResourceEvent resourceFailedEvent = new ResourceFailedLocalizationEvent(lr, (new Exception("test").getMessage()));
        // Backing up the resource to track its state change as it will be
        // removed after the failed event.
        LocalizedResource localizedResource = localrsrc.get(lr);
        tracker.handle(resourceFailedEvent);
        dispatcher.await();
        // After receiving failed resource event; all waiting containers will be
        // notified with Container Resource Failed Event.
        Assert.assertEquals(0, localrsrc.size());
        verify(containerEventHandler, timeout(1000).times(2)).handle(isA(ContainerResourceFailedEvent.class));
        Assert.assertEquals(ResourceState.FAILED, localizedResource.getState());
        // Container 1 trying to release the resource (This resource is already
        // deleted from the cache. This call should return silently without
        // exception.
        ResourceReleaseEvent relEvent1 = new ResourceReleaseEvent(lr, cId1);
        tracker.handle(relEvent1);
        dispatcher.await();
        // Container-3 now requests for the same resource. This request call
        // is coming prior to Container-2's release call.
        ContainerId cId3 = BuilderUtils.newContainerId(1, 1, 1, 3);
        LocalizerContext lc3 = new LocalizerContext(user, cId3, null);
        ResourceEvent reqEvent3 = new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc3);
        tracker.handle(reqEvent3);
        dispatcher.await();
        // Local resource cache now should have the requested resource and the
        // number of waiting containers should be 1.
        Assert.assertEquals(1, localrsrc.size());
        Assert.assertTrue(localrsrc.containsKey(lr));
        Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
        Assert.assertTrue(localrsrc.get(lr).ref.contains(cId3));
        // Container-2 Releases the resource
        ResourceReleaseEvent relEvent2 = new ResourceReleaseEvent(lr, cId2);
        tracker.handle(relEvent2);
        dispatcher.await();
        // Making sure that there is no change in the cache after the release.
        Assert.assertEquals(1, localrsrc.size());
        Assert.assertTrue(localrsrc.containsKey(lr));
        Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
        Assert.assertTrue(localrsrc.get(lr).ref.contains(cId3));
        // Sending ResourceLocalizedEvent to tracker. In turn resource should
        // send Container Resource Localized Event to waiting containers.
        Path localizedPath = new Path("/tmp/file1");
        ResourceLocalizedEvent localizedEvent = new ResourceLocalizedEvent(lr, localizedPath, 123L);
        tracker.handle(localizedEvent);
        dispatcher.await();
        // Verifying ContainerResourceLocalizedEvent .
        verify(containerEventHandler, timeout(1000).times(1)).handle(isA(ContainerResourceLocalizedEvent.class));
        Assert.assertEquals(ResourceState.LOCALIZED, localrsrc.get(lr).getState());
        Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
        // Container-3 releasing the resource.
        ResourceReleaseEvent relEvent3 = new ResourceReleaseEvent(lr, cId3);
        tracker.handle(relEvent3);
        dispatcher.await();
        Assert.assertEquals(0, localrsrc.get(lr).getRefCount());
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 2 with ResourceEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testStateStoreSuccessfulLocalization.

@Test
@SuppressWarnings("unchecked")
public void testStateStoreSuccessfulLocalization() throws Exception {
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(1, 1);
    // This is a random path. NO File creation will take place at this place.
    final Path localDir = new Path("/tmp");
    Configuration conf = new YarnConfiguration();
    DrainDispatcher dispatcher = null;
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);
    DeletionService mockDelService = mock(DeletionService.class);
    NMStateStoreService stateStore = mock(NMStateStoreService.class);
    try {
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore);
        // Container 1 needs lr1 resource
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        // Container 1 requests lr1 to be localized
        ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.APPLICATION, lc1);
        tracker.handle(reqEvent1);
        dispatcher.await();
        // Simulate the process of localization of lr1
        Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir, null);
        ArgumentCaptor<LocalResourceProto> localResourceCaptor = ArgumentCaptor.forClass(LocalResourceProto.class);
        ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
        verify(stateStore).startResourceLocalization(eq(user), eq(appId), localResourceCaptor.capture(), pathCaptor.capture());
        LocalResourceProto lrProto = localResourceCaptor.getValue();
        Path localizedPath1 = pathCaptor.getValue();
        Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
        Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());
        // Simulate lr1 getting localized
        ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(lr1, pathCaptor.getValue(), 120);
        tracker.handle(rle1);
        dispatcher.await();
        ArgumentCaptor<LocalizedResourceProto> localizedProtoCaptor = ArgumentCaptor.forClass(LocalizedResourceProto.class);
        verify(stateStore).finishResourceLocalization(eq(user), eq(appId), localizedProtoCaptor.capture());
        LocalizedResourceProto localizedProto = localizedProtoCaptor.getValue();
        Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(localizedProto.getResource())));
        Assert.assertEquals(localizedPath1.toString(), localizedProto.getLocalPath());
        LocalizedResource localizedRsrc1 = tracker.getLocalizedResource(lr1);
        Assert.assertNotNull(localizedRsrc1);
        // simulate release and retention processing
        tracker.handle(new ResourceReleaseEvent(lr1, cId1));
        dispatcher.await();
        boolean removeResult = tracker.remove(localizedRsrc1, mockDelService);
        Assert.assertTrue(removeResult);
        verify(stateStore).removeLocalizedResource(eq(user), eq(appId), eq(localizedPath1));
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 3 with ResourceEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testStateStoreFailedLocalization.

@Test
@SuppressWarnings("unchecked")
public void testStateStoreFailedLocalization() throws Exception {
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(1, 1);
    // This is a random path. NO File creation will take place at this place.
    final Path localDir = new Path("/tmp");
    Configuration conf = new YarnConfiguration();
    DrainDispatcher dispatcher = null;
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);
    NMStateStoreService stateStore = mock(NMStateStoreService.class);
    try {
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore);
        // Container 1 needs lr1 resource
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        // Container 1 requests lr1 to be localized
        ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.APPLICATION, lc1);
        tracker.handle(reqEvent1);
        dispatcher.await();
        // Simulate the process of localization of lr1
        Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir, null);
        ArgumentCaptor<LocalResourceProto> localResourceCaptor = ArgumentCaptor.forClass(LocalResourceProto.class);
        ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
        verify(stateStore).startResourceLocalization(eq(user), eq(appId), localResourceCaptor.capture(), pathCaptor.capture());
        LocalResourceProto lrProto = localResourceCaptor.getValue();
        Path localizedPath1 = pathCaptor.getValue();
        Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
        Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());
        ResourceFailedLocalizationEvent rfe1 = new ResourceFailedLocalizationEvent(lr1, new Exception("Test").toString());
        tracker.handle(rfe1);
        dispatcher.await();
        verify(stateStore).removeLocalizedResource(eq(user), eq(appId), eq(localizedPath1));
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 4 with ResourceEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testResourcePresentInGoodDir.

@SuppressWarnings("unchecked")
@Test
public void testResourcePresentInGoodDir() throws IOException {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        LocalResourceRequest req2 = createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
        LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
        LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        localrsrc.put(req1, lr1);
        localrsrc.put(req2, lr2);
        LocalDirsHandlerService dirsHandler = mock(LocalDirsHandlerService.class);
        List<String> goodDirs = new ArrayList<String>();
        // /tmp/somedir2 is bad
        goodDirs.add("/tmp/somedir1/");
        goodDirs.add("/tmp/somedir2");
        Mockito.when(dirsHandler.getLocalDirs()).thenReturn(goodDirs);
        Mockito.when(dirsHandler.getLocalDirsForRead()).thenReturn(goodDirs);
        LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, true, conf, new NMNullStateStoreService(), dirsHandler);
        ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
        ResourceEvent req21Event = new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);
        // Localize R1 for C1
        tracker.handle(req11Event);
        // Localize R2 for C1
        tracker.handle(req21Event);
        dispatcher.await();
        // Localize resource1
        Path p1 = tracker.getPathForLocalization(req1, new Path("/tmp/somedir1"), null);
        Path p2 = tracker.getPathForLocalization(req2, new Path("/tmp/somedir2"), null);
        ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(req1, p1, 1);
        tracker.handle(rle1);
        ResourceLocalizedEvent rle2 = new ResourceLocalizedEvent(req2, p2, 1);
        tracker.handle(rle2);
        dispatcher.await();
        // Remove somedir2 from gooddirs
        Assert.assertTrue(tracker.checkLocalResource(lr2));
        goodDirs.remove(1);
        Assert.assertFalse(tracker.checkLocalResource(lr2));
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 5 with ResourceEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method test.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void test() {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        DeletionService mockDelService = mock(DeletionService.class);
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2);
        LocalizerContext lc2 = new LocalizerContext(user, cId2, null);
        LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        LocalResourceRequest req2 = createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
        LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
        LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        localrsrc.put(req1, lr1);
        localrsrc.put(req2, lr2);
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, false, conf, new NMNullStateStoreService(), null);
        ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
        ResourceEvent req12Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc2);
        ResourceEvent req21Event = new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);
        ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);
        ResourceEvent rel12Event = new ResourceReleaseEvent(req1, cId2);
        ResourceEvent rel21Event = new ResourceReleaseEvent(req2, cId1);
        // Localize R1 for C1
        tracker.handle(req11Event);
        // Localize R1 for C2
        tracker.handle(req12Event);
        // Localize R2 for C1
        tracker.handle(req21Event);
        dispatcher.await();
        verify(localizerEventHandler, times(3)).handle(any(LocalizerResourceRequestEvent.class));
        // Verify refCount for R1 is 2
        Assert.assertEquals(2, lr1.getRefCount());
        // Verify refCount for R2 is 1
        Assert.assertEquals(1, lr2.getRefCount());
        // Release R2 for C1
        tracker.handle(rel21Event);
        dispatcher.await();
        verifyTrackedResourceCount(tracker, 2);
        // Verify resource with non zero ref count is not removed.
        Assert.assertEquals(2, lr1.getRefCount());
        Assert.assertFalse(tracker.remove(lr1, mockDelService));
        verifyTrackedResourceCount(tracker, 2);
        // Localize resource1
        ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path("file:///tmp/r1"), 1);
        lr1.handle(rle);
        Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
        // Release resource1
        tracker.handle(rel11Event);
        tracker.handle(rel12Event);
        Assert.assertEquals(0, lr1.getRefCount());
        // Verify resources in state LOCALIZED with ref-count=0 is removed.
        Assert.assertTrue(tracker.remove(lr1, mockDelService));
        verifyTrackedResourceCount(tracker, 1);
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}