Search in sources :

Example 1 with ResourceReleaseEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testLocalResourceCache.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void testLocalResourceCache() {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<ContainerEvent> containerEventHandler = mock(EventHandler.class);
        // Registering event handlers.
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, true, conf, new NMNullStateStoreService(), null);
        LocalResourceRequest lr = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        // Creating 2 containers for same application which will be requesting
        // same local resource.
        // Container 1 requesting local resource.
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        ResourceEvent reqEvent1 = new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc1);
        // No resource request is initially present in local cache
        Assert.assertEquals(0, localrsrc.size());
        // Container-1 requesting local resource.
        tracker.handle(reqEvent1);
        dispatcher.await();
        // New localized Resource should have been added to local resource map
        // and the requesting container will be added to its waiting queue.
        Assert.assertEquals(1, localrsrc.size());
        Assert.assertTrue(localrsrc.containsKey(lr));
        Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
        Assert.assertTrue(localrsrc.get(lr).ref.contains(cId1));
        Assert.assertEquals(ResourceState.DOWNLOADING, localrsrc.get(lr).getState());
        // Container 2 requesting the resource
        ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2);
        LocalizerContext lc2 = new LocalizerContext(user, cId2, null);
        ResourceEvent reqEvent2 = new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc2);
        tracker.handle(reqEvent2);
        dispatcher.await();
        // Container 2 should have been added to the waiting queue of the local
        // resource
        Assert.assertEquals(2, localrsrc.get(lr).getRefCount());
        Assert.assertTrue(localrsrc.get(lr).ref.contains(cId2));
        // Failing resource localization
        ResourceEvent resourceFailedEvent = new ResourceFailedLocalizationEvent(lr, (new Exception("test").getMessage()));
        // Backing up the resource to track its state change as it will be
        // removed after the failed event.
        LocalizedResource localizedResource = localrsrc.get(lr);
        tracker.handle(resourceFailedEvent);
        dispatcher.await();
        // After receiving failed resource event; all waiting containers will be
        // notified with Container Resource Failed Event.
        Assert.assertEquals(0, localrsrc.size());
        verify(containerEventHandler, timeout(1000).times(2)).handle(isA(ContainerResourceFailedEvent.class));
        Assert.assertEquals(ResourceState.FAILED, localizedResource.getState());
        // Container 1 trying to release the resource (This resource is already
        // deleted from the cache. This call should return silently without
        // exception.
        ResourceReleaseEvent relEvent1 = new ResourceReleaseEvent(lr, cId1);
        tracker.handle(relEvent1);
        dispatcher.await();
        // Container-3 now requests for the same resource. This request call
        // is coming prior to Container-2's release call.
        ContainerId cId3 = BuilderUtils.newContainerId(1, 1, 1, 3);
        LocalizerContext lc3 = new LocalizerContext(user, cId3, null);
        ResourceEvent reqEvent3 = new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc3);
        tracker.handle(reqEvent3);
        dispatcher.await();
        // Local resource cache now should have the requested resource and the
        // number of waiting containers should be 1.
        Assert.assertEquals(1, localrsrc.size());
        Assert.assertTrue(localrsrc.containsKey(lr));
        Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
        Assert.assertTrue(localrsrc.get(lr).ref.contains(cId3));
        // Container-2 Releases the resource
        ResourceReleaseEvent relEvent2 = new ResourceReleaseEvent(lr, cId2);
        tracker.handle(relEvent2);
        dispatcher.await();
        // Making sure that there is no change in the cache after the release.
        Assert.assertEquals(1, localrsrc.size());
        Assert.assertTrue(localrsrc.containsKey(lr));
        Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
        Assert.assertTrue(localrsrc.get(lr).ref.contains(cId3));
        // Sending ResourceLocalizedEvent to tracker. In turn resource should
        // send Container Resource Localized Event to waiting containers.
        Path localizedPath = new Path("/tmp/file1");
        ResourceLocalizedEvent localizedEvent = new ResourceLocalizedEvent(lr, localizedPath, 123L);
        tracker.handle(localizedEvent);
        dispatcher.await();
        // Verifying ContainerResourceLocalizedEvent .
        verify(containerEventHandler, timeout(1000).times(1)).handle(isA(ContainerResourceLocalizedEvent.class));
        Assert.assertEquals(ResourceState.LOCALIZED, localrsrc.get(lr).getState());
        Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
        // Container-3 releasing the resource.
        ResourceReleaseEvent relEvent3 = new ResourceReleaseEvent(lr, cId3);
        tracker.handle(relEvent3);
        dispatcher.await();
        Assert.assertEquals(0, localrsrc.get(lr).getRefCount());
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}
Also used : DrainDispatcher(org.apache.hadoop.yarn.event.DrainDispatcher) ResourceFailedLocalizationEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceFailedLocalizationEvent) ContainerResourceLocalizedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceLocalizedEvent) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ResourceLocalizedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent) ContainerResourceLocalizedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceLocalizedEvent) NMNullStateStoreService(org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) ContainerResourceFailedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent) ResourceEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ContainerEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent) Path(org.apache.hadoop.fs.Path) ResourceReleaseEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent) IOException(java.io.IOException) LocalizerEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent) LocalizerResourceRequestEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent) ResourceRequestEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent) Test(org.junit.Test)

Example 2 with ResourceReleaseEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testStateStoreSuccessfulLocalization.

@Test
@SuppressWarnings("unchecked")
public void testStateStoreSuccessfulLocalization() throws Exception {
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(1, 1);
    // This is a random path. NO File creation will take place at this place.
    final Path localDir = new Path("/tmp");
    Configuration conf = new YarnConfiguration();
    DrainDispatcher dispatcher = null;
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);
    DeletionService mockDelService = mock(DeletionService.class);
    NMStateStoreService stateStore = mock(NMStateStoreService.class);
    try {
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore);
        // Container 1 needs lr1 resource
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        // Container 1 requests lr1 to be localized
        ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.APPLICATION, lc1);
        tracker.handle(reqEvent1);
        dispatcher.await();
        // Simulate the process of localization of lr1
        Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir, null);
        ArgumentCaptor<LocalResourceProto> localResourceCaptor = ArgumentCaptor.forClass(LocalResourceProto.class);
        ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
        verify(stateStore).startResourceLocalization(eq(user), eq(appId), localResourceCaptor.capture(), pathCaptor.capture());
        LocalResourceProto lrProto = localResourceCaptor.getValue();
        Path localizedPath1 = pathCaptor.getValue();
        Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
        Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());
        // Simulate lr1 getting localized
        ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(lr1, pathCaptor.getValue(), 120);
        tracker.handle(rle1);
        dispatcher.await();
        ArgumentCaptor<LocalizedResourceProto> localizedProtoCaptor = ArgumentCaptor.forClass(LocalizedResourceProto.class);
        verify(stateStore).finishResourceLocalization(eq(user), eq(appId), localizedProtoCaptor.capture());
        LocalizedResourceProto localizedProto = localizedProtoCaptor.getValue();
        Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(localizedProto.getResource())));
        Assert.assertEquals(localizedPath1.toString(), localizedProto.getLocalPath());
        LocalizedResource localizedRsrc1 = tracker.getLocalizedResource(lr1);
        Assert.assertNotNull(localizedRsrc1);
        // simulate release and retention processing
        tracker.handle(new ResourceReleaseEvent(lr1, cId1));
        dispatcher.await();
        boolean removeResult = tracker.remove(localizedRsrc1, mockDelService);
        Assert.assertTrue(removeResult);
        verify(stateStore).removeLocalizedResource(eq(user), eq(appId), eq(localizedPath1));
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}
Also used : DrainDispatcher(org.apache.hadoop.yarn.event.DrainDispatcher) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ResourceLocalizedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent) ContainerResourceLocalizedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceLocalizedEvent) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) LocalizedResourceProto(org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto) ResourceEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent) Path(org.apache.hadoop.fs.Path) LocalResourcePBImpl(org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl) DeletionService(org.apache.hadoop.yarn.server.nodemanager.DeletionService) ResourceReleaseEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent) LocalResourceProto(org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto) NMStateStoreService(org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService) LocalizerEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent) LocalizerResourceRequestEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent) ResourceRequestEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) Test(org.junit.Test)

Example 3 with ResourceReleaseEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent in project hadoop by apache.

the class TestLocalizedResource method testNotification.

@Test
// mocked generic
@SuppressWarnings("unchecked")
public void testNotification() throws Exception {
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(new Configuration());
    try {
        dispatcher.start();
        EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
        EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class);
        dispatcher.register(ContainerEventType.class, containerBus);
        dispatcher.register(LocalizerEventType.class, localizerBus);
        // mock resource
        LocalResource apiRsrc = createMockResource();
        final ContainerId container0 = getMockContainer(0L);
        final Credentials creds0 = new Credentials();
        final LocalResourceVisibility vis0 = LocalResourceVisibility.PRIVATE;
        final LocalizerContext ctxt0 = new LocalizerContext("yak", container0, creds0);
        LocalResourceRequest rsrcA = new LocalResourceRequest(apiRsrc);
        LocalizedResource local = new LocalizedResource(rsrcA, dispatcher);
        local.handle(new ResourceRequestEvent(rsrcA, vis0, ctxt0));
        dispatcher.await();
        // Register C0, verify request event
        LocalizerEventMatcher matchesL0Req = new LocalizerEventMatcher(container0, creds0, vis0, LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
        verify(localizerBus).handle(argThat(matchesL0Req));
        assertEquals(ResourceState.DOWNLOADING, local.getState());
        // Register C1, verify request event
        final Credentials creds1 = new Credentials();
        final ContainerId container1 = getMockContainer(1L);
        final LocalizerContext ctxt1 = new LocalizerContext("yak", container1, creds1);
        final LocalResourceVisibility vis1 = LocalResourceVisibility.PUBLIC;
        local.handle(new ResourceRequestEvent(rsrcA, vis1, ctxt1));
        dispatcher.await();
        LocalizerEventMatcher matchesL1Req = new LocalizerEventMatcher(container1, creds1, vis1, LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
        verify(localizerBus).handle(argThat(matchesL1Req));
        // Release C0 container localization, verify no notification
        local.handle(new ResourceReleaseEvent(rsrcA, container0));
        dispatcher.await();
        verify(containerBus, never()).handle(isA(ContainerEvent.class));
        assertEquals(ResourceState.DOWNLOADING, local.getState());
        // Release C1 container localization, verify no notification
        local.handle(new ResourceReleaseEvent(rsrcA, container1));
        dispatcher.await();
        verify(containerBus, never()).handle(isA(ContainerEvent.class));
        assertEquals(ResourceState.DOWNLOADING, local.getState());
        // Register C2, C3
        final ContainerId container2 = getMockContainer(2L);
        final LocalResourceVisibility vis2 = LocalResourceVisibility.PRIVATE;
        final Credentials creds2 = new Credentials();
        final LocalizerContext ctxt2 = new LocalizerContext("yak", container2, creds2);
        final ContainerId container3 = getMockContainer(3L);
        final LocalResourceVisibility vis3 = LocalResourceVisibility.PRIVATE;
        final Credentials creds3 = new Credentials();
        final LocalizerContext ctxt3 = new LocalizerContext("yak", container3, creds3);
        local.handle(new ResourceRequestEvent(rsrcA, vis2, ctxt2));
        local.handle(new ResourceRequestEvent(rsrcA, vis3, ctxt3));
        dispatcher.await();
        LocalizerEventMatcher matchesL2Req = new LocalizerEventMatcher(container2, creds2, vis2, LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
        verify(localizerBus).handle(argThat(matchesL2Req));
        LocalizerEventMatcher matchesL3Req = new LocalizerEventMatcher(container3, creds3, vis3, LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
        verify(localizerBus).handle(argThat(matchesL3Req));
        // Successful localization. verify notification C2, C3
        Path locA = new Path("file:///cache/rsrcA");
        local.handle(new ResourceLocalizedEvent(rsrcA, locA, 10));
        dispatcher.await();
        ContainerEventMatcher matchesC2Localized = new ContainerEventMatcher(container2, ContainerEventType.RESOURCE_LOCALIZED);
        ContainerEventMatcher matchesC3Localized = new ContainerEventMatcher(container3, ContainerEventType.RESOURCE_LOCALIZED);
        verify(containerBus).handle(argThat(matchesC2Localized));
        verify(containerBus).handle(argThat(matchesC3Localized));
        assertEquals(ResourceState.LOCALIZED, local.getState());
        // Register C4, verify notification
        final ContainerId container4 = getMockContainer(4L);
        final Credentials creds4 = new Credentials();
        final LocalizerContext ctxt4 = new LocalizerContext("yak", container4, creds4);
        final LocalResourceVisibility vis4 = LocalResourceVisibility.PRIVATE;
        local.handle(new ResourceRequestEvent(rsrcA, vis4, ctxt4));
        dispatcher.await();
        ContainerEventMatcher matchesC4Localized = new ContainerEventMatcher(container4, ContainerEventType.RESOURCE_LOCALIZED);
        verify(containerBus).handle(argThat(matchesC4Localized));
        assertEquals(ResourceState.LOCALIZED, local.getState());
    } finally {
        dispatcher.stop();
    }
}
Also used : DrainDispatcher(org.apache.hadoop.yarn.event.DrainDispatcher) ContainerEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent) Path(org.apache.hadoop.fs.Path) Configuration(org.apache.hadoop.conf.Configuration) ResourceLocalizedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent) ResourceReleaseEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent) LocalResource(org.apache.hadoop.yarn.api.records.LocalResource) LocalResourceVisibility(org.apache.hadoop.yarn.api.records.LocalResourceVisibility) LocalizerEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) LocalizerResourceRequestEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent) ResourceRequestEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent) Credentials(org.apache.hadoop.security.Credentials) Test(org.junit.Test)

Example 4 with ResourceReleaseEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent in project hadoop by apache.

the class LocalResourcesTrackerImpl method handle.

/*
   * Synchronizing this method for avoiding races due to multiple ResourceEvent's
   * coming to LocalResourcesTracker from Public/Private localizer and
   * Resource Localization Service.
   */
@Override
public synchronized void handle(ResourceEvent event) {
    LocalResourceRequest req = event.getLocalResourceRequest();
    LocalizedResource rsrc = localrsrc.get(req);
    switch(event.getType()) {
        case LOCALIZED:
            if (useLocalCacheDirectoryManager) {
                inProgressLocalResourcesMap.remove(req);
            }
            break;
        case REQUEST:
            if (rsrc != null && (!isResourcePresent(rsrc))) {
                LOG.info("Resource " + rsrc.getLocalPath() + " is missing, localizing it again");
                removeResource(req);
                rsrc = null;
            }
            if (null == rsrc) {
                rsrc = new LocalizedResource(req, dispatcher);
                localrsrc.put(req, rsrc);
            }
            break;
        case RELEASE:
            if (null == rsrc) {
                // The container sent a release event on a resource which 
                // 1) Failed
                // 2) Removed for some reason (ex. disk is no longer accessible)
                ResourceReleaseEvent relEvent = (ResourceReleaseEvent) event;
                LOG.info("Container " + relEvent.getContainer() + " sent RELEASE event on a resource request " + req + " not present in cache.");
                return;
            }
            break;
        case LOCALIZATION_FAILED:
            /*
       * If resource localization fails then Localized resource will be
       * removed from local cache.
       */
            removeResource(req);
            break;
        case RECOVERED:
            if (rsrc != null) {
                LOG.warn("Ignoring attempt to recover existing resource " + rsrc);
                return;
            }
            rsrc = recoverResource(req, (ResourceRecoveredEvent) event);
            localrsrc.put(req, rsrc);
            break;
    }
    if (rsrc == null) {
        LOG.warn("Received " + event.getType() + " event for request " + req + " but localized resource is missing");
        return;
    }
    rsrc.handle(event);
    //       download is not associated with a container-specific localizer.
    if (event.getType() == ResourceEventType.RELEASE) {
        if (rsrc.getState() == ResourceState.DOWNLOADING && rsrc.getRefCount() <= 0 && rsrc.getRequest().getVisibility() != LocalResourceVisibility.PUBLIC) {
            removeResource(req);
        }
    }
    if (event.getType() == ResourceEventType.LOCALIZED) {
        if (rsrc.getLocalPath() != null) {
            try {
                stateStore.finishResourceLocalization(user, appId, buildLocalizedResourceProto(rsrc));
            } catch (IOException ioe) {
                LOG.error("Error storing resource state for " + rsrc, ioe);
            }
        } else {
            LOG.warn("Resource " + rsrc + " localized without a location");
        }
    }
}
Also used : ResourceRecoveredEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRecoveredEvent) IOException(java.io.IOException) ResourceReleaseEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent)

Example 5 with ResourceReleaseEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent in project hadoop by apache.

the class ResourceLocalizationService method handleCleanupContainerResources.

@SuppressWarnings("unchecked")
private void handleCleanupContainerResources(ContainerLocalizationCleanupEvent rsrcCleanup) {
    Container c = rsrcCleanup.getContainer();
    Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = rsrcCleanup.getResources();
    for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e : rsrcs.entrySet()) {
        LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(), c.getContainerId().getApplicationAttemptId().getApplicationId());
        for (LocalResourceRequest req : e.getValue()) {
            tracker.handle(new ResourceReleaseEvent(req, c.getContainerId()));
        }
    }
    String locId = c.getContainerId().toString();
    localizerTracker.cleanupPrivLocalizers(locId);
    // Delete the container directories
    String userName = c.getUser();
    String containerIDStr = c.toString();
    String appIDStr = c.getContainerId().getApplicationAttemptId().getApplicationId().toString();
    for (String localDir : dirsHandler.getLocalDirsForCleanup()) {
        // Delete the user-owned container-dir
        Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE);
        Path userdir = new Path(usersdir, userName);
        Path allAppsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
        Path appDir = new Path(allAppsdir, appIDStr);
        Path containerDir = new Path(appDir, containerIDStr);
        submitDirForDeletion(userName, containerDir);
        // Delete the nmPrivate container-dir
        Path sysDir = new Path(localDir, NM_PRIVATE_DIR);
        Path appSysDir = new Path(sysDir, appIDStr);
        Path containerSysDir = new Path(appSysDir, containerIDStr);
        submitDirForDeletion(null, containerSysDir);
    }
    dispatcher.getEventHandler().handle(new ContainerEvent(c.getContainerId(), ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
}
Also used : Path(org.apache.hadoop.fs.Path) ContainerEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent) Container(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container) Collection(java.util.Collection) ResourceReleaseEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ConcurrentMap(java.util.concurrent.ConcurrentMap) LocalResourceVisibility(org.apache.hadoop.yarn.api.records.LocalResourceVisibility)

Aggregations

ResourceReleaseEvent (org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent)9 Path (org.apache.hadoop.fs.Path)8 Configuration (org.apache.hadoop.conf.Configuration)7 ContainerId (org.apache.hadoop.yarn.api.records.ContainerId)7 DrainDispatcher (org.apache.hadoop.yarn.event.DrainDispatcher)7 LocalizerEvent (org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent)7 LocalizerResourceRequestEvent (org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent)7 ResourceLocalizedEvent (org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent)7 ResourceRequestEvent (org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent)7 Test (org.junit.Test)7 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)6 YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)6 ContainerResourceLocalizedEvent (org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceLocalizedEvent)6 ResourceEvent (org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent)6 NMNullStateStoreService (org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService)5 IOException (java.io.IOException)3 DeletionService (org.apache.hadoop.yarn.server.nodemanager.DeletionService)3 ContainerEvent (org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent)3 LocalResourceVisibility (org.apache.hadoop.yarn.api.records.LocalResourceVisibility)2 ResourceFailedLocalizationEvent (org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceFailedLocalizationEvent)2