Examples with ResourceRecoveredEvent org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRecoveredEvent used on opensource projects

Example 1 with ResourceRecoveredEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRecoveredEvent in project hadoop by apache.

the class LocalResourcesTrackerImpl method handle.

/*
   * Synchronizing this method for avoiding races due to multiple ResourceEvent's
   * coming to LocalResourcesTracker from Public/Private localizer and
   * Resource Localization Service.
   */
@Override
public synchronized void handle(ResourceEvent event) {
    LocalResourceRequest req = event.getLocalResourceRequest();
    LocalizedResource rsrc = localrsrc.get(req);
    switch(event.getType()) {
        case LOCALIZED:
            if (useLocalCacheDirectoryManager) {
                inProgressLocalResourcesMap.remove(req);
            }
            break;
        case REQUEST:
            if (rsrc != null && (!isResourcePresent(rsrc))) {
                LOG.info("Resource " + rsrc.getLocalPath() + " is missing, localizing it again");
                removeResource(req);
                rsrc = null;
            }
            if (null == rsrc) {
                rsrc = new LocalizedResource(req, dispatcher);
                localrsrc.put(req, rsrc);
            }
            break;
        case RELEASE:
            if (null == rsrc) {
                // The container sent a release event on a resource which 
                // 1) Failed
                // 2) Removed for some reason (ex. disk is no longer accessible)
                ResourceReleaseEvent relEvent = (ResourceReleaseEvent) event;
                LOG.info("Container " + relEvent.getContainer() + " sent RELEASE event on a resource request " + req + " not present in cache.");
                return;
            }
            break;
        case LOCALIZATION_FAILED:
            /*
       * If resource localization fails then Localized resource will be
       * removed from local cache.
       */
            removeResource(req);
            break;
        case RECOVERED:
            if (rsrc != null) {
                LOG.warn("Ignoring attempt to recover existing resource " + rsrc);
                return;
            }
            rsrc = recoverResource(req, (ResourceRecoveredEvent) event);
            localrsrc.put(req, rsrc);
            break;
    }
    if (rsrc == null) {
        LOG.warn("Received " + event.getType() + " event for request " + req + " but localized resource is missing");
        return;
    }
    rsrc.handle(event);
    //       download is not associated with a container-specific localizer.
    if (event.getType() == ResourceEventType.RELEASE) {
        if (rsrc.getState() == ResourceState.DOWNLOADING && rsrc.getRefCount() <= 0 && rsrc.getRequest().getVisibility() != LocalResourceVisibility.PUBLIC) {
            removeResource(req);
        }
    }
    if (event.getType() == ResourceEventType.LOCALIZED) {
        if (rsrc.getLocalPath() != null) {
            try {
                stateStore.finishResourceLocalization(user, appId, buildLocalizedResourceProto(rsrc));
            } catch (IOException ioe) {
                LOG.error("Error storing resource state for " + rsrc, ioe);
            }
        } else {
            LOG.warn("Resource " + rsrc + " localized without a location");
        }
    }
}

Example 2 with ResourceRecoveredEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRecoveredEvent in project hadoop by apache.

the class ResourceLocalizationService method recoverTrackerResources.

private void recoverTrackerResources(LocalResourcesTracker tracker, LocalResourceTrackerState state) throws URISyntaxException {
    for (LocalizedResourceProto proto : state.getLocalizedResources()) {
        LocalResource rsrc = new LocalResourcePBImpl(proto.getResource());
        LocalResourceRequest req = new LocalResourceRequest(rsrc);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Recovering localized resource " + req + " at " + proto.getLocalPath());
        }
        tracker.handle(new ResourceRecoveredEvent(req, new Path(proto.getLocalPath()), proto.getSize()));
    }
    for (Map.Entry<LocalResourceProto, Path> entry : state.getInProgressResources().entrySet()) {
        LocalResource rsrc = new LocalResourcePBImpl(entry.getKey());
        LocalResourceRequest req = new LocalResourceRequest(rsrc);
        Path localPath = entry.getValue();
        tracker.handle(new ResourceRecoveredEvent(req, localPath, 0));
        // delete any in-progress localizations, containers will request again
        LOG.info("Deleting in-progress localization for " + req + " at " + localPath);
        tracker.remove(tracker.getLocalizedResource(req), delService);
    }
// TODO: remove untracked directories in local filesystem
}

Example 3 with ResourceRecoveredEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRecoveredEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testRecoveredResource.

@Test
@SuppressWarnings("unchecked")
public void testRecoveredResource() throws Exception {
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(1, 1);
    // This is a random path. NO File creation will take place at this place.
    final Path localDir = new Path("/tmp/localdir");
    Configuration conf = new YarnConfiguration();
    DrainDispatcher dispatcher = null;
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);
    NMStateStoreService stateStore = mock(NMStateStoreService.class);
    try {
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore);
        // Container 1 needs lr1 resource
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION);
        Assert.assertNull(tracker.getLocalizedResource(lr1));
        final long localizedId1 = 52;
        Path hierarchicalPath1 = new Path(localDir, Long.toString(localizedId1));
        Path localizedPath1 = new Path(hierarchicalPath1, "resource.jar");
        tracker.handle(new ResourceRecoveredEvent(lr1, localizedPath1, 120));
        dispatcher.await();
        Assert.assertNotNull(tracker.getLocalizedResource(lr1));
        // verify new paths reflect recovery of previous resources
        LocalResourceRequest lr2 = createLocalResourceRequest(user, 2, 2, LocalResourceVisibility.APPLICATION);
        LocalizerContext lc2 = new LocalizerContext(user, cId1, null);
        ResourceEvent reqEvent2 = new ResourceRequestEvent(lr2, LocalResourceVisibility.APPLICATION, lc2);
        tracker.handle(reqEvent2);
        dispatcher.await();
        Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir, null);
        long localizedId2 = Long.parseLong(hierarchicalPath2.getName());
        Assert.assertEquals(localizedId1 + 1, localizedId2);
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 4 with ResourceRecoveredEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRecoveredEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testRecoveredResourceWithDirCacheMgr.

@Test
@SuppressWarnings("unchecked")
public void testRecoveredResourceWithDirCacheMgr() throws Exception {
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(1, 1);
    // This is a random path. NO File creation will take place at this place.
    final Path localDirRoot = new Path("/tmp/localdir");
    Configuration conf = new YarnConfiguration();
    DrainDispatcher dispatcher = null;
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);
    NMStateStoreService stateStore = mock(NMStateStoreService.class);
    try {
        LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, true, conf, stateStore);
        LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        Assert.assertNull(tracker.getLocalizedResource(lr1));
        final long localizedId1 = 52;
        Path hierarchicalPath1 = new Path(localDirRoot + "/4/2", Long.toString(localizedId1));
        Path localizedPath1 = new Path(hierarchicalPath1, "resource.jar");
        tracker.handle(new ResourceRecoveredEvent(lr1, localizedPath1, 120));
        dispatcher.await();
        Assert.assertNotNull(tracker.getLocalizedResource(lr1));
        LocalCacheDirectoryManager dirMgrRoot = tracker.getDirectoryManager(localDirRoot);
        Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount());
        Assert.assertEquals(1, dirMgrRoot.getDirectory("4/2").getCount());
        LocalResourceRequest lr2 = createLocalResourceRequest(user, 2, 2, LocalResourceVisibility.PUBLIC);
        Assert.assertNull(tracker.getLocalizedResource(lr2));
        final long localizedId2 = localizedId1 + 1;
        Path hierarchicalPath2 = new Path(localDirRoot + "/4/2", Long.toString(localizedId2));
        Path localizedPath2 = new Path(hierarchicalPath2, "resource.jar");
        tracker.handle(new ResourceRecoveredEvent(lr2, localizedPath2, 120));
        dispatcher.await();
        Assert.assertNotNull(tracker.getLocalizedResource(lr2));
        Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount());
        Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount());
        LocalResourceRequest lr3 = createLocalResourceRequest(user, 3, 3, LocalResourceVisibility.PUBLIC);
        Assert.assertNull(tracker.getLocalizedResource(lr3));
        final long localizedId3 = 128;
        Path hierarchicalPath3 = new Path(localDirRoot + "/4/3", Long.toString(localizedId3));
        Path localizedPath3 = new Path(hierarchicalPath3, "resource.jar");
        tracker.handle(new ResourceRecoveredEvent(lr3, localizedPath3, 120));
        dispatcher.await();
        Assert.assertNotNull(tracker.getLocalizedResource(lr3));
        Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount());
        Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount());
        Assert.assertEquals(1, dirMgrRoot.getDirectory("4/3").getCount());
        LocalResourceRequest lr4 = createLocalResourceRequest(user, 4, 4, LocalResourceVisibility.PUBLIC);
        Assert.assertNull(tracker.getLocalizedResource(lr4));
        final long localizedId4 = 256;
        Path hierarchicalPath4 = new Path(localDirRoot + "/4", Long.toString(localizedId4));
        Path localizedPath4 = new Path(hierarchicalPath4, "resource.jar");
        tracker.handle(new ResourceRecoveredEvent(lr4, localizedPath4, 120));
        dispatcher.await();
        Assert.assertNotNull(tracker.getLocalizedResource(lr4));
        Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount());
        Assert.assertEquals(1, dirMgrRoot.getDirectory("4").getCount());
        Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount());
        Assert.assertEquals(1, dirMgrRoot.getDirectory("4/3").getCount());
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}