Examples with ResourceRequestEvent org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent used on opensource projects

Example 6 with ResourceRequestEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent in project hadoop by apache.

the class ResourceLocalizationService method handleInitContainerResources.

/**
   * For each of the requested resources for a container, determines the
   * appropriate {@link LocalResourcesTracker} and forwards a 
   * {@link LocalResourceRequest} to that tracker.
   */
private void handleInitContainerResources(ContainerLocalizationRequestEvent rsrcReqs) {
    Container c = rsrcReqs.getContainer();
    EnumSet<ContainerState> set = EnumSet.of(ContainerState.LOCALIZING, ContainerState.RUNNING, ContainerState.REINITIALIZING);
    if (!set.contains(c.getContainerState())) {
        LOG.warn(c.getContainerId() + " is at " + c.getContainerState() + " state, do not localize resources.");
        return;
    }
    // create a loading cache for the file statuses
    LoadingCache<Path, Future<FileStatus>> statCache = CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig()));
    LocalizerContext ctxt = new LocalizerContext(c.getUser(), c.getContainerId(), c.getCredentials(), statCache);
    Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = rsrcReqs.getRequestedResources();
    for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e : rsrcs.entrySet()) {
        LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(), c.getContainerId().getApplicationAttemptId().getApplicationId());
        for (LocalResourceRequest req : e.getValue()) {
            tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
            if (LOG.isDebugEnabled()) {
                LOG.debug("Localizing " + req.getPath() + " for container " + c.getContainerId());
            }
        }
    }
}

Example 7 with ResourceRequestEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method test.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void test() {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        DeletionService mockDelService = mock(DeletionService.class);
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2);
        LocalizerContext lc2 = new LocalizerContext(user, cId2, null);
        LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        LocalResourceRequest req2 = createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
        LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
        LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        localrsrc.put(req1, lr1);
        localrsrc.put(req2, lr2);
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, false, conf, new NMNullStateStoreService(), null);
        ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
        ResourceEvent req12Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc2);
        ResourceEvent req21Event = new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);
        ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);
        ResourceEvent rel12Event = new ResourceReleaseEvent(req1, cId2);
        ResourceEvent rel21Event = new ResourceReleaseEvent(req2, cId1);
        // Localize R1 for C1
        tracker.handle(req11Event);
        // Localize R1 for C2
        tracker.handle(req12Event);
        // Localize R2 for C1
        tracker.handle(req21Event);
        dispatcher.await();
        verify(localizerEventHandler, times(3)).handle(any(LocalizerResourceRequestEvent.class));
        // Verify refCount for R1 is 2
        Assert.assertEquals(2, lr1.getRefCount());
        // Verify refCount for R2 is 1
        Assert.assertEquals(1, lr2.getRefCount());
        // Release R2 for C1
        tracker.handle(rel21Event);
        dispatcher.await();
        verifyTrackedResourceCount(tracker, 2);
        // Verify resource with non zero ref count is not removed.
        Assert.assertEquals(2, lr1.getRefCount());
        Assert.assertFalse(tracker.remove(lr1, mockDelService));
        verifyTrackedResourceCount(tracker, 2);
        // Localize resource1
        ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path("file:///tmp/r1"), 1);
        lr1.handle(rle);
        Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
        // Release resource1
        tracker.handle(rel11Event);
        tracker.handle(rel12Event);
        Assert.assertEquals(0, lr1.getRefCount());
        // Verify resources in state LOCALIZED with ref-count=0 is removed.
        Assert.assertTrue(tracker.remove(lr1, mockDelService));
        verifyTrackedResourceCount(tracker, 1);
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 8 with ResourceRequestEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testConsistency.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void testConsistency() {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        localrsrc.put(req1, lr1);
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, false, conf, new NMNullStateStoreService(), null);
        ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
        ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);
        // Localize R1 for C1
        tracker.handle(req11Event);
        dispatcher.await();
        // Verify refCount for R1 is 1
        Assert.assertEquals(1, lr1.getRefCount());
        dispatcher.await();
        verifyTrackedResourceCount(tracker, 1);
        // Localize resource1
        ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path("file:///tmp/r1"), 1);
        lr1.handle(rle);
        Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
        Assert.assertTrue(createdummylocalizefile(new Path("file:///tmp/r1")));
        LocalizedResource rsrcbefore = tracker.iterator().next();
        File resFile = new File(lr1.getLocalPath().toUri().getRawPath().toString());
        Assert.assertTrue(resFile.exists());
        Assert.assertTrue(resFile.delete());
        // Localize R1 for C1
        tracker.handle(req11Event);
        dispatcher.await();
        lr1.handle(rle);
        Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
        LocalizedResource rsrcafter = tracker.iterator().next();
        if (rsrcbefore == rsrcafter) {
            Assert.fail("Localized resource should not be equal");
        }
        // Release resource1
        tracker.handle(rel11Event);
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 9 with ResourceRequestEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testRecoveredResource.

@Test
@SuppressWarnings("unchecked")
public void testRecoveredResource() throws Exception {
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(1, 1);
    // This is a random path. NO File creation will take place at this place.
    final Path localDir = new Path("/tmp/localdir");
    Configuration conf = new YarnConfiguration();
    DrainDispatcher dispatcher = null;
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);
    NMStateStoreService stateStore = mock(NMStateStoreService.class);
    try {
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore);
        // Container 1 needs lr1 resource
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION);
        Assert.assertNull(tracker.getLocalizedResource(lr1));
        final long localizedId1 = 52;
        Path hierarchicalPath1 = new Path(localDir, Long.toString(localizedId1));
        Path localizedPath1 = new Path(hierarchicalPath1, "resource.jar");
        tracker.handle(new ResourceRecoveredEvent(lr1, localizedPath1, 120));
        dispatcher.await();
        Assert.assertNotNull(tracker.getLocalizedResource(lr1));
        // verify new paths reflect recovery of previous resources
        LocalResourceRequest lr2 = createLocalResourceRequest(user, 2, 2, LocalResourceVisibility.APPLICATION);
        LocalizerContext lc2 = new LocalizerContext(user, cId1, null);
        ResourceEvent reqEvent2 = new ResourceRequestEvent(lr2, LocalResourceVisibility.APPLICATION, lc2);
        tracker.handle(reqEvent2);
        dispatcher.await();
        Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir, null);
        long localizedId2 = Long.parseLong(hierarchicalPath2.getName());
        Assert.assertEquals(localizedId1 + 1, localizedId2);
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 10 with ResourceRequestEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testHierarchicalLocalCacheDirectories.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void testHierarchicalLocalCacheDirectories() {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        // setting per directory file limit to 1.
        conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "37");
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        DeletionService mockDelService = mock(DeletionService.class);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, true, conf, new NMNullStateStoreService(), null);
        // This is a random path. NO File creation will take place at this place.
        Path localDir = new Path("/tmp");
        // Container 1 needs lr1 resource
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        // Container 1 requests lr1 to be localized
        ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.PUBLIC, lc1);
        tracker.handle(reqEvent1);
        // Simulate the process of localization of lr1
        // NOTE: Localization path from tracker has resource ID at end
        Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir, null).getParent();
        // Simulate lr1 getting localized
        ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(lr1, new Path(hierarchicalPath1.toUri().toString() + Path.SEPARATOR + "file1"), 120);
        tracker.handle(rle1);
        // Localization successful.
        LocalResourceRequest lr2 = createLocalResourceRequest(user, 3, 3, LocalResourceVisibility.PUBLIC);
        // Container 1 requests lr2 to be localized.
        ResourceEvent reqEvent2 = new ResourceRequestEvent(lr2, LocalResourceVisibility.PUBLIC, lc1);
        tracker.handle(reqEvent2);
        Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir, null).getParent();
        // localization failed.
        ResourceFailedLocalizationEvent rfe2 = new ResourceFailedLocalizationEvent(lr2, new Exception("Test").toString());
        tracker.handle(rfe2);
        /*
       * The path returned for two localization should be different because we
       * are limiting one file per sub-directory.
       */
        Assert.assertNotSame(hierarchicalPath1, hierarchicalPath2);
        LocalResourceRequest lr3 = createLocalResourceRequest(user, 2, 2, LocalResourceVisibility.PUBLIC);
        ResourceEvent reqEvent3 = new ResourceRequestEvent(lr3, LocalResourceVisibility.PUBLIC, lc1);
        tracker.handle(reqEvent3);
        Path hierarchicalPath3 = tracker.getPathForLocalization(lr3, localDir, null).getParent();
        // localization successful
        ResourceLocalizedEvent rle3 = new ResourceLocalizedEvent(lr3, new Path(hierarchicalPath3.toUri().toString() + Path.SEPARATOR + "file3"), 120);
        tracker.handle(rle3);
        // Verifying that path created is inside the subdirectory
        Assert.assertEquals(hierarchicalPath3.toUri().toString(), hierarchicalPath1.toUri().toString() + Path.SEPARATOR + "0");
        // Container 1 releases resource lr1
        ResourceEvent relEvent1 = new ResourceReleaseEvent(lr1, cId1);
        tracker.handle(relEvent1);
        // Validate the file counts now
        int resources = 0;
        Iterator<LocalizedResource> iter = tracker.iterator();
        while (iter.hasNext()) {
            iter.next();
            resources++;
        }
        // There should be only two resources lr1 and lr3 now.
        Assert.assertEquals(2, resources);
        // Now simulate cache cleanup - removes unused resources.
        iter = tracker.iterator();
        while (iter.hasNext()) {
            LocalizedResource rsrc = iter.next();
            if (rsrc.getRefCount() == 0) {
                Assert.assertTrue(tracker.remove(rsrc, mockDelService));
                resources--;
            }
        }
        // lr1 is not used by anyone and will be removed, only lr3 will hang
        // around
        Assert.assertEquals(1, resources);
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}