Examples with DeletionService org.apache.hadoop.yarn.server.nodemanager.DeletionService used on opensource projects

Example 31 with DeletionService

use of org.apache.hadoop.yarn.server.nodemanager.DeletionService in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testHierarchicalLocalCacheDirectories.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void testHierarchicalLocalCacheDirectories() {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        // setting per directory file limit to 1.
        conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "37");
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        DeletionService mockDelService = mock(DeletionService.class);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, true, conf, new NMNullStateStoreService(), null);
        // This is a random path. NO File creation will take place at this place.
        Path localDir = new Path("/tmp");
        // Container 1 needs lr1 resource
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        // Container 1 requests lr1 to be localized
        ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.PUBLIC, lc1);
        tracker.handle(reqEvent1);
        // Simulate the process of localization of lr1
        // NOTE: Localization path from tracker has resource ID at end
        Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir, null).getParent();
        // Simulate lr1 getting localized
        ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(lr1, new Path(hierarchicalPath1.toUri().toString() + Path.SEPARATOR + "file1"), 120);
        tracker.handle(rle1);
        // Localization successful.
        LocalResourceRequest lr2 = createLocalResourceRequest(user, 3, 3, LocalResourceVisibility.PUBLIC);
        // Container 1 requests lr2 to be localized.
        ResourceEvent reqEvent2 = new ResourceRequestEvent(lr2, LocalResourceVisibility.PUBLIC, lc1);
        tracker.handle(reqEvent2);
        Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir, null).getParent();
        // localization failed.
        ResourceFailedLocalizationEvent rfe2 = new ResourceFailedLocalizationEvent(lr2, new Exception("Test").toString());
        tracker.handle(rfe2);
        /*
       * The path returned for two localization should be different because we
       * are limiting one file per sub-directory.
       */
        Assert.assertNotSame(hierarchicalPath1, hierarchicalPath2);
        LocalResourceRequest lr3 = createLocalResourceRequest(user, 2, 2, LocalResourceVisibility.PUBLIC);
        ResourceEvent reqEvent3 = new ResourceRequestEvent(lr3, LocalResourceVisibility.PUBLIC, lc1);
        tracker.handle(reqEvent3);
        Path hierarchicalPath3 = tracker.getPathForLocalization(lr3, localDir, null).getParent();
        // localization successful
        ResourceLocalizedEvent rle3 = new ResourceLocalizedEvent(lr3, new Path(hierarchicalPath3.toUri().toString() + Path.SEPARATOR + "file3"), 120);
        tracker.handle(rle3);
        // Verifying that path created is inside the subdirectory
        Assert.assertEquals(hierarchicalPath3.toUri().toString(), hierarchicalPath1.toUri().toString() + Path.SEPARATOR + "0");
        // Container 1 releases resource lr1
        ResourceEvent relEvent1 = new ResourceReleaseEvent(lr1, cId1);
        tracker.handle(relEvent1);
        // Validate the file counts now
        int resources = 0;
        Iterator<LocalizedResource> iter = tracker.iterator();
        while (iter.hasNext()) {
            iter.next();
            resources++;
        }
        // There should be only two resources lr1 and lr3 now.
        Assert.assertEquals(2, resources);
        // Now simulate cache cleanup - removes unused resources.
        iter = tracker.iterator();
        while (iter.hasNext()) {
            LocalizedResource rsrc = iter.next();
            if (rsrc.getRefCount() == 0) {
                Assert.assertTrue(tracker.remove(rsrc, mockDelService));
                resources--;
            }
        }
        // lr1 is not used by anyone and will be removed, only lr3 will hang
        // around
        Assert.assertEquals(1, resources);
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 32 with DeletionService

use of org.apache.hadoop.yarn.server.nodemanager.DeletionService in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testGetPathForLocalization.

@Test
@SuppressWarnings("unchecked")
public void testGetPathForLocalization() throws Exception {
    FileContext lfs = FileContext.getLocalFSFileContext();
    Path base_path = new Path("target", TestLocalResourcesTrackerImpl.class.getSimpleName());
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(1, 1);
    Configuration conf = new YarnConfiguration();
    DrainDispatcher dispatcher = null;
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);
    NMStateStoreService stateStore = mock(NMStateStoreService.class);
    DeletionService delService = mock(DeletionService.class);
    try {
        LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        localrsrc.put(req1, lr1);
        LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, localrsrc, true, conf, stateStore, null);
        Path conflictPath = new Path(base_path, "10");
        Path qualifiedConflictPath = lfs.makeQualified(conflictPath);
        lfs.mkdir(qualifiedConflictPath, null, true);
        Path rPath = tracker.getPathForLocalization(req1, base_path, delService);
        Assert.assertFalse(lfs.util().exists(rPath));
        verify(delService, times(1)).delete(eq(user), eq(conflictPath));
    } finally {
        lfs.delete(base_path, true);
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 33 with DeletionService

use of org.apache.hadoop.yarn.server.nodemanager.DeletionService in project hadoop by apache.

the class TestLogAggregationService method testLogAggregatorCleanup.

@Test
public void testLogAggregatorCleanup() throws Exception {
    DeletionService delSrvc = mock(DeletionService.class);
    // get the AppLogAggregationImpl thread to crash
    LocalDirsHandlerService mockedDirSvc = mock(LocalDirsHandlerService.class);
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, delSrvc, mockedDirSvc);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
    logAggregationService.handle(new LogHandlerAppStartedEvent(application1, this.user, null, this.acls));
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application1));
    dispatcher.await();
    int timeToWait = 20 * 1000;
    while (timeToWait > 0 && logAggregationService.getNumAggregators() > 0) {
        Thread.sleep(100);
        timeToWait -= 100;
    }
    Assert.assertEquals("Log aggregator failed to cleanup!", 0, logAggregationService.getNumAggregators());
    logAggregationService.stop();
    logAggregationService.close();
}

Example 34 with DeletionService

use of org.apache.hadoop.yarn.server.nodemanager.DeletionService in project hadoop by apache.

the class TestLogAggregationService method testFailedDirsLocalFileDeletionAfterUpload.

/*
   * Test to make sure we handle cases where the directories we get back from
   * the LocalDirsHandler may have issues including the log dir not being
   * present as well as other issues. The test uses helper functions from
   * TestNonAggregatingLogHandler.
   */
@Test
public void testFailedDirsLocalFileDeletionAfterUpload() throws Exception {
    // setup conf and services
    DeletionService mockDelService = mock(DeletionService.class);
    File[] localLogDirs = TestNonAggregatingLogHandler.getLocalLogDirFiles(this.getClass().getName(), 7);
    final List<String> localLogDirPaths = new ArrayList<String>(localLogDirs.length);
    for (int i = 0; i < localLogDirs.length; i++) {
        localLogDirPaths.add(localLogDirs[i].getAbsolutePath());
    }
    String localLogDirsString = StringUtils.join(localLogDirPaths, ",");
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDirsString);
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    this.conf.setLong(YarnConfiguration.NM_DISK_HEALTH_CHECK_INTERVAL_MS, 500);
    ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(application1, 1);
    this.dirsHandler = new LocalDirsHandlerService();
    LocalDirsHandlerService mockDirsHandler = mock(LocalDirsHandlerService.class);
    LogAggregationService logAggregationService = spy(new LogAggregationService(dispatcher, this.context, mockDelService, mockDirsHandler));
    AbstractFileSystem spylfs = spy(FileContext.getLocalFSFileContext().getDefaultFileSystem());
    FileContext lfs = FileContext.getFileContext(spylfs, conf);
    doReturn(lfs).when(logAggregationService).getLocalFileContext(isA(Configuration.class));
    logAggregationService.init(this.conf);
    logAggregationService.start();
    TestNonAggregatingLogHandler.runMockedFailedDirs(logAggregationService, application1, user, mockDelService, mockDirsHandler, conf, spylfs, lfs, localLogDirs);
    logAggregationService.stop();
    assertEquals(0, logAggregationService.getNumAggregators());
    verify(logAggregationService).closeFileSystems(any(UserGroupInformation.class));
    ApplicationEvent[] expectedEvents = new ApplicationEvent[] { new ApplicationEvent(appAttemptId.getApplicationId(), ApplicationEventType.APPLICATION_LOG_HANDLING_INITED), new ApplicationEvent(appAttemptId.getApplicationId(), ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED) };
    checkEvents(appEventHandler, expectedEvents, true, "getType", "getApplicationID");
}