Examples with NMNullStateStoreService org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService used on opensource projects

Example 6 with NMNullStateStoreService

use of org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testResourcePresentInGoodDir.

@SuppressWarnings("unchecked")
@Test
public void testResourcePresentInGoodDir() throws IOException {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        LocalResourceRequest req2 = createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
        LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
        LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        localrsrc.put(req1, lr1);
        localrsrc.put(req2, lr2);
        LocalDirsHandlerService dirsHandler = mock(LocalDirsHandlerService.class);
        List<String> goodDirs = new ArrayList<String>();
        // /tmp/somedir2 is bad
        goodDirs.add("/tmp/somedir1/");
        goodDirs.add("/tmp/somedir2");
        Mockito.when(dirsHandler.getLocalDirs()).thenReturn(goodDirs);
        Mockito.when(dirsHandler.getLocalDirsForRead()).thenReturn(goodDirs);
        LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, true, conf, new NMNullStateStoreService(), dirsHandler);
        ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
        ResourceEvent req21Event = new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);
        // Localize R1 for C1
        tracker.handle(req11Event);
        // Localize R2 for C1
        tracker.handle(req21Event);
        dispatcher.await();
        // Localize resource1
        Path p1 = tracker.getPathForLocalization(req1, new Path("/tmp/somedir1"), null);
        Path p2 = tracker.getPathForLocalization(req2, new Path("/tmp/somedir2"), null);
        ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(req1, p1, 1);
        tracker.handle(rle1);
        ResourceLocalizedEvent rle2 = new ResourceLocalizedEvent(req2, p2, 1);
        tracker.handle(rle2);
        dispatcher.await();
        // Remove somedir2 from gooddirs
        Assert.assertTrue(tracker.checkLocalResource(lr2));
        goodDirs.remove(1);
        Assert.assertFalse(tracker.checkLocalResource(lr2));
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 7 with NMNullStateStoreService

use of org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService in project hadoop by apache.

the class TestContainerManagerRecovery method testContainerCleanupOnShutdown.

@Test
public void testContainerCleanupOnShutdown() throws Exception {
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId cid = ContainerId.newContainerId(attemptId, 1);
    Map<String, LocalResource> localResources = Collections.emptyMap();
    Map<String, String> containerEnv = Collections.emptyMap();
    List<String> containerCmds = Collections.emptyList();
    Map<String, ByteBuffer> serviceData = Collections.emptyMap();
    Credentials containerCreds = new Credentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    containerCreds.writeTokenStorageToStream(dob);
    ByteBuffer containerTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    Map<ApplicationAccessType, String> acls = Collections.emptyMap();
    ContainerLaunchContext clc = ContainerLaunchContext.newInstance(localResources, containerEnv, containerCmds, serviceData, containerTokens, acls);
    // create the logAggregationContext
    LogAggregationContext logAggregationContext = LogAggregationContext.newInstance("includePattern", "excludePattern");
    // verify containers are stopped on shutdown without recovery
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, false);
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, false);
    Context context = createContext(conf, new NMNullStateStoreService());
    ContainerManagerImpl cm = spy(createContainerManager(context));
    cm.init(conf);
    cm.start();
    StartContainersResponse startResponse = startContainer(context, cm, cid, clc, logAggregationContext);
    assertEquals(1, startResponse.getSuccessfullyStartedContainers().size());
    cm.stop();
    verify(cm).handle(isA(CMgrCompletedAppsEvent.class));
    // verify containers are stopped on shutdown with unsupervised recovery
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, false);
    NMMemoryStateStoreService memStore = new NMMemoryStateStoreService();
    memStore.init(conf);
    memStore.start();
    context = createContext(conf, memStore);
    cm = spy(createContainerManager(context));
    cm.init(conf);
    cm.start();
    startResponse = startContainer(context, cm, cid, clc, logAggregationContext);
    assertEquals(1, startResponse.getSuccessfullyStartedContainers().size());
    cm.stop();
    memStore.close();
    verify(cm).handle(isA(CMgrCompletedAppsEvent.class));
    // verify containers are not stopped on shutdown with supervised recovery
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, true);
    memStore = new NMMemoryStateStoreService();
    memStore.init(conf);
    memStore.start();
    context = createContext(conf, memStore);
    cm = spy(createContainerManager(context));
    cm.init(conf);
    cm.start();
    startResponse = startContainer(context, cm, cid, clc, logAggregationContext);
    assertEquals(1, startResponse.getSuccessfullyStartedContainers().size());
    cm.stop();
    memStore.close();
    verify(cm, never()).handle(isA(CMgrCompletedAppsEvent.class));
}

Example 8 with NMNullStateStoreService

use of org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService in project hadoop by apache.

the class NodeManager method initAndStartRecoveryStore.

private void initAndStartRecoveryStore(Configuration conf) throws IOException {
    boolean recoveryEnabled = conf.getBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, YarnConfiguration.DEFAULT_NM_RECOVERY_ENABLED);
    if (recoveryEnabled) {
        FileSystem recoveryFs = FileSystem.getLocal(conf);
        String recoveryDirName = conf.get(YarnConfiguration.NM_RECOVERY_DIR);
        if (recoveryDirName == null) {
            throw new IllegalArgumentException("Recovery is enabled but " + YarnConfiguration.NM_RECOVERY_DIR + " is not set.");
        }
        Path recoveryRoot = new Path(recoveryDirName);
        recoveryFs.mkdirs(recoveryRoot, new FsPermission((short) 0700));
        nmStore = new NMLeveldbStateStoreService();
    } else {
        nmStore = new NMNullStateStoreService();
    }
    nmStore.init(conf);
    nmStore.start();
}

Example 9 with NMNullStateStoreService

use of org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService in project hadoop by apache.

the class TestResourceLocalizationService method setup.

@Before
public void setup() throws IOException {
    conf = new Configuration();
    spylfs = spy(FileContext.getLocalFSFileContext().getDefaultFileSystem());
    lfs = FileContext.getFileContext(spylfs, conf);
    String logDir = lfs.makeQualified(new Path(basedir, "logdir ")).toString();
    conf.set(YarnConfiguration.NM_LOG_DIRS, logDir);
    nmContext = new NMContext(new NMContainerTokenSecretManager(conf), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), new NMNullStateStoreService(), false, conf);
}

Example 10 with NMNullStateStoreService

use of org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService in project hadoop by apache.

the class TestNonAggregatingLogHandler method testStop.

@Test
public void testStop() throws Exception {
    NonAggregatingLogHandler aggregatingLogHandler = new NonAggregatingLogHandler(null, null, null, new NMNullStateStoreService());
    // It should not throw NullPointerException
    aggregatingLogHandler.stop();
    NonAggregatingLogHandlerWithMockExecutor logHandler = new NonAggregatingLogHandlerWithMockExecutor(null, null, null);
    logHandler.init(new Configuration());
    logHandler.stop();
    verify(logHandler.mockSched).shutdown();
    verify(logHandler.mockSched).awaitTermination(eq(10l), eq(TimeUnit.SECONDS));
    verify(logHandler.mockSched).shutdownNow();
    logHandler.close();
    aggregatingLogHandler.close();
}