Search in sources :

Example 11 with NMStateStoreService

use of org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testRecoveredResource.

@Test
@SuppressWarnings("unchecked")
public void testRecoveredResource() throws Exception {
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(1, 1);
    // This is a random path. NO File creation will take place at this place.
    final Path localDir = new Path("/tmp/localdir");
    Configuration conf = new YarnConfiguration();
    DrainDispatcher dispatcher = null;
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);
    NMStateStoreService stateStore = mock(NMStateStoreService.class);
    try {
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore);
        // Container 1 needs lr1 resource
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION);
        Assert.assertNull(tracker.getLocalizedResource(lr1));
        final long localizedId1 = 52;
        Path hierarchicalPath1 = new Path(localDir, Long.toString(localizedId1));
        Path localizedPath1 = new Path(hierarchicalPath1, "resource.jar");
        tracker.handle(new ResourceRecoveredEvent(lr1, localizedPath1, 120));
        dispatcher.await();
        Assert.assertNotNull(tracker.getLocalizedResource(lr1));
        // verify new paths reflect recovery of previous resources
        LocalResourceRequest lr2 = createLocalResourceRequest(user, 2, 2, LocalResourceVisibility.APPLICATION);
        LocalizerContext lc2 = new LocalizerContext(user, cId1, null);
        ResourceEvent reqEvent2 = new ResourceRequestEvent(lr2, LocalResourceVisibility.APPLICATION, lc2);
        tracker.handle(reqEvent2);
        dispatcher.await();
        Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir, null);
        long localizedId2 = Long.parseLong(hierarchicalPath2.getName());
        Assert.assertEquals(localizedId1 + 1, localizedId2);
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}
Also used : Path(org.apache.hadoop.fs.Path) DrainDispatcher(org.apache.hadoop.yarn.event.DrainDispatcher) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ResourceRecoveredEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRecoveredEvent) NMStateStoreService(org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService) LocalizerEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) LocalizerResourceRequestEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent) ResourceRequestEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent) ResourceEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) Test(org.junit.Test)

Example 12 with NMStateStoreService

use of org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testRecoveredResourceWithDirCacheMgr.

@Test
@SuppressWarnings("unchecked")
public void testRecoveredResourceWithDirCacheMgr() throws Exception {
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(1, 1);
    // This is a random path. NO File creation will take place at this place.
    final Path localDirRoot = new Path("/tmp/localdir");
    Configuration conf = new YarnConfiguration();
    DrainDispatcher dispatcher = null;
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);
    NMStateStoreService stateStore = mock(NMStateStoreService.class);
    try {
        LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, true, conf, stateStore);
        LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        Assert.assertNull(tracker.getLocalizedResource(lr1));
        final long localizedId1 = 52;
        Path hierarchicalPath1 = new Path(localDirRoot + "/4/2", Long.toString(localizedId1));
        Path localizedPath1 = new Path(hierarchicalPath1, "resource.jar");
        tracker.handle(new ResourceRecoveredEvent(lr1, localizedPath1, 120));
        dispatcher.await();
        Assert.assertNotNull(tracker.getLocalizedResource(lr1));
        LocalCacheDirectoryManager dirMgrRoot = tracker.getDirectoryManager(localDirRoot);
        Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount());
        Assert.assertEquals(1, dirMgrRoot.getDirectory("4/2").getCount());
        LocalResourceRequest lr2 = createLocalResourceRequest(user, 2, 2, LocalResourceVisibility.PUBLIC);
        Assert.assertNull(tracker.getLocalizedResource(lr2));
        final long localizedId2 = localizedId1 + 1;
        Path hierarchicalPath2 = new Path(localDirRoot + "/4/2", Long.toString(localizedId2));
        Path localizedPath2 = new Path(hierarchicalPath2, "resource.jar");
        tracker.handle(new ResourceRecoveredEvent(lr2, localizedPath2, 120));
        dispatcher.await();
        Assert.assertNotNull(tracker.getLocalizedResource(lr2));
        Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount());
        Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount());
        LocalResourceRequest lr3 = createLocalResourceRequest(user, 3, 3, LocalResourceVisibility.PUBLIC);
        Assert.assertNull(tracker.getLocalizedResource(lr3));
        final long localizedId3 = 128;
        Path hierarchicalPath3 = new Path(localDirRoot + "/4/3", Long.toString(localizedId3));
        Path localizedPath3 = new Path(hierarchicalPath3, "resource.jar");
        tracker.handle(new ResourceRecoveredEvent(lr3, localizedPath3, 120));
        dispatcher.await();
        Assert.assertNotNull(tracker.getLocalizedResource(lr3));
        Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount());
        Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount());
        Assert.assertEquals(1, dirMgrRoot.getDirectory("4/3").getCount());
        LocalResourceRequest lr4 = createLocalResourceRequest(user, 4, 4, LocalResourceVisibility.PUBLIC);
        Assert.assertNull(tracker.getLocalizedResource(lr4));
        final long localizedId4 = 256;
        Path hierarchicalPath4 = new Path(localDirRoot + "/4", Long.toString(localizedId4));
        Path localizedPath4 = new Path(hierarchicalPath4, "resource.jar");
        tracker.handle(new ResourceRecoveredEvent(lr4, localizedPath4, 120));
        dispatcher.await();
        Assert.assertNotNull(tracker.getLocalizedResource(lr4));
        Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount());
        Assert.assertEquals(1, dirMgrRoot.getDirectory("4").getCount());
        Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount());
        Assert.assertEquals(1, dirMgrRoot.getDirectory("4/3").getCount());
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}
Also used : Path(org.apache.hadoop.fs.Path) DrainDispatcher(org.apache.hadoop.yarn.event.DrainDispatcher) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ResourceRecoveredEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRecoveredEvent) NMStateStoreService(org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService) LocalizerEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) Test(org.junit.Test)

Example 13 with NMStateStoreService

use of org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testGetPathForLocalization.

@Test
@SuppressWarnings("unchecked")
public void testGetPathForLocalization() throws Exception {
    FileContext lfs = FileContext.getLocalFSFileContext();
    Path base_path = new Path("target", TestLocalResourcesTrackerImpl.class.getSimpleName());
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(1, 1);
    Configuration conf = new YarnConfiguration();
    DrainDispatcher dispatcher = null;
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);
    NMStateStoreService stateStore = mock(NMStateStoreService.class);
    DeletionService delService = mock(DeletionService.class);
    try {
        LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        localrsrc.put(req1, lr1);
        LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, localrsrc, true, conf, stateStore, null);
        Path conflictPath = new Path(base_path, "10");
        Path qualifiedConflictPath = lfs.makeQualified(conflictPath);
        lfs.mkdir(qualifiedConflictPath, null, true);
        Path rPath = tracker.getPathForLocalization(req1, base_path, delService);
        Assert.assertFalse(lfs.util().exists(rPath));
        verify(delService, times(1)).delete(eq(user), eq(conflictPath));
    } finally {
        lfs.delete(base_path, true);
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}
Also used : Path(org.apache.hadoop.fs.Path) DrainDispatcher(org.apache.hadoop.yarn.event.DrainDispatcher) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) DeletionService(org.apache.hadoop.yarn.server.nodemanager.DeletionService) NMStateStoreService(org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService) LocalizerEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) FileContext(org.apache.hadoop.fs.FileContext) Test(org.junit.Test)

Example 14 with NMStateStoreService

use of org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService in project hadoop by apache.

the class TestNMWebServer method testNMWebApp.

@Test
public void testNMWebApp() throws IOException, YarnException {
    Configuration conf = new Configuration();
    Context nmContext = new NodeManager.NMContext(null, null, null, null, null, false, conf);
    ResourceView resourceView = new ResourceView() {

        @Override
        public long getVmemAllocatedForContainers() {
            return 0;
        }

        @Override
        public long getPmemAllocatedForContainers() {
            return 0;
        }

        @Override
        public long getVCoresAllocatedForContainers() {
            return 0;
        }

        @Override
        public boolean isVmemCheckEnabled() {
            return true;
        }

        @Override
        public boolean isPmemCheckEnabled() {
            return true;
        }
    };
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
    NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);
    healthChecker.init(conf);
    LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
    WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager(conf), dirsHandler);
    server.init(conf);
    server.start();
    // Add an application and the corresponding containers
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf);
    Dispatcher dispatcher = new AsyncDispatcher();
    String user = "nobody";
    long clusterTimeStamp = 1234;
    ApplicationId appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
    Application app = mock(Application.class);
    when(app.getUser()).thenReturn(user);
    when(app.getAppId()).thenReturn(appId);
    nmContext.getApplications().put(appId, app);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0);
    ContainerId container2 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 1);
    NodeManagerMetrics metrics = mock(NodeManagerMetrics.class);
    NMStateStoreService stateStore = new NMNullStateStoreService();
    for (ContainerId containerId : new ContainerId[] { container1, container2 }) {
        // TODO: Use builder utils
        ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
        long currentTime = System.currentTimeMillis();
        Token containerToken = BuilderUtils.newContainerToken(containerId, 0, "127.0.0.1", 1234, user, BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123, "password".getBytes(), currentTime);
        Context context = mock(Context.class);
        Container container = new ContainerImpl(conf, dispatcher, launchContext, null, metrics, BuilderUtils.newContainerTokenIdentifier(containerToken), context) {

            @Override
            public ContainerState getContainerState() {
                return ContainerState.RUNNING;
            }

            ;
        };
        nmContext.getContainers().put(containerId, container);
        //TODO: Gross hack. Fix in code.
        ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
        nmContext.getApplications().get(applicationId).getContainers().put(containerId, container);
        writeContainerLogs(nmContext, containerId, dirsHandler);
    }
// TODO: Pull logs and test contents.
//    Thread.sleep(1000000);
}
Also used : ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) Context(org.apache.hadoop.yarn.server.nodemanager.Context) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) Configuration(org.apache.hadoop.conf.Configuration) NodeHealthCheckerService(org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService) Token(org.apache.hadoop.yarn.api.records.Token) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) LocalDirsHandlerService(org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService) Dispatcher(org.apache.hadoop.yarn.event.Dispatcher) AsyncDispatcher(org.apache.hadoop.yarn.event.AsyncDispatcher) NMNullStateStoreService(org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService) NMStateStoreService(org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService) ResourceView(org.apache.hadoop.yarn.server.nodemanager.ResourceView) ApplicationACLsManager(org.apache.hadoop.yarn.server.security.ApplicationACLsManager) Container(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container) RecordFactory(org.apache.hadoop.yarn.factories.RecordFactory) AsyncDispatcher(org.apache.hadoop.yarn.event.AsyncDispatcher) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) ContainerImpl(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl) NodeManagerMetrics(org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) Application(org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application) Test(org.junit.Test)

Aggregations

NMStateStoreService (org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService)14 Test (org.junit.Test)13 ApplicationId (org.apache.hadoop.yarn.api.records.ApplicationId)10 Configuration (org.apache.hadoop.conf.Configuration)9 YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)9 Path (org.apache.hadoop.fs.Path)8 ContainerId (org.apache.hadoop.yarn.api.records.ContainerId)7 DrainDispatcher (org.apache.hadoop.yarn.event.DrainDispatcher)6 FileContext (org.apache.hadoop.fs.FileContext)5 ContainerLaunchContext (org.apache.hadoop.yarn.api.records.ContainerLaunchContext)5 NMContext (org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext)5 Application (org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application)5 LocalizerEvent (org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent)5 ArrayList (java.util.ArrayList)4 ApplicationAttemptId (org.apache.hadoop.yarn.api.records.ApplicationAttemptId)4 LocalResource (org.apache.hadoop.yarn.api.records.LocalResource)4 DeletionService (org.apache.hadoop.yarn.server.nodemanager.DeletionService)4 ByteBuffer (java.nio.ByteBuffer)3 HashMap (java.util.HashMap)3 StartContainersResponse (org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse)3