Search in sources :

Example 1 with NodeManagerMetrics

use of org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics in project hadoop by apache.

the class TestEventFlow method testSuccessfulContainerLaunch.

@Test
public void testSuccessfulContainerLaunch() throws InterruptedException, IOException, YarnException {
    FileContext localFS = FileContext.getLocalFSFileContext();
    localFS.delete(new Path(localDir.getAbsolutePath()), true);
    localFS.delete(new Path(localLogDir.getAbsolutePath()), true);
    localFS.delete(new Path(remoteLogDir.getAbsolutePath()), true);
    localDir.mkdir();
    localLogDir.mkdir();
    remoteLogDir.mkdir();
    YarnConfiguration conf = new YarnConfiguration();
    Context context = new NMContext(new NMContainerTokenSecretManager(conf), new NMTokenSecretManagerInNM(), null, null, new NMNullStateStoreService(), false, conf) {

        @Override
        public int getHttpPort() {
            return 1234;
        }
    };
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "0.0.0.0:" + ServerSocketUtil.getPort(8040, 10));
    ContainerExecutor exec = new DefaultContainerExecutor();
    exec.setConf(conf);
    DeletionService del = new DeletionService(exec);
    Dispatcher dispatcher = new AsyncDispatcher();
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(NodeManager.getNodeHealthScriptRunner(conf), dirsHandler);
    healthChecker.init(conf);
    NodeManagerMetrics metrics = NodeManagerMetrics.create();
    NodeStatusUpdater nodeStatusUpdater = new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics) {

        @Override
        protected ResourceTracker getRMClient() {
            return new LocalRMInterface();
        }

        ;

        @Override
        protected void stopRMProxy() {
            return;
        }

        @Override
        protected void startStatusUpdater() {
            // Don't start any updating thread.
            return;
        }

        @Override
        public long getRMIdentifier() {
            return SIMULATED_RM_IDENTIFIER;
        }
    };
    DummyContainerManager containerManager = new DummyContainerManager(context, exec, del, nodeStatusUpdater, metrics, dirsHandler);
    nodeStatusUpdater.init(conf);
    ((NMContext) context).setContainerManager(containerManager);
    nodeStatusUpdater.start();
    ((NMContext) context).setNodeStatusUpdater(nodeStatusUpdater);
    containerManager.init(conf);
    containerManager.start();
    ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    ApplicationId applicationId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
    ContainerId cID = ContainerId.newContainerId(applicationAttemptId, 0);
    String user = "testing";
    StartContainerRequest scRequest = StartContainerRequest.newInstance(launchContext, TestContainerManager.createContainerToken(cID, SIMULATED_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    BaseContainerManagerTest.waitForContainerState(containerManager, cID, Arrays.asList(ContainerState.RUNNING, ContainerState.SCHEDULED), 20);
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(cID);
    StopContainersRequest stopRequest = StopContainersRequest.newInstance(containerIds);
    containerManager.stopContainers(stopRequest);
    BaseContainerManagerTest.waitForContainerState(containerManager, cID, ContainerState.COMPLETE);
    containerManager.stop();
}
Also used : ArrayList(java.util.ArrayList) NMTokenSecretManagerInNM(org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM) Dispatcher(org.apache.hadoop.yarn.event.Dispatcher) AsyncDispatcher(org.apache.hadoop.yarn.event.AsyncDispatcher) NMNullStateStoreService(org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) NMContainerTokenSecretManager(org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager) NodeManagerMetrics(org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics) StopContainersRequest(org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest) Path(org.apache.hadoop.fs.Path) FileContext(org.apache.hadoop.fs.FileContext) NMContext(org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext) ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) StartContainersRequest(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest) ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) StartContainerRequest(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest) NMContext(org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext) AsyncDispatcher(org.apache.hadoop.yarn.event.AsyncDispatcher) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) FileContext(org.apache.hadoop.fs.FileContext) BaseContainerManagerTest(org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest) Test(org.junit.Test)

Example 2 with NodeManagerMetrics

use of org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics in project hadoop by apache.

the class TestLocalDirsHandlerService method testGetFullDirs.

@Test
public void testGetFullDirs() throws Exception {
    Configuration conf = new YarnConfiguration();
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
    FileContext localFs = FileContext.getLocalFSFileContext(conf);
    String localDir1 = new File(testDir, "localDir1").getPath();
    String localDir2 = new File(testDir, "localDir2").getPath();
    String logDir1 = new File(testDir, "logDir1").getPath();
    String logDir2 = new File(testDir, "logDir2").getPath();
    Path localDir1Path = new Path(localDir1);
    Path logDir1Path = new Path(logDir1);
    FsPermission dirPermissions = new FsPermission((short) 0410);
    localFs.mkdir(localDir1Path, dirPermissions, true);
    localFs.mkdir(logDir1Path, dirPermissions, true);
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir1 + "," + localDir2);
    conf.set(YarnConfiguration.NM_LOG_DIRS, logDir1 + "," + logDir2);
    conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 0.0f);
    NodeManagerMetrics nm = NodeManagerMetrics.create();
    LocalDirsHandlerService dirSvc = new LocalDirsHandlerService(nm);
    dirSvc.init(conf);
    Assert.assertEquals(0, dirSvc.getLocalDirs().size());
    Assert.assertEquals(0, dirSvc.getLogDirs().size());
    Assert.assertEquals(1, dirSvc.getDiskFullLocalDirs().size());
    Assert.assertEquals(1, dirSvc.getDiskFullLogDirs().size());
    // check the metrics
    Assert.assertEquals(2, nm.getBadLocalDirs());
    Assert.assertEquals(2, nm.getBadLogDirs());
    Assert.assertEquals(0, nm.getGoodLocalDirsDiskUtilizationPerc());
    Assert.assertEquals(0, nm.getGoodLogDirsDiskUtilizationPerc());
    Assert.assertEquals("", dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOCAL_DIRS));
    Assert.assertEquals("", dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOG_DIRS));
    Assert.assertEquals(localDir1 + "," + localDir2, dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS));
    Assert.assertEquals(logDir1 + "," + logDir2, dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS));
    conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100.0f);
    nm = NodeManagerMetrics.create();
    dirSvc = new LocalDirsHandlerService(nm);
    dirSvc.init(conf);
    Assert.assertEquals(1, dirSvc.getLocalDirs().size());
    Assert.assertEquals(1, dirSvc.getLogDirs().size());
    Assert.assertEquals(0, dirSvc.getDiskFullLocalDirs().size());
    Assert.assertEquals(0, dirSvc.getDiskFullLogDirs().size());
    // check the metrics
    File dir = new File(localDir1);
    int utilizationPerc = (int) ((dir.getTotalSpace() - dir.getUsableSpace()) * 100 / dir.getTotalSpace());
    Assert.assertEquals(1, nm.getBadLocalDirs());
    Assert.assertEquals(1, nm.getBadLogDirs());
    Assert.assertEquals(utilizationPerc, nm.getGoodLocalDirsDiskUtilizationPerc());
    Assert.assertEquals(utilizationPerc, nm.getGoodLogDirsDiskUtilizationPerc());
    Assert.assertEquals(localDir2, dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOCAL_DIRS));
    Assert.assertEquals(logDir2, dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOG_DIRS));
    Assert.assertEquals(localDir1 + "," + localDir2, dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS));
    Assert.assertEquals(logDir1 + "," + logDir2, dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS));
    FileUtils.deleteDirectory(new File(localDir1));
    FileUtils.deleteDirectory(new File(localDir2));
    FileUtils.deleteDirectory(new File(logDir1));
    FileUtils.deleteDirectory(new File(logDir2));
    dirSvc.close();
}
Also used : Path(org.apache.hadoop.fs.Path) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) NodeManagerMetrics(org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics) FsPermission(org.apache.hadoop.fs.permission.FsPermission) File(java.io.File) FileContext(org.apache.hadoop.fs.FileContext) Test(org.junit.Test)

Example 3 with NodeManagerMetrics

use of org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics in project hadoop by apache.

the class TestNMWebServer method testNMWebApp.

@Test
public void testNMWebApp() throws IOException, YarnException {
    Configuration conf = new Configuration();
    Context nmContext = new NodeManager.NMContext(null, null, null, null, null, false, conf);
    ResourceView resourceView = new ResourceView() {

        @Override
        public long getVmemAllocatedForContainers() {
            return 0;
        }

        @Override
        public long getPmemAllocatedForContainers() {
            return 0;
        }

        @Override
        public long getVCoresAllocatedForContainers() {
            return 0;
        }

        @Override
        public boolean isVmemCheckEnabled() {
            return true;
        }

        @Override
        public boolean isPmemCheckEnabled() {
            return true;
        }
    };
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
    NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);
    healthChecker.init(conf);
    LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
    WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager(conf), dirsHandler);
    server.init(conf);
    server.start();
    // Add an application and the corresponding containers
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf);
    Dispatcher dispatcher = new AsyncDispatcher();
    String user = "nobody";
    long clusterTimeStamp = 1234;
    ApplicationId appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
    Application app = mock(Application.class);
    when(app.getUser()).thenReturn(user);
    when(app.getAppId()).thenReturn(appId);
    nmContext.getApplications().put(appId, app);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0);
    ContainerId container2 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 1);
    NodeManagerMetrics metrics = mock(NodeManagerMetrics.class);
    NMStateStoreService stateStore = new NMNullStateStoreService();
    for (ContainerId containerId : new ContainerId[] { container1, container2 }) {
        // TODO: Use builder utils
        ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
        long currentTime = System.currentTimeMillis();
        Token containerToken = BuilderUtils.newContainerToken(containerId, 0, "127.0.0.1", 1234, user, BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123, "password".getBytes(), currentTime);
        Context context = mock(Context.class);
        Container container = new ContainerImpl(conf, dispatcher, launchContext, null, metrics, BuilderUtils.newContainerTokenIdentifier(containerToken), context) {

            @Override
            public ContainerState getContainerState() {
                return ContainerState.RUNNING;
            }

            ;
        };
        nmContext.getContainers().put(containerId, container);
        //TODO: Gross hack. Fix in code.
        ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
        nmContext.getApplications().get(applicationId).getContainers().put(containerId, container);
        writeContainerLogs(nmContext, containerId, dirsHandler);
    }
// TODO: Pull logs and test contents.
//    Thread.sleep(1000000);
}
Also used : ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) Context(org.apache.hadoop.yarn.server.nodemanager.Context) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) Configuration(org.apache.hadoop.conf.Configuration) NodeHealthCheckerService(org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService) Token(org.apache.hadoop.yarn.api.records.Token) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) LocalDirsHandlerService(org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService) Dispatcher(org.apache.hadoop.yarn.event.Dispatcher) AsyncDispatcher(org.apache.hadoop.yarn.event.AsyncDispatcher) NMNullStateStoreService(org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService) NMStateStoreService(org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService) ResourceView(org.apache.hadoop.yarn.server.nodemanager.ResourceView) ApplicationACLsManager(org.apache.hadoop.yarn.server.security.ApplicationACLsManager) Container(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container) RecordFactory(org.apache.hadoop.yarn.factories.RecordFactory) AsyncDispatcher(org.apache.hadoop.yarn.event.AsyncDispatcher) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) ContainerImpl(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl) NodeManagerMetrics(org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) Application(org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application) Test(org.junit.Test)

Aggregations

YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)3 NodeManagerMetrics (org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics)3 Test (org.junit.Test)3 Configuration (org.apache.hadoop.conf.Configuration)2 FileContext (org.apache.hadoop.fs.FileContext)2 Path (org.apache.hadoop.fs.Path)2 ApplicationAttemptId (org.apache.hadoop.yarn.api.records.ApplicationAttemptId)2 ApplicationId (org.apache.hadoop.yarn.api.records.ApplicationId)2 ContainerId (org.apache.hadoop.yarn.api.records.ContainerId)2 ContainerLaunchContext (org.apache.hadoop.yarn.api.records.ContainerLaunchContext)2 AsyncDispatcher (org.apache.hadoop.yarn.event.AsyncDispatcher)2 Dispatcher (org.apache.hadoop.yarn.event.Dispatcher)2 NMNullStateStoreService (org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService)2 File (java.io.File)1 ArrayList (java.util.ArrayList)1 FsPermission (org.apache.hadoop.fs.permission.FsPermission)1 StartContainerRequest (org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest)1 StartContainersRequest (org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)1 StopContainersRequest (org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest)1 Token (org.apache.hadoop.yarn.api.records.Token)1