Examples with ContainerId org.apache.hadoop.yarn.api.records.ContainerId used on opensource projects

Example 66 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class TestYarnServerApiClasses method getIncreasedContainer.

private Container getIncreasedContainer(int containerID, int appAttemptId, int memory, int vCores) {
    ContainerId containerId = getContainerId(containerID, appAttemptId);
    Resource capability = Resource.newInstance(memory, vCores);
    return Container.newInstance(containerId, null, null, capability, null, null);
}

Example 67 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class TestNMLeveldbStateStoreService method testContainerStorage.

@Test
public void testContainerStorage() throws IOException {
    // test empty when no state
    List<RecoveredContainerState> recoveredContainers = stateStore.loadContainersState();
    assertTrue(recoveredContainers.isEmpty());
    // create a container request
    ApplicationId appId = ApplicationId.newInstance(1234, 3);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 4);
    ContainerId containerId = ContainerId.newContainerId(appAttemptId, 5);
    StartContainerRequest containerReq = createContainerRequest(containerId);
    // store a container and verify recovered
    stateStore.storeContainer(containerId, 0, containerReq);
    // verify the container version key is not stored for new containers
    DB db = stateStore.getDB();
    assertNull("version key present for new container", db.get(bytes(stateStore.getContainerVersionKey(containerId.toString()))));
    restartStateStore();
    recoveredContainers = stateStore.loadContainersState();
    assertEquals(1, recoveredContainers.size());
    RecoveredContainerState rcs = recoveredContainers.get(0);
    assertEquals(0, rcs.getVersion());
    assertEquals(RecoveredContainerStatus.REQUESTED, rcs.getStatus());
    assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode());
    assertEquals(false, rcs.getKilled());
    assertEquals(containerReq, rcs.getStartRequest());
    assertTrue(rcs.getDiagnostics().isEmpty());
    // store a new container record without StartContainerRequest
    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 6);
    stateStore.storeContainerLaunched(containerId1);
    recoveredContainers = stateStore.loadContainersState();
    // check whether the new container record is discarded
    assertEquals(1, recoveredContainers.size());
    // queue the container, and verify recovered
    stateStore.storeContainerQueued(containerId);
    restartStateStore();
    recoveredContainers = stateStore.loadContainersState();
    assertEquals(1, recoveredContainers.size());
    rcs = recoveredContainers.get(0);
    assertEquals(RecoveredContainerStatus.QUEUED, rcs.getStatus());
    assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode());
    assertEquals(false, rcs.getKilled());
    assertEquals(containerReq, rcs.getStartRequest());
    assertTrue(rcs.getDiagnostics().isEmpty());
    // launch the container, add some diagnostics, and verify recovered
    StringBuilder diags = new StringBuilder();
    stateStore.storeContainerLaunched(containerId);
    diags.append("some diags for container");
    stateStore.storeContainerDiagnostics(containerId, diags);
    restartStateStore();
    recoveredContainers = stateStore.loadContainersState();
    assertEquals(1, recoveredContainers.size());
    rcs = recoveredContainers.get(0);
    assertEquals(RecoveredContainerStatus.LAUNCHED, rcs.getStatus());
    assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode());
    assertEquals(false, rcs.getKilled());
    assertEquals(containerReq, rcs.getStartRequest());
    assertEquals(diags.toString(), rcs.getDiagnostics());
    // increase the container size, and verify recovered
    stateStore.storeContainerResourceChanged(containerId, 2, Resource.newInstance(2468, 4));
    restartStateStore();
    recoveredContainers = stateStore.loadContainersState();
    assertEquals(1, recoveredContainers.size());
    rcs = recoveredContainers.get(0);
    assertEquals(2, rcs.getVersion());
    assertEquals(RecoveredContainerStatus.LAUNCHED, rcs.getStatus());
    assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode());
    assertEquals(false, rcs.getKilled());
    assertEquals(Resource.newInstance(2468, 4), rcs.getCapability());
    // mark the container killed, add some more diags, and verify recovered
    diags.append("some more diags for container");
    stateStore.storeContainerDiagnostics(containerId, diags);
    stateStore.storeContainerKilled(containerId);
    restartStateStore();
    recoveredContainers = stateStore.loadContainersState();
    assertEquals(1, recoveredContainers.size());
    rcs = recoveredContainers.get(0);
    assertEquals(RecoveredContainerStatus.LAUNCHED, rcs.getStatus());
    assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode());
    assertTrue(rcs.getKilled());
    assertEquals(containerReq, rcs.getStartRequest());
    assertEquals(diags.toString(), rcs.getDiagnostics());
    // add yet more diags, mark container completed, and verify recovered
    diags.append("some final diags");
    stateStore.storeContainerDiagnostics(containerId, diags);
    stateStore.storeContainerCompleted(containerId, 21);
    restartStateStore();
    recoveredContainers = stateStore.loadContainersState();
    assertEquals(1, recoveredContainers.size());
    rcs = recoveredContainers.get(0);
    assertEquals(RecoveredContainerStatus.COMPLETED, rcs.getStatus());
    assertEquals(21, rcs.getExitCode());
    assertTrue(rcs.getKilled());
    assertEquals(containerReq, rcs.getStartRequest());
    assertEquals(diags.toString(), rcs.getDiagnostics());
    // store remainingRetryAttempts, workDir and logDir
    stateStore.storeContainerRemainingRetryAttempts(containerId, 6);
    stateStore.storeContainerWorkDir(containerId, "/test/workdir");
    stateStore.storeContainerLogDir(containerId, "/test/logdir");
    restartStateStore();
    recoveredContainers = stateStore.loadContainersState();
    assertEquals(1, recoveredContainers.size());
    rcs = recoveredContainers.get(0);
    assertEquals(6, rcs.getRemainingRetryAttempts());
    assertEquals("/test/workdir", rcs.getWorkDir());
    assertEquals("/test/logdir", rcs.getLogDir());
    // remove the container and verify not recovered
    stateStore.removeContainer(containerId);
    restartStateStore();
    recoveredContainers = stateStore.loadContainersState();
    assertTrue(recoveredContainers.isEmpty());
}

Example 68 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class TestContainerLogsPage method testContainerLogPageAccess.

@Test(timeout = 10000)
public void testContainerLogPageAccess() throws IOException {
    // SecureIOUtils require Native IO to be enabled. This test will run
    // only if it is enabled.
    assumeTrue(NativeIO.isAvailable());
    String user = "randomUser" + System.currentTimeMillis();
    File absLogDir = null, appDir = null, containerDir = null, syslog = null;
    try {
        // target log directory
        absLogDir = new File("target", TestContainerLogsPage.class.getSimpleName() + "LogDir").getAbsoluteFile();
        absLogDir.mkdir();
        Configuration conf = new Configuration();
        conf.set(YarnConfiguration.NM_LOG_DIRS, absLogDir.toURI().toString());
        conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
        UserGroupInformation.setConfiguration(conf);
        NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);
        healthChecker.init(conf);
        LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
        // Add an application and the corresponding containers
        RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf);
        long clusterTimeStamp = 1234;
        ApplicationId appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
        Application app = mock(Application.class);
        when(app.getAppId()).thenReturn(appId);
        // Making sure that application returns a random user. This is required
        // for SecureIOUtils' file owner check.
        when(app.getUser()).thenReturn(user);
        ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
        ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0);
        // Testing secure read access for log files
        // Creating application and container directory and syslog file.
        appDir = new File(absLogDir, appId.toString());
        appDir.mkdir();
        containerDir = new File(appDir, container1.toString());
        containerDir.mkdir();
        syslog = new File(containerDir, "syslog");
        syslog.createNewFile();
        BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(syslog));
        out.write("Log file Content".getBytes());
        out.close();
        Context context = mock(Context.class);
        ConcurrentMap<ApplicationId, Application> appMap = new ConcurrentHashMap<ApplicationId, Application>();
        appMap.put(appId, app);
        when(context.getApplications()).thenReturn(appMap);
        ConcurrentHashMap<ContainerId, Container> containers = new ConcurrentHashMap<ContainerId, Container>();
        when(context.getContainers()).thenReturn(containers);
        when(context.getLocalDirsHandler()).thenReturn(dirsHandler);
        MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1);
        container.setState(ContainerState.RUNNING);
        context.getContainers().put(container1, container);
        ContainersLogsBlock cLogsBlock = new ContainersLogsBlock(context);
        Map<String, String> params = new HashMap<String, String>();
        params.put(YarnWebParams.CONTAINER_ID, container1.toString());
        params.put(YarnWebParams.CONTAINER_LOG_TYPE, "syslog");
        Injector injector = WebAppTests.testPage(ContainerLogsPage.class, ContainersLogsBlock.class, cLogsBlock, params, (Module[]) null);
        PrintWriter spyPw = WebAppTests.getPrintWriter(injector);
        verify(spyPw).write("Exception reading log file. Application submitted by '" + user + "' doesn't own requested log file : syslog");
    } finally {
        if (syslog != null) {
            syslog.delete();
        }
        if (containerDir != null) {
            containerDir.delete();
        }
        if (appDir != null) {
            appDir.delete();
        }
        if (absLogDir != null) {
            absLogDir.delete();
        }
    }
}

Example 69 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class TestContainerLogsPage method testLogDirWithDriveLetter.

@Test
public void testLogDirWithDriveLetter() throws Exception {
    //To verify that logs paths which include drive letters (Windows)
    //do not lose their drive letter specification
    LocalDirsHandlerService localDirs = mock(LocalDirsHandlerService.class);
    List<String> logDirs = new ArrayList<String>();
    logDirs.add("F:/nmlogs");
    when(localDirs.getLogDirsForRead()).thenReturn(logDirs);
    ApplicationIdPBImpl appId = mock(ApplicationIdPBImpl.class);
    when(appId.toString()).thenReturn("app_id_1");
    ApplicationAttemptIdPBImpl appAttemptId = mock(ApplicationAttemptIdPBImpl.class);
    when(appAttemptId.getApplicationId()).thenReturn(appId);
    ContainerId containerId = mock(ContainerIdPBImpl.class);
    when(containerId.getApplicationAttemptId()).thenReturn(appAttemptId);
    List<File> logDirFiles = ContainerLogsUtils.getContainerLogDirs(containerId, localDirs);
    Assert.assertTrue("logDir lost drive letter " + logDirFiles.get(0), logDirFiles.get(0).toString().indexOf("F:" + File.separator + "nmlogs") > -1);
}

Example 70 with ContainerId

use of org.apache.hadoop.yarn.api.records.ContainerId in project hadoop by apache.

the class TestContainerLogsPage method testContainerLogDirs.

@Test(timeout = 30000)
public void testContainerLogDirs() throws IOException, YarnException {
    File absLogDir = new File("target", TestNMWebServer.class.getSimpleName() + "LogDir").getAbsoluteFile();
    String logdirwithFile = absLogDir.toURI().toString();
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.NM_LOG_DIRS, logdirwithFile);
    NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);
    healthChecker.init(conf);
    LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
    NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler, new ApplicationACLsManager(conf), new NMNullStateStoreService(), false, conf);
    // Add an application and the corresponding containers
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf);
    String user = "nobody";
    long clusterTimeStamp = 1234;
    ApplicationId appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
    Application app = mock(Application.class);
    when(app.getUser()).thenReturn(user);
    when(app.getAppId()).thenReturn(appId);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0);
    nmContext.getApplications().put(appId, app);
    MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1);
    container.setState(ContainerState.RUNNING);
    nmContext.getContainers().put(container1, container);
    List<File> files = null;
    files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
    Assert.assertTrue(!(files.get(0).toString().contains("file:")));
    // After container is completed, it is removed from nmContext
    nmContext.getContainers().remove(container1);
    Assert.assertNull(nmContext.getContainers().get(container1));
    files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
    Assert.assertTrue(!(files.get(0).toString().contains("file:")));
    // Create a new context to check if correct container log dirs are fetched
    // on full disk.
    LocalDirsHandlerService dirsHandlerForFullDisk = spy(dirsHandler);
    // good log dirs are empty and nm log dir is in the full log dir list.
    when(dirsHandlerForFullDisk.getLogDirs()).thenReturn(new ArrayList<String>());
    when(dirsHandlerForFullDisk.getLogDirsForRead()).thenReturn(Arrays.asList(new String[] { absLogDir.getAbsolutePath() }));
    nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk, new ApplicationACLsManager(conf), new NMNullStateStoreService(), false, conf);
    nmContext.getApplications().put(appId, app);
    container.setState(ContainerState.RUNNING);
    nmContext.getContainers().put(container1, container);
    List<File> dirs = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
    File containerLogDir = new File(absLogDir, appId + "/" + container1);
    Assert.assertTrue(dirs.contains(containerLogDir));
}