Examples with ContainerLaunchContext org.apache.hadoop.yarn.api.records.ContainerLaunchContext used on opensource projects

Example 56 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestContainerManager method prepareContainerUpgrade.

/**
   * Prepare a launch Context for container upgrade and request the
   * Container Manager to re-initialize a running container using the
   * new launch context.
   * @param autoCommit Enable autoCommit.
   * @param failCmd injects a start script that intentionally fails.
   * @param failLoc injects a bad file Location that will fail localization.
   */
private void prepareContainerUpgrade(boolean autoCommit, boolean failCmd, boolean failLoc, ContainerId cId, File startFile) throws FileNotFoundException, YarnException, InterruptedException {
    // Re-write scriptfile and processStartFile
    File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile_new");
    PrintWriter fileWriter = new PrintWriter(scriptFile);
    writeScriptFile(fileWriter, "Upgrade World!", startFile, cId, failCmd);
    ContainerLaunchContext containerLaunchContext = prepareContainerLaunchContext(scriptFile, "dest_file_new", failLoc, 0);
    containerManager.reInitializeContainer(cId, containerLaunchContext, autoCommit);
    try {
        containerManager.reInitializeContainer(cId, containerLaunchContext, autoCommit);
    } catch (Exception e) {
        Assert.assertTrue(e.getMessage().contains("Cannot perform RE_INIT"));
    }
    int timeoutSecs = 0;
    int maxTimeToWait = failLoc ? 10 : 20;
    // Wait for new processStartfile to be created
    while (!startFile.exists() && timeoutSecs++ < maxTimeToWait) {
        Thread.sleep(1000);
        LOG.info("Waiting for New process start-file to be created");
    }
}

Example 57 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestNodeManagerShutdown method startContainer.

public static void startContainer(NodeManager nm, ContainerId cId, FileContext localFS, File scriptFileDir, File processStartFile, final int port) throws IOException, YarnException {
    File scriptFile = createUnhaltingScriptFile(cId, scriptFileDir, processStartFile);
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    NodeId nodeId = BuilderUtils.newNodeId(InetAddress.getByName("localhost").getCanonicalHostName(), port);
    URL localResourceUri = URL.fromPath(localFS.makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource localResource = recordFactory.newRecordInstance(LocalResource.class);
    localResource.setResource(localResourceUri);
    localResource.setSize(-1);
    localResource.setVisibility(LocalResourceVisibility.APPLICATION);
    localResource.setType(LocalResourceType.FILE);
    localResource.setTimestamp(scriptFile.lastModified());
    String destinationFile = "dest_file";
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(destinationFile, localResource);
    containerLaunchContext.setLocalResources(localResources);
    List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
    containerLaunchContext.setCommands(commands);
    final InetSocketAddress containerManagerBindAddress = NetUtils.createSocketAddrForHost("127.0.0.1", port);
    UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(cId.toString());
    org.apache.hadoop.security.token.Token<NMTokenIdentifier> nmToken = ConverterUtils.convertFromYarn(nm.getNMContext().getNMTokenSecretManager().createNMToken(cId.getApplicationAttemptId(), nodeId, user), containerManagerBindAddress);
    currentUser.addToken(nmToken);
    ContainerManagementProtocol containerManager = currentUser.doAs(new PrivilegedAction<ContainerManagementProtocol>() {

        @Override
        public ContainerManagementProtocol run() {
            Configuration conf = new Configuration();
            YarnRPC rpc = YarnRPC.create(conf);
            InetSocketAddress containerManagerBindAddress = NetUtils.createSocketAddrForHost("127.0.0.1", port);
            return (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, containerManagerBindAddress, conf);
        }
    });
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, TestContainerManager.createContainerToken(cId, 0, nodeId, user, nm.getNMContext().getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(cId);
    GetContainerStatusesRequest request = GetContainerStatusesRequest.newInstance(containerIds);
    ContainerStatus containerStatus = containerManager.getContainerStatuses(request).getContainerStatuses().get(0);
    Assert.assertTrue(EnumSet.of(ContainerState.RUNNING, ContainerState.SCHEDULED).contains(containerStatus.getState()));
}

Example 58 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestApplication method createMockedContainer.

private Container createMockedContainer(ApplicationId appId, int containerId) {
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    ContainerId cId = BuilderUtils.newContainerId(appAttemptId, containerId);
    Container c = mock(Container.class);
    when(c.getContainerId()).thenReturn(cId);
    ContainerLaunchContext launchContext = mock(ContainerLaunchContext.class);
    when(c.getLaunchContext()).thenReturn(launchContext);
    when(launchContext.getApplicationACLs()).thenReturn(new HashMap<ApplicationAccessType, String>());
    when(c.cloneAndGetContainerStatus()).thenReturn(BuilderUtils.newContainerStatus(cId, ContainerState.NEW, "", 0, Resource.newInstance(1024, 1)));
    return c;
}

Example 59 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestNodeManagerReboot method testClearLocalDirWhenNodeReboot.

@Test(timeout = 2000000)
public void testClearLocalDirWhenNodeReboot() throws IOException, YarnException, InterruptedException {
    nm = new MyNodeManager();
    nm.start();
    final ContainerManagementProtocol containerManager = nm.getContainerManager();
    // create files under fileCache
    createFiles(nmLocalDir.getAbsolutePath(), ContainerLocalizer.FILECACHE, 100);
    localResourceDir.mkdirs();
    ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
    // Construct the Container-id
    ContainerId cId = createContainerId();
    URL localResourceUri = URL.fromPath(localFS.makeQualified(new Path(localResourceDir.getAbsolutePath())));
    LocalResource localResource = LocalResource.newInstance(localResourceUri, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, -1, localResourceDir.lastModified());
    String destinationFile = "dest_file";
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(destinationFile, localResource);
    containerLaunchContext.setLocalResources(localResources);
    List<String> commands = new ArrayList<String>();
    containerLaunchContext.setCommands(commands);
    NodeId nodeId = nm.getNMContext().getNodeId();
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, TestContainerManager.createContainerToken(cId, 0, nodeId, destinationFile, nm.getNMContext().getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    final StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    final UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(cId.getApplicationAttemptId().toString());
    NMTokenIdentifier nmIdentifier = new NMTokenIdentifier(cId.getApplicationAttemptId(), nodeId, user, 123);
    currentUser.addTokenIdentifier(nmIdentifier);
    currentUser.doAs(new PrivilegedExceptionAction<Void>() {

        @Override
        public Void run() throws YarnException, IOException {
            nm.getContainerManager().startContainers(allRequests);
            return null;
        }
    });
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(cId);
    GetContainerStatusesRequest request = GetContainerStatusesRequest.newInstance(containerIds);
    Container container = nm.getNMContext().getContainers().get(request.getContainerIds().get(0));
    final int MAX_TRIES = 20;
    int numTries = 0;
    while (!container.getContainerState().equals(ContainerState.DONE) && numTries <= MAX_TRIES) {
        try {
            Thread.sleep(500);
        } catch (InterruptedException ex) {
        // Do nothing
        }
        numTries++;
    }
    Assert.assertEquals(ContainerState.DONE, container.getContainerState());
    Assert.assertTrue("The container should create a subDir named currentUser: " + user + "under localDir/usercache", numOfLocalDirs(nmLocalDir.getAbsolutePath(), ContainerLocalizer.USERCACHE) > 0);
    Assert.assertTrue("There should be files or Dirs under nm_private when " + "container is launched", numOfLocalDirs(nmLocalDir.getAbsolutePath(), ResourceLocalizationService.NM_PRIVATE_DIR) > 0);
    // restart the NodeManager
    restartNM(MAX_TRIES);
    checkNumOfLocalDirs();
    verify(delService, times(1)).delete((String) isNull(), argThat(new PathInclude(ResourceLocalizationService.NM_PRIVATE_DIR + "_DEL_")));
    verify(delService, times(1)).delete((String) isNull(), argThat(new PathInclude(ContainerLocalizer.FILECACHE + "_DEL_")));
    verify(delService, times(1)).scheduleFileDeletionTask(argThat(new FileDeletionInclude(user, null, new String[] { destinationFile })));
    verify(delService, times(1)).scheduleFileDeletionTask(argThat(new FileDeletionInclude(null, ContainerLocalizer.USERCACHE + "_DEL_", new String[] {})));
    // restart the NodeManager again
    // this time usercache directory should be empty
    restartNM(MAX_TRIES);
    checkNumOfLocalDirs();
}

Example 60 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestSystemMetricsPublisher method createRMApp.

private static RMApp createRMApp(ApplicationId appId) {
    RMApp app = mock(RMAppImpl.class);
    when(app.getApplicationId()).thenReturn(appId);
    when(app.getName()).thenReturn("test app");
    when(app.getApplicationType()).thenReturn("test app type");
    when(app.getUser()).thenReturn("test user");
    when(app.getQueue()).thenReturn("test queue");
    when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L);
    when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L);
    when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L);
    when(app.getDiagnostics()).thenReturn(new StringBuilder("test diagnostics info"));
    RMAppAttempt appAttempt = mock(RMAppAttempt.class);
    when(appAttempt.getAppAttemptId()).thenReturn(ApplicationAttemptId.newInstance(appId, 1));
    when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
    when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);
    when(app.getRMAppMetrics()).thenReturn(new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE, Integer.MAX_VALUE, Long.MAX_VALUE));
    Set<String> appTags = new HashSet<String>();
    appTags.add("test");
    appTags.add("tags");
    when(app.getApplicationTags()).thenReturn(appTags);
    ApplicationSubmissionContext asc = mock(ApplicationSubmissionContext.class);
    when(asc.getUnmanagedAM()).thenReturn(false);
    when(asc.getPriority()).thenReturn(Priority.newInstance(10));
    when(asc.getNodeLabelExpression()).thenReturn("high-cpu");
    ContainerLaunchContext containerLaunchContext = mock(ContainerLaunchContext.class);
    when(containerLaunchContext.getCommands()).thenReturn(Collections.singletonList("java -Xmx1024m"));
    when(asc.getAMContainerSpec()).thenReturn(containerLaunchContext);
    when(app.getApplicationSubmissionContext()).thenReturn(asc);
    when(app.getAppNodeLabelExpression()).thenCallRealMethod();
    ResourceRequest amReq = mock(ResourceRequest.class);
    when(amReq.getNodeLabelExpression()).thenReturn("high-mem");
    when(app.getAMResourceRequest()).thenReturn(amReq);
    when(app.getAmNodeLabelExpression()).thenCallRealMethod();
    when(app.getApplicationPriority()).thenReturn(Priority.newInstance(10));
    when(app.getCallerContext()).thenReturn(new CallerContext.Builder("context").build());
    return app;
}