Examples with StartContainerRequest org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest used on opensource projects

Example 31 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class ContainerManagerImpl method startContainers.

/**
   * Start a list of containers on this NodeManager.
   */
@Override
public StartContainersResponse startContainers(StartContainersRequest requests) throws YarnException, IOException {
    if (blockNewContainerRequests.get()) {
        throw new NMNotYetReadyException("Rejecting new containers as NodeManager has not" + " yet connected with ResourceManager");
    }
    UserGroupInformation remoteUgi = getRemoteUgi();
    NMTokenIdentifier nmTokenIdentifier = selectNMTokenIdentifier(remoteUgi);
    authorizeUser(remoteUgi, nmTokenIdentifier);
    List<ContainerId> succeededContainers = new ArrayList<ContainerId>();
    Map<ContainerId, SerializedException> failedContainers = new HashMap<ContainerId, SerializedException>();
    // been added to the containers map in NMContext.
    synchronized (this.context) {
        for (StartContainerRequest request : requests.getStartContainerRequests()) {
            ContainerId containerId = null;
            try {
                if (request.getContainerToken() == null || request.getContainerToken().getIdentifier() == null) {
                    throw new IOException(INVALID_CONTAINERTOKEN_MSG);
                }
                ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils.newContainerTokenIdentifier(request.getContainerToken());
                verifyAndGetContainerTokenIdentifier(request.getContainerToken(), containerTokenIdentifier);
                containerId = containerTokenIdentifier.getContainerID();
                // type AM and if the AMRMProxy service is enabled
                if (amrmProxyEnabled && containerTokenIdentifier.getContainerType().equals(ContainerType.APPLICATION_MASTER)) {
                    this.getAMRMProxyService().processApplicationStartRequest(request);
                }
                performContainerPreStartChecks(nmTokenIdentifier, request, containerTokenIdentifier);
                startContainerInternal(containerTokenIdentifier, request);
                succeededContainers.add(containerId);
            } catch (YarnException e) {
                failedContainers.put(containerId, SerializedException.newInstance(e));
            } catch (InvalidToken ie) {
                failedContainers.put(containerId, SerializedException.newInstance(ie));
                throw ie;
            } catch (IOException e) {
                throw RPCUtil.getRemoteException(e);
            }
        }
        return StartContainersResponse.newInstance(getAuxServiceMetaData(), succeededContainers, failedContainers);
    }
}

Example 32 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class TestContainerManager method testLocalingResourceWhileContainerRunning.

// Start the container
// While the container is running, localize new resources.
// Verify the symlink is created properly
@Test
public void testLocalingResourceWhileContainerRunning() throws Exception {
    // Real del service
    delSrvc = new DeletionService(exec);
    delSrvc.init(conf);
    ((NodeManager.NMContext) context).setContainerExecutor(exec);
    containerManager = createContainerManager(delSrvc);
    containerManager.init(conf);
    containerManager.start();
    // set up local resources
    Map<String, LocalResource> localResource = setupLocalResources("file", "symLink1");
    ContainerLaunchContext context = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    context.setLocalResources(localResource);
    // a long running container - sleep
    context.setCommands(Arrays.asList("sleep 6"));
    ContainerId cId = createContainerId(0);
    // start the container
    StartContainerRequest scRequest = StartContainerRequest.newInstance(context, createContainerToken(cId, DUMMY_RM_IDENTIFIER, this.context.getNodeId(), user, this.context.getContainerTokenSecretManager()));
    StartContainersRequest allRequests = StartContainersRequest.newInstance(Arrays.asList(scRequest));
    containerManager.startContainers(allRequests);
    BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.RUNNING);
    BaseContainerManagerTest.waitForApplicationState(containerManager, cId.getApplicationAttemptId().getApplicationId(), ApplicationState.RUNNING);
    checkResourceLocalized(cId, "symLink1");
    // Localize new local resources while container is running
    Map<String, LocalResource> localResource2 = setupLocalResources("file2", "symLink2");
    ResourceLocalizationRequest request = ResourceLocalizationRequest.newInstance(cId, localResource2);
    containerManager.localize(request);
    // Verify resource is localized and symlink is created.
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        public Boolean get() {
            try {
                checkResourceLocalized(cId, "symLink2");
                return true;
            } catch (Throwable e) {
                return false;
            }
        }
    }, 500, 20000);
    BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE);
    // Verify container cannot localize resources while at non-running state.
    try {
        containerManager.localize(request);
        Assert.fail();
    } catch (YarnException e) {
        Assert.assertTrue(e.getMessage().contains("Cannot perform LOCALIZE"));
    }
}

Example 33 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class TestContainerManager method testUnauthorizedRequests.

@Test
public void testUnauthorizedRequests() throws IOException, YarnException {
    containerManager.start();
    // Create a containerId that belongs to an unauthorized appId
    ContainerId cId = createContainerId(0, 1);
    // startContainers()
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    StartContainersResponse startResponse = containerManager.startContainers(allRequests);
    Assert.assertFalse("Should not be authorized to start container", startResponse.getSuccessfullyStartedContainers().contains(cId));
    Assert.assertTrue("Start container request should fail", startResponse.getFailedRequests().containsKey(cId));
    // Insert the containerId into context, make it as if it is running
    ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils.newContainerTokenIdentifier(scRequest.getContainerToken());
    Container container = new ContainerImpl(conf, null, containerLaunchContext, null, metrics, containerTokenIdentifier, context);
    context.getContainers().put(cId, container);
    // stopContainers()
    List<ContainerId> containerIds = new ArrayList<>();
    containerIds.add(cId);
    StopContainersRequest stopRequest = StopContainersRequest.newInstance(containerIds);
    StopContainersResponse stopResponse = containerManager.stopContainers(stopRequest);
    Assert.assertFalse("Should not be authorized to stop container", stopResponse.getSuccessfullyStoppedContainers().contains(cId));
    Assert.assertTrue("Stop container request should fail", stopResponse.getFailedRequests().containsKey(cId));
    // getContainerStatuses()
    containerIds = new ArrayList<>();
    containerIds.add(cId);
    GetContainerStatusesRequest request = GetContainerStatusesRequest.newInstance(containerIds);
    GetContainerStatusesResponse response = containerManager.getContainerStatuses(request);
    Assert.assertEquals("Should not be authorized to get container status", response.getContainerStatuses().size(), 0);
    Assert.assertTrue("Get status request should fail", response.getFailedRequests().containsKey(cId));
}

Example 34 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class TestContainerManager method testContainerLaunchAndExit.

protected void testContainerLaunchAndExit(int exitCode) throws IOException, InterruptedException, YarnException {
    File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
    PrintWriter fileWriter = new PrintWriter(scriptFile);
    File processStartFile = new File(tmpDir, "start_file.txt").getAbsoluteFile();
    // ////// Construct the Container-id
    ContainerId cId = createContainerId(0);
    if (Shell.WINDOWS) {
        fileWriter.println("@echo Hello World!> " + processStartFile);
        fileWriter.println("@echo " + cId + ">> " + processStartFile);
        if (exitCode != 0) {
            fileWriter.println("@exit " + exitCode);
        }
    } else {
        // So that start file is readable by the test
        fileWriter.write("\numask 0");
        fileWriter.write("\necho Hello World! > " + processStartFile);
        fileWriter.write("\necho $$ >> " + processStartFile);
        // Have script throw an exit code at the end
        if (exitCode != 0) {
            fileWriter.write("\nexit " + exitCode);
        }
    }
    fileWriter.close();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    URL resource_alpha = URL.fromPath(localFS.makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
    rsrc_alpha.setResource(resource_alpha);
    rsrc_alpha.setSize(-1);
    rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
    rsrc_alpha.setType(LocalResourceType.FILE);
    rsrc_alpha.setTimestamp(scriptFile.lastModified());
    String destinationFile = "dest_file";
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(destinationFile, rsrc_alpha);
    containerLaunchContext.setLocalResources(localResources);
    List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
    containerLaunchContext.setCommands(commands);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE);
    List<ContainerId> containerIds = new ArrayList<>();
    containerIds.add(cId);
    GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.newInstance(containerIds);
    ContainerStatus containerStatus = containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
    // Verify exit status matches exit state of script
    Assert.assertEquals(exitCode, containerStatus.getExitStatus());
}

Example 35 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class TestContainerManagerRecovery method startContainer.

private StartContainersResponse startContainer(Context context, final ContainerManagerImpl cm, ContainerId cid, ContainerLaunchContext clc, LogAggregationContext logAggregationContext) throws Exception {
    UserGroupInformation user = UserGroupInformation.createRemoteUser(cid.getApplicationAttemptId().toString());
    StartContainerRequest scReq = StartContainerRequest.newInstance(clc, TestContainerManager.createContainerToken(cid, 0, context.getNodeId(), user.getShortUserName(), context.getContainerTokenSecretManager(), logAggregationContext));
    final List<StartContainerRequest> scReqList = new ArrayList<StartContainerRequest>();
    scReqList.add(scReq);
    NMTokenIdentifier nmToken = new NMTokenIdentifier(cid.getApplicationAttemptId(), context.getNodeId(), user.getShortUserName(), context.getNMTokenSecretManager().getCurrentKey().getKeyId());
    user.addTokenIdentifier(nmToken);
    return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {

        @Override
        public StartContainersResponse run() throws Exception {
            return cm.startContainers(StartContainersRequest.newInstance(scReqList));
        }
    });
}