Examples with StartContainersRequest org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest used on opensource projects

Example 36 with StartContainersRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.

the class TestContainerManager method testContainerSetup.

@Test
public void testContainerSetup() throws Exception {
    containerManager.start();
    // ////// Create the resources for the container
    File dir = new File(tmpDir, "dir");
    dir.mkdirs();
    File file = new File(dir, "file");
    PrintWriter fileWriter = new PrintWriter(file);
    fileWriter.write("Hello World!");
    fileWriter.close();
    // ////// Construct the Container-id
    ContainerId cId = createContainerId(0);
    // ////// Construct the container-spec.
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    URL resource_alpha = URL.fromPath(localFS.makeQualified(new Path(file.getAbsolutePath())));
    LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
    rsrc_alpha.setResource(resource_alpha);
    rsrc_alpha.setSize(-1);
    rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
    rsrc_alpha.setType(LocalResourceType.FILE);
    rsrc_alpha.setTimestamp(file.lastModified());
    String destinationFile = "dest_file";
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(destinationFile, rsrc_alpha);
    containerLaunchContext.setLocalResources(localResources);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE, 40);
    // Now ascertain that the resources are localised correctly.
    ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
    String appIDStr = appId.toString();
    String containerIDStr = cId.toString();
    File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
    File userDir = new File(userCacheDir, user);
    File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
    File appDir = new File(appCache, appIDStr);
    File containerDir = new File(appDir, containerIDStr);
    File targetFile = new File(containerDir, destinationFile);
    File sysDir = new File(localDir, ResourceLocalizationService.NM_PRIVATE_DIR);
    File appSysDir = new File(sysDir, appIDStr);
    File containerSysDir = new File(appSysDir, containerIDStr);
    for (File f : new File[] { localDir, sysDir, userCacheDir, appDir, appSysDir, containerDir, containerSysDir }) {
        Assert.assertTrue(f.getAbsolutePath() + " doesn't exist!!", f.exists());
        Assert.assertTrue(f.getAbsolutePath() + " is not a directory!!", f.isDirectory());
    }
    Assert.assertTrue(targetFile.getAbsolutePath() + " doesn't exist!!", targetFile.exists());
    // Now verify the contents of the file
    BufferedReader reader = new BufferedReader(new FileReader(targetFile));
    Assert.assertEquals("Hello World!", reader.readLine());
    Assert.assertEquals(null, reader.readLine());
}

Example 37 with StartContainersRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.

the class TestContainerManager method testContainerLaunchFromPreviousRM.

@Test
public void testContainerLaunchFromPreviousRM() throws IOException, InterruptedException, YarnException {
    containerManager.start();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    ContainerId cId1 = createContainerId(0);
    ContainerId cId2 = createContainerId(0);
    containerLaunchContext.setLocalResources(new HashMap<String, LocalResource>());
    // Construct the Container with Invalid RMIdentifier
    StartContainerRequest startRequest1 = StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(cId1, ResourceManagerConstants.RM_INVALID_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(startRequest1);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    boolean catchException = false;
    try {
        StartContainersResponse response = containerManager.startContainers(allRequests);
        if (response.getFailedRequests().containsKey(cId1)) {
            throw response.getFailedRequests().get(cId1).deSerialize();
        }
    } catch (Throwable e) {
        e.printStackTrace();
        catchException = true;
        Assert.assertTrue(e.getMessage().contains("Container " + cId1 + " rejected as it is allocated by a previous RM"));
        Assert.assertTrue(e.getClass().getName().equalsIgnoreCase(InvalidContainerException.class.getName()));
    }
    // Verify that startContainer fail because of invalid container request
    Assert.assertTrue(catchException);
    // Construct the Container with a RMIdentifier within current RM
    StartContainerRequest startRequest2 = StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(cId2, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager()));
    List<StartContainerRequest> list2 = new ArrayList<>();
    list.add(startRequest2);
    StartContainersRequest allRequests2 = StartContainersRequest.newInstance(list2);
    containerManager.startContainers(allRequests2);
    boolean noException = true;
    try {
        containerManager.startContainers(allRequests2);
    } catch (YarnException e) {
        noException = false;
    }
    // Verify that startContainer get no YarnException
    Assert.assertTrue(noException);
}

Example 38 with StartContainersRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.

the class TestContainerManager method testMultipleContainersLaunch.

@Test
public void testMultipleContainersLaunch() throws Exception {
    containerManager.start();
    List<StartContainerRequest> list = new ArrayList<>();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    for (int i = 0; i < 10; i++) {
        ContainerId cId = createContainerId(i);
        long identifier = 0;
        if ((i & 1) == 0)
            // container with even id fail
            identifier = ResourceManagerConstants.RM_INVALID_IDENTIFIER;
        else
            identifier = DUMMY_RM_IDENTIFIER;
        Token containerToken = createContainerToken(cId, identifier, context.getNodeId(), user, context.getContainerTokenSecretManager());
        StartContainerRequest request = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
        list.add(request);
    }
    StartContainersRequest requestList = StartContainersRequest.newInstance(list);
    StartContainersResponse response = containerManager.startContainers(requestList);
    Thread.sleep(5000);
    Assert.assertEquals(5, response.getSuccessfullyStartedContainers().size());
    for (ContainerId id : response.getSuccessfullyStartedContainers()) {
        // Containers with odd id should succeed.
        Assert.assertEquals(1, id.getContainerId() & 1);
    }
    Assert.assertEquals(5, response.getFailedRequests().size());
    for (Map.Entry<ContainerId, SerializedException> entry : response.getFailedRequests().entrySet()) {
        // Containers with even id should fail.
        Assert.assertEquals(0, entry.getKey().getContainerId() & 1);
        Assert.assertTrue(entry.getValue().getMessage().contains("Container " + entry.getKey() + " rejected as it is allocated by a previous RM"));
    }
}

Example 39 with StartContainersRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.

the class TestContainerManager method testMultipleContainersStopAndGetStatus.

@Test
public void testMultipleContainersStopAndGetStatus() throws Exception {
    containerManager.start();
    List<StartContainerRequest> startRequest = new ArrayList<>();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    List<ContainerId> containerIds = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        ContainerId cId;
        if ((i & 1) == 0) {
            // Containers with even id belong to an unauthorized app
            cId = createContainerId(i, 1);
        } else {
            cId = createContainerId(i, 0);
        }
        Token containerToken = createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager());
        StartContainerRequest request = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
        startRequest.add(request);
        containerIds.add(cId);
    }
    // start containers
    StartContainersRequest requestList = StartContainersRequest.newInstance(startRequest);
    containerManager.startContainers(requestList);
    Thread.sleep(5000);
    // Get container statuses
    GetContainerStatusesRequest statusRequest = GetContainerStatusesRequest.newInstance(containerIds);
    GetContainerStatusesResponse statusResponse = containerManager.getContainerStatuses(statusRequest);
    Assert.assertEquals(5, statusResponse.getContainerStatuses().size());
    for (ContainerStatus status : statusResponse.getContainerStatuses()) {
        // Containers with odd id should succeed
        Assert.assertEquals(1, status.getContainerId().getContainerId() & 1);
    }
    Assert.assertEquals(5, statusResponse.getFailedRequests().size());
    for (Map.Entry<ContainerId, SerializedException> entry : statusResponse.getFailedRequests().entrySet()) {
        // Containers with even id should fail.
        Assert.assertEquals(0, entry.getKey().getContainerId() & 1);
        Assert.assertTrue(entry.getValue().getMessage().contains("attempted to get status for non-application container"));
    }
    // stop containers
    StopContainersRequest stopRequest = StopContainersRequest.newInstance(containerIds);
    StopContainersResponse stopResponse = containerManager.stopContainers(stopRequest);
    Assert.assertEquals(5, stopResponse.getSuccessfullyStoppedContainers().size());
    for (ContainerId id : stopResponse.getSuccessfullyStoppedContainers()) {
        // Containers with odd id should succeed.
        Assert.assertEquals(1, id.getContainerId() & 1);
    }
    Assert.assertEquals(5, stopResponse.getFailedRequests().size());
    for (Map.Entry<ContainerId, SerializedException> entry : stopResponse.getFailedRequests().entrySet()) {
        // Containers with even id should fail.
        Assert.assertEquals(0, entry.getKey().getContainerId() & 1);
        Assert.assertTrue(entry.getValue().getMessage().contains("attempted to stop non-application container"));
    }
}

Example 40 with StartContainersRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.

the class TestLogAggregationService method testLogAggregationForRealContainerLaunch.

@Test
public void testLogAggregationForRealContainerLaunch() throws IOException, InterruptedException, YarnException {
    this.containerManager.start();
    File scriptFile = new File(tmpDir, "scriptFile.sh");
    PrintWriter fileWriter = new PrintWriter(scriptFile);
    fileWriter.write("\necho Hello World! Stdout! > " + new File(localLogDir, "stdout"));
    fileWriter.write("\necho Hello World! Stderr! > " + new File(localLogDir, "stderr"));
    fileWriter.write("\necho Hello World! Syslog! > " + new File(localLogDir, "syslog"));
    fileWriter.close();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    // ////// Construct the Container-id
    ApplicationId appId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0);
    URL resource_alpha = URL.fromPath(localFS.makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
    rsrc_alpha.setResource(resource_alpha);
    rsrc_alpha.setSize(-1);
    rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
    rsrc_alpha.setType(LocalResourceType.FILE);
    rsrc_alpha.setTimestamp(scriptFile.lastModified());
    String destinationFile = "dest_file";
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(destinationFile, rsrc_alpha);
    containerLaunchContext.setLocalResources(localResources);
    List<String> commands = new ArrayList<String>();
    commands.add("/bin/bash");
    commands.add(scriptFile.getAbsolutePath());
    containerLaunchContext.setCommands(commands);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, TestContainerManager.createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    this.containerManager.startContainers(allRequests);
    BaseContainerManagerTest.waitForContainerState(this.containerManager, cId, ContainerState.COMPLETE);
    this.containerManager.handle(new CMgrCompletedAppsEvent(Arrays.asList(appId), CMgrCompletedAppsEvent.Reason.ON_SHUTDOWN));
    this.containerManager.stop();
}