Examples with StartContainerRequest org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest used on opensource projects

Example 41 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class TestContainerManager method testMultipleContainersLaunch.

@Test
public void testMultipleContainersLaunch() throws Exception {
    containerManager.start();
    List<StartContainerRequest> list = new ArrayList<>();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    for (int i = 0; i < 10; i++) {
        ContainerId cId = createContainerId(i);
        long identifier = 0;
        if ((i & 1) == 0)
            // container with even id fail
            identifier = ResourceManagerConstants.RM_INVALID_IDENTIFIER;
        else
            identifier = DUMMY_RM_IDENTIFIER;
        Token containerToken = createContainerToken(cId, identifier, context.getNodeId(), user, context.getContainerTokenSecretManager());
        StartContainerRequest request = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
        list.add(request);
    }
    StartContainersRequest requestList = StartContainersRequest.newInstance(list);
    StartContainersResponse response = containerManager.startContainers(requestList);
    Thread.sleep(5000);
    Assert.assertEquals(5, response.getSuccessfullyStartedContainers().size());
    for (ContainerId id : response.getSuccessfullyStartedContainers()) {
        // Containers with odd id should succeed.
        Assert.assertEquals(1, id.getContainerId() & 1);
    }
    Assert.assertEquals(5, response.getFailedRequests().size());
    for (Map.Entry<ContainerId, SerializedException> entry : response.getFailedRequests().entrySet()) {
        // Containers with even id should fail.
        Assert.assertEquals(0, entry.getKey().getContainerId() & 1);
        Assert.assertTrue(entry.getValue().getMessage().contains("Container " + entry.getKey() + " rejected as it is allocated by a previous RM"));
    }
}

Example 42 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class TestContainerManager method testMultipleContainersStopAndGetStatus.

@Test
public void testMultipleContainersStopAndGetStatus() throws Exception {
    containerManager.start();
    List<StartContainerRequest> startRequest = new ArrayList<>();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    List<ContainerId> containerIds = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        ContainerId cId;
        if ((i & 1) == 0) {
            // Containers with even id belong to an unauthorized app
            cId = createContainerId(i, 1);
        } else {
            cId = createContainerId(i, 0);
        }
        Token containerToken = createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager());
        StartContainerRequest request = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
        startRequest.add(request);
        containerIds.add(cId);
    }
    // start containers
    StartContainersRequest requestList = StartContainersRequest.newInstance(startRequest);
    containerManager.startContainers(requestList);
    Thread.sleep(5000);
    // Get container statuses
    GetContainerStatusesRequest statusRequest = GetContainerStatusesRequest.newInstance(containerIds);
    GetContainerStatusesResponse statusResponse = containerManager.getContainerStatuses(statusRequest);
    Assert.assertEquals(5, statusResponse.getContainerStatuses().size());
    for (ContainerStatus status : statusResponse.getContainerStatuses()) {
        // Containers with odd id should succeed
        Assert.assertEquals(1, status.getContainerId().getContainerId() & 1);
    }
    Assert.assertEquals(5, statusResponse.getFailedRequests().size());
    for (Map.Entry<ContainerId, SerializedException> entry : statusResponse.getFailedRequests().entrySet()) {
        // Containers with even id should fail.
        Assert.assertEquals(0, entry.getKey().getContainerId() & 1);
        Assert.assertTrue(entry.getValue().getMessage().contains("attempted to get status for non-application container"));
    }
    // stop containers
    StopContainersRequest stopRequest = StopContainersRequest.newInstance(containerIds);
    StopContainersResponse stopResponse = containerManager.stopContainers(stopRequest);
    Assert.assertEquals(5, stopResponse.getSuccessfullyStoppedContainers().size());
    for (ContainerId id : stopResponse.getSuccessfullyStoppedContainers()) {
        // Containers with odd id should succeed.
        Assert.assertEquals(1, id.getContainerId() & 1);
    }
    Assert.assertEquals(5, stopResponse.getFailedRequests().size());
    for (Map.Entry<ContainerId, SerializedException> entry : stopResponse.getFailedRequests().entrySet()) {
        // Containers with even id should fail.
        Assert.assertEquals(0, entry.getKey().getContainerId() & 1);
        Assert.assertTrue(entry.getValue().getMessage().contains("attempted to stop non-application container"));
    }
}

Example 43 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class TestLogAggregationService method testLogAggregationForRealContainerLaunch.

@Test
public void testLogAggregationForRealContainerLaunch() throws IOException, InterruptedException, YarnException {
    this.containerManager.start();
    File scriptFile = new File(tmpDir, "scriptFile.sh");
    PrintWriter fileWriter = new PrintWriter(scriptFile);
    fileWriter.write("\necho Hello World! Stdout! > " + new File(localLogDir, "stdout"));
    fileWriter.write("\necho Hello World! Stderr! > " + new File(localLogDir, "stderr"));
    fileWriter.write("\necho Hello World! Syslog! > " + new File(localLogDir, "syslog"));
    fileWriter.close();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    // ////// Construct the Container-id
    ApplicationId appId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0);
    URL resource_alpha = URL.fromPath(localFS.makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
    rsrc_alpha.setResource(resource_alpha);
    rsrc_alpha.setSize(-1);
    rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
    rsrc_alpha.setType(LocalResourceType.FILE);
    rsrc_alpha.setTimestamp(scriptFile.lastModified());
    String destinationFile = "dest_file";
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(destinationFile, rsrc_alpha);
    containerLaunchContext.setLocalResources(localResources);
    List<String> commands = new ArrayList<String>();
    commands.add("/bin/bash");
    commands.add(scriptFile.getAbsolutePath());
    containerLaunchContext.setCommands(commands);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, TestContainerManager.createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    this.containerManager.startContainers(allRequests);
    BaseContainerManagerTest.waitForContainerState(this.containerManager, cId, ContainerState.COMPLETE);
    this.containerManager.handle(new CMgrCompletedAppsEvent(Arrays.asList(appId), CMgrCompletedAppsEvent.Reason.ON_SHUTDOWN));
    this.containerManager.stop();
}

Example 44 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class TestContainerSchedulerQueuing method testKillMultipleOpportunisticContainers.

/**
   * Submit three OPPORTUNISTIC containers that can run concurrently, and one
   * GUARANTEED that needs to kill two of the OPPORTUNISTIC for it to run.
   * @throws Exception
   */
@Test
public void testKillMultipleOpportunisticContainers() throws Exception {
    containerManager.start();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(3), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(1500, 1), context.getContainerTokenSecretManager(), null, ExecutionType.GUARANTEED)));
    allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    BaseContainerManagerTest.waitForNMContainerState(containerManager, createContainerId(0), Arrays.asList(ContainerState.DONE, ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL), 40);
    Thread.sleep(5000);
    // Get container statuses. Container 0 should be killed, container 1
    // should be queued and container 2 should be running.
    int killedContainers = 0;
    List<ContainerId> statList = new ArrayList<ContainerId>();
    for (int i = 0; i < 4; i++) {
        statList.add(createContainerId(i));
    }
    GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.newInstance(statList);
    List<ContainerStatus> containerStatuses = containerManager.getContainerStatuses(statRequest).getContainerStatuses();
    for (ContainerStatus status : containerStatuses) {
        if (status.getDiagnostics().contains("Container Killed to make room for Guaranteed Container")) {
            killedContainers++;
        }
        System.out.println("\nStatus : [" + status + "]\n");
    }
    Assert.assertEquals(2, killedContainers);
}

Example 45 with StartContainerRequest

use of org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest in project hadoop by apache.

the class TestContainerSchedulerQueuing method testStartAndQueueMultipleContainers.

/**
   * Starts one OPPORTUNISTIC container that takes up the whole node's
   * resources, and submit two more that will be queued.
   * @throws Exception
   */
@Test
public void testStartAndQueueMultipleContainers() throws Exception {
    containerManager.start();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(1024, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(1024, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    Thread.sleep(5000);
    // Ensure first container is running and others are queued.
    List<ContainerId> statList = new ArrayList<ContainerId>();
    for (int i = 0; i < 3; i++) {
        statList.add(createContainerId(i));
    }
    GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.newInstance(Arrays.asList(createContainerId(0)));
    List<ContainerStatus> containerStatuses = containerManager.getContainerStatuses(statRequest).getContainerStatuses();
    for (ContainerStatus status : containerStatuses) {
        if (status.getContainerId().equals(createContainerId(0))) {
            Assert.assertEquals(org.apache.hadoop.yarn.api.records.ContainerState.RUNNING, status.getState());
        } else {
            Assert.assertEquals(org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED, status.getState());
        }
    }
    ContainerScheduler containerScheduler = containerManager.getContainerScheduler();
    // Ensure two containers are properly queued.
    Assert.assertEquals(2, containerScheduler.getNumQueuedContainers());
    Assert.assertEquals(0, containerScheduler.getNumQueuedGuaranteedContainers());
    Assert.assertEquals(2, containerScheduler.getNumQueuedOpportunisticContainers());
}