Search in sources :

Example 21 with GetContainerStatusesRequest

use of org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest in project hadoop by apache.

the class TestContainerManager method testMultipleContainersStopAndGetStatus.

@Test
public void testMultipleContainersStopAndGetStatus() throws Exception {
    containerManager.start();
    List<StartContainerRequest> startRequest = new ArrayList<>();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    List<ContainerId> containerIds = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        ContainerId cId;
        if ((i & 1) == 0) {
            // Containers with even id belong to an unauthorized app
            cId = createContainerId(i, 1);
        } else {
            cId = createContainerId(i, 0);
        }
        Token containerToken = createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager());
        StartContainerRequest request = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
        startRequest.add(request);
        containerIds.add(cId);
    }
    // start containers
    StartContainersRequest requestList = StartContainersRequest.newInstance(startRequest);
    containerManager.startContainers(requestList);
    Thread.sleep(5000);
    // Get container statuses
    GetContainerStatusesRequest statusRequest = GetContainerStatusesRequest.newInstance(containerIds);
    GetContainerStatusesResponse statusResponse = containerManager.getContainerStatuses(statusRequest);
    Assert.assertEquals(5, statusResponse.getContainerStatuses().size());
    for (ContainerStatus status : statusResponse.getContainerStatuses()) {
        // Containers with odd id should succeed
        Assert.assertEquals(1, status.getContainerId().getContainerId() & 1);
    }
    Assert.assertEquals(5, statusResponse.getFailedRequests().size());
    for (Map.Entry<ContainerId, SerializedException> entry : statusResponse.getFailedRequests().entrySet()) {
        // Containers with even id should fail.
        Assert.assertEquals(0, entry.getKey().getContainerId() & 1);
        Assert.assertTrue(entry.getValue().getMessage().contains("attempted to get status for non-application container"));
    }
    // stop containers
    StopContainersRequest stopRequest = StopContainersRequest.newInstance(containerIds);
    StopContainersResponse stopResponse = containerManager.stopContainers(stopRequest);
    Assert.assertEquals(5, stopResponse.getSuccessfullyStoppedContainers().size());
    for (ContainerId id : stopResponse.getSuccessfullyStoppedContainers()) {
        // Containers with odd id should succeed.
        Assert.assertEquals(1, id.getContainerId() & 1);
    }
    Assert.assertEquals(5, stopResponse.getFailedRequests().size());
    for (Map.Entry<ContainerId, SerializedException> entry : stopResponse.getFailedRequests().entrySet()) {
        // Containers with even id should fail.
        Assert.assertEquals(0, entry.getKey().getContainerId() & 1);
        Assert.assertTrue(entry.getValue().getMessage().contains("attempted to stop non-application container"));
    }
}
Also used : StartContainersRequest(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest) GetContainerStatusesRequest(org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest) SerializedException(org.apache.hadoop.yarn.api.records.SerializedException) ArrayList(java.util.ArrayList) Token(org.apache.hadoop.yarn.api.records.Token) ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) StartContainerRequest(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest) GetContainerStatusesResponse(org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse) ContainerStatus(org.apache.hadoop.yarn.api.records.ContainerStatus) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) Map(java.util.Map) HashMap(java.util.HashMap) StopContainersRequest(org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest) StopContainersResponse(org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse) Test(org.junit.Test)

Example 22 with GetContainerStatusesRequest

use of org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest in project hadoop by apache.

the class TestContainerSchedulerQueuing method testKillMultipleOpportunisticContainers.

/**
   * Submit three OPPORTUNISTIC containers that can run concurrently, and one
   * GUARANTEED that needs to kill two of the OPPORTUNISTIC for it to run.
   * @throws Exception
   */
@Test
public void testKillMultipleOpportunisticContainers() throws Exception {
    containerManager.start();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(3), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(1500, 1), context.getContainerTokenSecretManager(), null, ExecutionType.GUARANTEED)));
    allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    BaseContainerManagerTest.waitForNMContainerState(containerManager, createContainerId(0), Arrays.asList(ContainerState.DONE, ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL), 40);
    Thread.sleep(5000);
    // Get container statuses. Container 0 should be killed, container 1
    // should be queued and container 2 should be running.
    int killedContainers = 0;
    List<ContainerId> statList = new ArrayList<ContainerId>();
    for (int i = 0; i < 4; i++) {
        statList.add(createContainerId(i));
    }
    GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.newInstance(statList);
    List<ContainerStatus> containerStatuses = containerManager.getContainerStatuses(statRequest).getContainerStatuses();
    for (ContainerStatus status : containerStatuses) {
        if (status.getDiagnostics().contains("Container Killed to make room for Guaranteed Container")) {
            killedContainers++;
        }
        System.out.println("\nStatus : [" + status + "]\n");
    }
    Assert.assertEquals(2, killedContainers);
}
Also used : StartContainersRequest(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest) ContainerStatus(org.apache.hadoop.yarn.api.records.ContainerStatus) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) GetContainerStatusesRequest(org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest) ArrayList(java.util.ArrayList) ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) StartContainerRequest(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest) BaseContainerManagerTest(org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest) Test(org.junit.Test)

Example 23 with GetContainerStatusesRequest

use of org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest in project hadoop by apache.

the class TestContainerSchedulerQueuing method testStartAndQueueMultipleContainers.

/**
   * Starts one OPPORTUNISTIC container that takes up the whole node's
   * resources, and submit two more that will be queued.
   * @throws Exception
   */
@Test
public void testStartAndQueueMultipleContainers() throws Exception {
    containerManager.start();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(1024, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(1024, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    Thread.sleep(5000);
    // Ensure first container is running and others are queued.
    List<ContainerId> statList = new ArrayList<ContainerId>();
    for (int i = 0; i < 3; i++) {
        statList.add(createContainerId(i));
    }
    GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.newInstance(Arrays.asList(createContainerId(0)));
    List<ContainerStatus> containerStatuses = containerManager.getContainerStatuses(statRequest).getContainerStatuses();
    for (ContainerStatus status : containerStatuses) {
        if (status.getContainerId().equals(createContainerId(0))) {
            Assert.assertEquals(org.apache.hadoop.yarn.api.records.ContainerState.RUNNING, status.getState());
        } else {
            Assert.assertEquals(org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED, status.getState());
        }
    }
    ContainerScheduler containerScheduler = containerManager.getContainerScheduler();
    // Ensure two containers are properly queued.
    Assert.assertEquals(2, containerScheduler.getNumQueuedContainers());
    Assert.assertEquals(0, containerScheduler.getNumQueuedGuaranteedContainers());
    Assert.assertEquals(2, containerScheduler.getNumQueuedOpportunisticContainers());
}
Also used : StartContainersRequest(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest) ContainerStatus(org.apache.hadoop.yarn.api.records.ContainerStatus) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) GetContainerStatusesRequest(org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest) ArrayList(java.util.ArrayList) ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) StartContainerRequest(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest) BaseContainerManagerTest(org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest) Test(org.junit.Test)

Example 24 with GetContainerStatusesRequest

use of org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest in project hadoop by apache.

the class TestContainerSchedulerQueuing method testStopQueuedContainer.

/**
   * Start running one GUARANTEED container and queue two OPPORTUNISTIC ones.
   * Try killing one of the two queued containers.
   * @throws Exception
   */
@Test
public void testStopQueuedContainer() throws Exception {
    containerManager.start();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), context.getContainerTokenSecretManager(), null, ExecutionType.GUARANTEED)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    Thread.sleep(2000);
    // Assert there is initially one container running and two queued.
    int runningContainersNo = 0;
    int queuedContainersNo = 0;
    List<ContainerId> statList = new ArrayList<ContainerId>();
    for (int i = 0; i < 3; i++) {
        statList.add(createContainerId(i));
    }
    GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.newInstance(statList);
    List<ContainerStatus> containerStatuses = containerManager.getContainerStatuses(statRequest).getContainerStatuses();
    for (ContainerStatus status : containerStatuses) {
        if (status.getState() == org.apache.hadoop.yarn.api.records.ContainerState.RUNNING) {
            runningContainersNo++;
        } else if (status.getState() == org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED) {
            queuedContainersNo++;
        }
        System.out.println("\nStatus : [" + status + "]\n");
    }
    Assert.assertEquals(1, runningContainersNo);
    Assert.assertEquals(2, queuedContainersNo);
    // Stop one of the two queued containers.
    StopContainersRequest stopRequest = StopContainersRequest.newInstance(Arrays.asList(createContainerId(1)));
    containerManager.stopContainers(stopRequest);
    Thread.sleep(2000);
    // Assert queued container got properly stopped.
    statList.clear();
    for (int i = 0; i < 3; i++) {
        statList.add(createContainerId(i));
    }
    statRequest = GetContainerStatusesRequest.newInstance(statList);
    HashMap<org.apache.hadoop.yarn.api.records.ContainerState, ContainerStatus> map = new HashMap<>();
    for (int i = 0; i < 10; i++) {
        containerStatuses = containerManager.getContainerStatuses(statRequest).getContainerStatuses();
        for (ContainerStatus status : containerStatuses) {
            System.out.println("\nStatus : [" + status + "]\n");
            map.put(status.getState(), status);
            if (map.containsKey(org.apache.hadoop.yarn.api.records.ContainerState.RUNNING) && map.containsKey(org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED) && map.containsKey(org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE)) {
                break;
            }
            Thread.sleep(1000);
        }
    }
    Assert.assertEquals(createContainerId(0), map.get(org.apache.hadoop.yarn.api.records.ContainerState.RUNNING).getContainerId());
    Assert.assertEquals(createContainerId(1), map.get(org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE).getContainerId());
    Assert.assertEquals(createContainerId(2), map.get(org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED).getContainerId());
}
Also used : StartContainersRequest(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest) GetContainerStatusesRequest(org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) ContainerState(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState) StartContainerRequest(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest) ContainerStatus(org.apache.hadoop.yarn.api.records.ContainerStatus) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) StopContainersRequest(org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest) BaseContainerManagerTest(org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest) Test(org.junit.Test)

Aggregations

ArrayList (java.util.ArrayList)24 GetContainerStatusesRequest (org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest)24 ContainerId (org.apache.hadoop.yarn.api.records.ContainerId)24 StartContainerRequest (org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest)20 StartContainersRequest (org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)20 ContainerLaunchContext (org.apache.hadoop.yarn.api.records.ContainerLaunchContext)20 ContainerStatus (org.apache.hadoop.yarn.api.records.ContainerStatus)20 Test (org.junit.Test)16 HashMap (java.util.HashMap)11 BaseContainerManagerTest (org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest)11 Path (org.apache.hadoop.fs.Path)9 LocalResource (org.apache.hadoop.yarn.api.records.LocalResource)9 URL (org.apache.hadoop.yarn.api.records.URL)9 File (java.io.File)8 PrintWriter (java.io.PrintWriter)7 StopContainersRequest (org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest)7 Token (org.apache.hadoop.yarn.api.records.Token)6 BufferedReader (java.io.BufferedReader)5 FileReader (java.io.FileReader)5 GetContainerStatusesResponse (org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse)5