Examples with ContainerLaunchContext org.apache.hadoop.yarn.api.records.ContainerLaunchContext used on opensource projects

Example 26 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestNMLeveldbStateStoreService method createContainerRequest.

private StartContainerRequest createContainerRequest(ContainerId containerId) {
    LocalResource lrsrc = LocalResource.newInstance(URL.newInstance("hdfs", "somehost", 12345, "/some/path/to/rsrc"), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 123L, 1234567890L);
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put("rsrc", lrsrc);
    Map<String, String> env = new HashMap<String, String>();
    env.put("somevar", "someval");
    List<String> containerCmds = new ArrayList<String>();
    containerCmds.add("somecmd");
    containerCmds.add("somearg");
    Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
    serviceData.put("someservice", ByteBuffer.wrap(new byte[] { 0x1, 0x2, 0x3 }));
    ByteBuffer containerTokens = ByteBuffer.wrap(new byte[] { 0x7, 0x8, 0x9, 0xa });
    Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>();
    acls.put(ApplicationAccessType.VIEW_APP, "viewuser");
    acls.put(ApplicationAccessType.MODIFY_APP, "moduser");
    ContainerLaunchContext clc = ContainerLaunchContext.newInstance(localResources, env, containerCmds, serviceData, containerTokens, acls);
    Resource containerRsrc = Resource.newInstance(1357, 3);
    ContainerTokenIdentifier containerTokenId = new ContainerTokenIdentifier(containerId, "host", "user", containerRsrc, 9876543210L, 42, 2468, Priority.newInstance(7), 13579);
    Token containerToken = Token.newInstance(containerTokenId.getBytes(), ContainerTokenIdentifier.KIND.toString(), "password".getBytes(), "tokenservice");
    return StartContainerRequest.newInstance(clc, containerToken);
}

Example 27 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestContainerSchedulerQueuing method testQueueMultipleContainers.

/**
   * Submit both a GUARANTEED and an OPPORTUNISTIC container, each of which
   * requires more resources than available at the node, and make sure they
   * are both queued.
   * @throws Exception
   */
@Test
public void testQueueMultipleContainers() throws Exception {
    containerManager.start();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(3072, 1), context.getContainerTokenSecretManager(), null, ExecutionType.GUARANTEED)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(3072, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    Thread.sleep(5000);
    // Ensure both containers are queued.
    List<ContainerId> statList = new ArrayList<ContainerId>();
    for (int i = 0; i < 2; i++) {
        statList.add(createContainerId(i));
    }
    GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.newInstance(statList);
    List<ContainerStatus> containerStatuses = containerManager.getContainerStatuses(statRequest).getContainerStatuses();
    for (ContainerStatus status : containerStatuses) {
        Assert.assertEquals(org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED, status.getState());
    }
    ContainerScheduler containerScheduler = containerManager.getContainerScheduler();
    // Ensure both containers are properly queued.
    Assert.assertEquals(2, containerScheduler.getNumQueuedContainers());
    Assert.assertEquals(1, containerScheduler.getNumQueuedGuaranteedContainers());
    Assert.assertEquals(1, containerScheduler.getNumQueuedOpportunisticContainers());
}

Example 28 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestContainerSchedulerQueuing method testContainerDeQueuedAfterAMKill.

/**
   * 1. Submit a long running GUARANTEED container to hog all NM resources.
   * 2. Submit 2 OPPORTUNISTIC containers, both of which will be queued.
   * 3. Send Stop Container to one of the queued containers.
   * 4. Ensure container is removed from the queue.
   * @throws Exception
   */
@Test
public void testContainerDeQueuedAfterAMKill() throws Exception {
    containerManager.start();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    containerLaunchContext.setCommands(Arrays.asList("sleep 100"));
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), context.getContainerTokenSecretManager(), null, ExecutionType.GUARANTEED)));
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    ContainerScheduler containerScheduler = containerManager.getContainerScheduler();
    // Ensure both containers are properly queued.
    int numTries = 30;
    while ((containerScheduler.getNumQueuedContainers() < 2) && (numTries-- > 0)) {
        Thread.sleep(100);
    }
    Assert.assertEquals(2, containerScheduler.getNumQueuedContainers());
    containerManager.stopContainers(StopContainersRequest.newInstance(Arrays.asList(createContainerId(2))));
    numTries = 30;
    while ((containerScheduler.getNumQueuedContainers() > 1) && (numTries-- > 0)) {
        Thread.sleep(100);
    }
    Assert.assertEquals(1, containerScheduler.getNumQueuedContainers());
}

Example 29 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestContainerSchedulerQueuing method testStartMultipleContainers.

/**
   * Starting one GUARANTEED and one OPPORTUNISTIC container.
   * @throws Exception
   */
@Test
public void testStartMultipleContainers() throws Exception {
    containerManager.start();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(1024, 1), context.getContainerTokenSecretManager(), null, ExecutionType.GUARANTEED)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(1024, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    BaseContainerManagerTest.waitForContainerState(containerManager, createContainerId(0), org.apache.hadoop.yarn.api.records.ContainerState.RUNNING);
    BaseContainerManagerTest.waitForContainerState(containerManager, createContainerId(1), org.apache.hadoop.yarn.api.records.ContainerState.RUNNING);
    // Ensure all containers are running.
    List<ContainerId> statList = new ArrayList<ContainerId>();
    for (int i = 0; i < 2; i++) {
        statList.add(createContainerId(i));
    }
    GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.newInstance(statList);
    List<ContainerStatus> containerStatuses = containerManager.getContainerStatuses(statRequest).getContainerStatuses();
    for (ContainerStatus status : containerStatuses) {
        Assert.assertEquals(org.apache.hadoop.yarn.api.records.ContainerState.RUNNING, status.getState());
    }
}

Example 30 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestContainerSchedulerQueuing method testQueueShedding.

/**
   * 1. Submit a long running GUARANTEED container to hog all NM resources.
   * 2. Submit 6 OPPORTUNISTIC containers, all of which will be queued.
   * 3. Update the Queue Limit to 2.
   * 4. Ensure only 2 containers remain in the Queue, and 4 are de-Queued.
   * @throws Exception
   */
@Test
public void testQueueShedding() throws Exception {
    containerManager.start();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    containerLaunchContext.setCommands(Arrays.asList("sleep 100"));
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), context.getContainerTokenSecretManager(), null, ExecutionType.GUARANTEED)));
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    list = new ArrayList<>();
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(3), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(4), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(5), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(6), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
    allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    ContainerScheduler containerScheduler = containerManager.getContainerScheduler();
    // Ensure all containers are properly queued.
    int numTries = 30;
    while ((containerScheduler.getNumQueuedContainers() < 6) && (numTries-- > 0)) {
        Thread.sleep(100);
    }
    Assert.assertEquals(6, containerScheduler.getNumQueuedContainers());
    ContainerQueuingLimit containerQueuingLimit = ContainerQueuingLimit.newInstance();
    containerQueuingLimit.setMaxQueueLength(2);
    containerScheduler.updateQueuingLimit(containerQueuingLimit);
    numTries = 30;
    while ((containerScheduler.getNumQueuedContainers() > 2) && (numTries-- > 0)) {
        Thread.sleep(100);
    }
    Assert.assertEquals(2, containerScheduler.getNumQueuedContainers());
    List<ContainerId> statList = new ArrayList<ContainerId>();
    for (int i = 1; i < 7; i++) {
        statList.add(createContainerId(i));
    }
    GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.newInstance(statList);
    List<ContainerStatus> containerStatuses = containerManager.getContainerStatuses(statRequest).getContainerStatuses();
    int deQueuedContainers = 0;
    int numQueuedOppContainers = 0;
    for (ContainerStatus status : containerStatuses) {
        if (status.getExecutionType() == ExecutionType.OPPORTUNISTIC) {
            if (status.getDiagnostics().contains("Container De-queued to meet NM queuing limits")) {
                deQueuedContainers++;
            }
            if (status.getState() == org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED) {
                numQueuedOppContainers++;
            }
        }
    }
    Assert.assertEquals(4, deQueuedContainers);
    Assert.assertEquals(2, numQueuedOppContainers);
}