use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.
the class TestContainerManagerSecurity method startContainer.
private void startContainer(final YarnRPC rpc, org.apache.hadoop.yarn.api.records.Token nmToken, org.apache.hadoop.yarn.api.records.Token containerToken, NodeId nodeId, String user) throws Exception {
ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
StartContainerRequest scRequest = StartContainerRequest.newInstance(context, containerToken);
List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
list.add(scRequest);
StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
ContainerManagementProtocol proxy = null;
try {
proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
StartContainersResponse response = proxy.startContainers(allRequests);
for (SerializedException ex : response.getFailedRequests().values()) {
parseAndThrowException(ex.deSerialize());
}
} finally {
if (proxy != null) {
rpc.stopProxy(proxy, conf);
}
}
}
use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.
the class AMLauncher method launch.
private void launch() throws IOException, YarnException {
connect();
ContainerId masterContainerID = masterContainer.getId();
ApplicationSubmissionContext applicationContext = application.getSubmissionContext();
LOG.info("Setting up container " + masterContainer + " for AM " + application.getAppAttemptId());
ContainerLaunchContext launchContext = createAMContainerLaunchContext(applicationContext, masterContainerID);
StartContainerRequest scRequest = StartContainerRequest.newInstance(launchContext, masterContainer.getContainerToken());
List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
list.add(scRequest);
StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
StartContainersResponse response = containerMgrProxy.startContainers(allRequests);
if (response.getFailedRequests() != null && response.getFailedRequests().containsKey(masterContainerID)) {
Throwable t = response.getFailedRequests().get(masterContainerID).deSerialize();
parseAndThrowException(t);
} else {
LOG.info("Done launching container " + masterContainer + " for AM " + application.getAppAttemptId());
}
}
use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.
the class TestContainerSchedulerQueuing method testQueueMultipleContainers.
/**
* Submit both a GUARANTEED and an OPPORTUNISTIC container, each of which
* requires more resources than available at the node, and make sure they
* are both queued.
* @throws Exception
*/
@Test
public void testQueueMultipleContainers() throws Exception {
containerManager.start();
ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
List<StartContainerRequest> list = new ArrayList<>();
list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(3072, 1), context.getContainerTokenSecretManager(), null, ExecutionType.GUARANTEED)));
list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(3072, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
containerManager.startContainers(allRequests);
Thread.sleep(5000);
// Ensure both containers are queued.
List<ContainerId> statList = new ArrayList<ContainerId>();
for (int i = 0; i < 2; i++) {
statList.add(createContainerId(i));
}
GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.newInstance(statList);
List<ContainerStatus> containerStatuses = containerManager.getContainerStatuses(statRequest).getContainerStatuses();
for (ContainerStatus status : containerStatuses) {
Assert.assertEquals(org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED, status.getState());
}
ContainerScheduler containerScheduler = containerManager.getContainerScheduler();
// Ensure both containers are properly queued.
Assert.assertEquals(2, containerScheduler.getNumQueuedContainers());
Assert.assertEquals(1, containerScheduler.getNumQueuedGuaranteedContainers());
Assert.assertEquals(1, containerScheduler.getNumQueuedOpportunisticContainers());
}
use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.
the class TestContainerSchedulerQueuing method testContainerDeQueuedAfterAMKill.
/**
* 1. Submit a long running GUARANTEED container to hog all NM resources.
* 2. Submit 2 OPPORTUNISTIC containers, both of which will be queued.
* 3. Send Stop Container to one of the queued containers.
* 4. Ensure container is removed from the queue.
* @throws Exception
*/
@Test
public void testContainerDeQueuedAfterAMKill() throws Exception {
containerManager.start();
ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
containerLaunchContext.setCommands(Arrays.asList("sleep 100"));
List<StartContainerRequest> list = new ArrayList<>();
list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(2048, 1), context.getContainerTokenSecretManager(), null, ExecutionType.GUARANTEED)));
StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
containerManager.startContainers(allRequests);
list = new ArrayList<>();
list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(2), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(512, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
allRequests = StartContainersRequest.newInstance(list);
containerManager.startContainers(allRequests);
ContainerScheduler containerScheduler = containerManager.getContainerScheduler();
// Ensure both containers are properly queued.
int numTries = 30;
while ((containerScheduler.getNumQueuedContainers() < 2) && (numTries-- > 0)) {
Thread.sleep(100);
}
Assert.assertEquals(2, containerScheduler.getNumQueuedContainers());
containerManager.stopContainers(StopContainersRequest.newInstance(Arrays.asList(createContainerId(2))));
numTries = 30;
while ((containerScheduler.getNumQueuedContainers() > 1) && (numTries-- > 0)) {
Thread.sleep(100);
}
Assert.assertEquals(1, containerScheduler.getNumQueuedContainers());
}
use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.
the class TestContainerSchedulerQueuing method testStartMultipleContainers.
/**
* Starting one GUARANTEED and one OPPORTUNISTIC container.
* @throws Exception
*/
@Test
public void testStartMultipleContainers() throws Exception {
containerManager.start();
ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
List<StartContainerRequest> list = new ArrayList<>();
list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(1024, 1), context.getContainerTokenSecretManager(), null, ExecutionType.GUARANTEED)));
list.add(StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER, context.getNodeId(), user, BuilderUtils.newResource(1024, 1), context.getContainerTokenSecretManager(), null, ExecutionType.OPPORTUNISTIC)));
StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
containerManager.startContainers(allRequests);
BaseContainerManagerTest.waitForContainerState(containerManager, createContainerId(0), org.apache.hadoop.yarn.api.records.ContainerState.RUNNING);
BaseContainerManagerTest.waitForContainerState(containerManager, createContainerId(1), org.apache.hadoop.yarn.api.records.ContainerState.RUNNING);
// Ensure all containers are running.
List<ContainerId> statList = new ArrayList<ContainerId>();
for (int i = 0; i < 2; i++) {
statList.add(createContainerId(i));
}
GetContainerStatusesRequest statRequest = GetContainerStatusesRequest.newInstance(statList);
List<ContainerStatus> containerStatuses = containerManager.getContainerStatuses(statRequest).getContainerStatuses();
for (ContainerStatus status : containerStatuses) {
Assert.assertEquals(org.apache.hadoop.yarn.api.records.ContainerState.RUNNING, status.getState());
}
}
Aggregations