Examples with IncreaseContainersResourceRequest org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest used on opensource projects

Example 1 with IncreaseContainersResourceRequest

use of org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest in project hadoop by apache.

the class NMClientImpl method increaseContainerResource.

@Override
public void increaseContainerResource(Container container) throws YarnException, IOException {
    ContainerManagementProtocolProxyData proxy = null;
    try {
        proxy = cmProxy.getProxy(container.getNodeId().toString(), container.getId());
        List<Token> increaseTokens = new ArrayList<>();
        increaseTokens.add(container.getContainerToken());
        IncreaseContainersResourceRequest increaseRequest = IncreaseContainersResourceRequest.newInstance(increaseTokens);
        IncreaseContainersResourceResponse response = proxy.getContainerManagementProtocol().increaseContainersResource(increaseRequest);
        if (response.getFailedRequests() != null && response.getFailedRequests().containsKey(container.getId())) {
            Throwable t = response.getFailedRequests().get(container.getId()).deSerialize();
            parseAndThrowException(t);
        }
    } finally {
        if (proxy != null) {
            cmProxy.mayBeCloseProxy(proxy);
        }
    }
}

Example 2 with IncreaseContainersResourceRequest

use of org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest in project hadoop by apache.

the class TestContainerManager method testChangeContainerResource.

@Test
public void testChangeContainerResource() throws Exception {
    containerManager.start();
    File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
    PrintWriter fileWriter = new PrintWriter(scriptFile);
    // Construct the Container-id
    ContainerId cId = createContainerId(0);
    if (Shell.WINDOWS) {
        fileWriter.println("@ping -n 100 127.0.0.1 >nul");
    } else {
        fileWriter.write("\numask 0");
        fileWriter.write("\nexec sleep 100");
    }
    fileWriter.close();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    URL resource_alpha = URL.fromPath(localFS.makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
    rsrc_alpha.setResource(resource_alpha);
    rsrc_alpha.setSize(-1);
    rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
    rsrc_alpha.setType(LocalResourceType.FILE);
    rsrc_alpha.setTimestamp(scriptFile.lastModified());
    String destinationFile = "dest_file";
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(destinationFile, rsrc_alpha);
    containerLaunchContext.setLocalResources(localResources);
    List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
    containerLaunchContext.setCommands(commands);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    // Make sure the container reaches RUNNING state
    BaseContainerManagerTest.waitForNMContainerState(containerManager, cId, org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.RUNNING);
    // Construct container resource increase request,
    List<Token> increaseTokens = new ArrayList<>();
    // Add increase request.
    Resource targetResource = Resource.newInstance(4096, 2);
    Token containerToken = createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, targetResource, context.getContainerTokenSecretManager(), null);
    increaseTokens.add(containerToken);
    IncreaseContainersResourceRequest increaseRequest = IncreaseContainersResourceRequest.newInstance(increaseTokens);
    IncreaseContainersResourceResponse increaseResponse = containerManager.increaseContainersResource(increaseRequest);
    Assert.assertEquals(1, increaseResponse.getSuccessfullyIncreasedContainers().size());
    Assert.assertTrue(increaseResponse.getFailedRequests().isEmpty());
    // Check status
    List<ContainerId> containerIds = new ArrayList<>();
    containerIds.add(cId);
    GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.newInstance(containerIds);
    ContainerStatus containerStatus = containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
    // Check status immediately as resource increase is blocking
    assertEquals(targetResource, containerStatus.getCapability());
    // Simulate a decrease request
    List<org.apache.hadoop.yarn.api.records.Container> containersToDecrease = new ArrayList<>();
    targetResource = Resource.newInstance(2048, 2);
    org.apache.hadoop.yarn.api.records.Container decreasedContainer = org.apache.hadoop.yarn.api.records.Container.newInstance(cId, null, null, targetResource, null, null);
    containersToDecrease.add(decreasedContainer);
    containerManager.handle(new CMgrDecreaseContainersResourceEvent(containersToDecrease));
    // Check status with retry
    containerStatus = containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
    int retry = 0;
    while (!targetResource.equals(containerStatus.getCapability()) && (retry++ < 5)) {
        Thread.sleep(200);
        containerStatus = containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
    }
    assertEquals(targetResource, containerStatus.getCapability());
}

Example 3 with IncreaseContainersResourceRequest

use of org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest in project hadoop by apache.

the class TestContainerManager method testIncreaseContainerResourceWithInvalidResource.

@Test
public void testIncreaseContainerResourceWithInvalidResource() throws Exception {
    containerManager.start();
    File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
    PrintWriter fileWriter = new PrintWriter(scriptFile);
    // Construct the Container-id
    ContainerId cId = createContainerId(0);
    if (Shell.WINDOWS) {
        fileWriter.println("@ping -n 100 127.0.0.1 >nul");
    } else {
        fileWriter.write("\numask 0");
        fileWriter.write("\nexec sleep 100");
    }
    fileWriter.close();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    URL resource_alpha = URL.fromPath(localFS.makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
    rsrc_alpha.setResource(resource_alpha);
    rsrc_alpha.setSize(-1);
    rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
    rsrc_alpha.setType(LocalResourceType.FILE);
    rsrc_alpha.setTimestamp(scriptFile.lastModified());
    String destinationFile = "dest_file";
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(destinationFile, rsrc_alpha);
    containerLaunchContext.setLocalResources(localResources);
    List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
    containerLaunchContext.setCommands(commands);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    // Make sure the container reaches RUNNING state
    BaseContainerManagerTest.waitForNMContainerState(containerManager, cId, org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.RUNNING);
    // Construct container resource increase request,
    List<Token> increaseTokens = new ArrayList<>();
    // Add increase request. The increase request should fail
    // as the current resource does not fit in the target resource
    Token containerToken = createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, Resource.newInstance(512, 1), context.getContainerTokenSecretManager(), null);
    increaseTokens.add(containerToken);
    IncreaseContainersResourceRequest increaseRequest = IncreaseContainersResourceRequest.newInstance(increaseTokens);
    IncreaseContainersResourceResponse increaseResponse = containerManager.increaseContainersResource(increaseRequest);
    // Check response
    Assert.assertEquals(0, increaseResponse.getSuccessfullyIncreasedContainers().size());
    Assert.assertEquals(1, increaseResponse.getFailedRequests().size());
    for (Map.Entry<ContainerId, SerializedException> entry : increaseResponse.getFailedRequests().entrySet()) {
        if (cId.equals(entry.getKey())) {
            Assert.assertNotNull("Failed message", entry.getValue().getMessage());
            Assert.assertTrue(entry.getValue().getMessage().contains("The target resource " + Resource.newInstance(512, 1).toString() + " is smaller than the current resource " + Resource.newInstance(1024, 1)));
        } else {
            throw new YarnException("Received failed request from wrong" + " container: " + entry.getKey().toString());
        }
    }
}

Example 4 with IncreaseContainersResourceRequest

use of org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest in project hadoop by apache.

the class TestContainerManager method testIncreaseContainerResourceWithInvalidRequests.

@Test
public void testIncreaseContainerResourceWithInvalidRequests() throws Exception {
    containerManager.start();
    // Start 4 containers 0..4 with default resource (1024, 1)
    List<StartContainerRequest> list = new ArrayList<>();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    for (int i = 0; i < 4; i++) {
        ContainerId cId = createContainerId(i);
        long identifier = DUMMY_RM_IDENTIFIER;
        Token containerToken = createContainerToken(cId, identifier, context.getNodeId(), user, context.getContainerTokenSecretManager());
        StartContainerRequest request = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
        list.add(request);
    }
    StartContainersRequest requestList = StartContainersRequest.newInstance(list);
    StartContainersResponse response = containerManager.startContainers(requestList);
    Assert.assertEquals(4, response.getSuccessfullyStartedContainers().size());
    int i = 0;
    for (ContainerId id : response.getSuccessfullyStartedContainers()) {
        Assert.assertEquals(i, id.getContainerId());
        i++;
    }
    Thread.sleep(2000);
    // Construct container resource increase request,
    List<Token> increaseTokens = new ArrayList<>();
    // Add increase request for container-0, the request will fail as the
    // container will have exited, and won't be in RUNNING state
    ContainerId cId0 = createContainerId(0);
    Token containerToken = createContainerToken(cId0, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, Resource.newInstance(1234, 3), context.getContainerTokenSecretManager(), null);
    increaseTokens.add(containerToken);
    // Add increase request for container-7, the request will fail as the
    // container does not exist
    ContainerId cId7 = createContainerId(7);
    containerToken = createContainerToken(cId7, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, Resource.newInstance(1234, 3), context.getContainerTokenSecretManager(), null);
    increaseTokens.add(containerToken);
    IncreaseContainersResourceRequest increaseRequest = IncreaseContainersResourceRequest.newInstance(increaseTokens);
    IncreaseContainersResourceResponse increaseResponse = containerManager.increaseContainersResource(increaseRequest);
    // Check response
    Assert.assertEquals(0, increaseResponse.getSuccessfullyIncreasedContainers().size());
    Assert.assertEquals(2, increaseResponse.getFailedRequests().size());
    for (Map.Entry<ContainerId, SerializedException> entry : increaseResponse.getFailedRequests().entrySet()) {
        Assert.assertNotNull("Failed message", entry.getValue().getMessage());
        if (cId0.equals(entry.getKey())) {
            Assert.assertTrue(entry.getValue().getMessage().contains("Resource can only be changed when a " + "container is in RUNNING state"));
        } else if (cId7.equals(entry.getKey())) {
            Assert.assertTrue(entry.getValue().getMessage().contains("Container " + cId7.toString() + " is not handled by this NodeManager"));
        } else {
            throw new YarnException("Received failed request from wrong" + " container: " + entry.getKey().toString());
        }
    }
}

Example 5 with IncreaseContainersResourceRequest

use of org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest in project hadoop by apache.

the class TestContainerManagerRecovery method increaseContainersResource.

private IncreaseContainersResourceResponse increaseContainersResource(Context context, final ContainerManagerImpl cm, ContainerId cid, Resource capability) throws Exception {
    UserGroupInformation user = UserGroupInformation.createRemoteUser(cid.getApplicationAttemptId().toString());
    // construct container resource increase request
    final List<Token> increaseTokens = new ArrayList<Token>();
    // add increase request
    Token containerToken = TestContainerManager.createContainerToken(cid, 0, context.getNodeId(), user.getShortUserName(), capability, context.getContainerTokenSecretManager(), null);
    increaseTokens.add(containerToken);
    final IncreaseContainersResourceRequest increaseRequest = IncreaseContainersResourceRequest.newInstance(increaseTokens);
    NMTokenIdentifier nmToken = new NMTokenIdentifier(cid.getApplicationAttemptId(), context.getNodeId(), user.getShortUserName(), context.getNMTokenSecretManager().getCurrentKey().getKeyId());
    user.addTokenIdentifier(nmToken);
    return user.doAs(new PrivilegedExceptionAction<IncreaseContainersResourceResponse>() {

        @Override
        public IncreaseContainersResourceResponse run() throws Exception {
            return cm.increaseContainersResource(increaseRequest);
        }
    });
}