Examples with AllocateRequest org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest used on opensource projects

Example 26 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestDistributedScheduling method testMixedExecutionTypeRequestE2E.

/**
   * Validates if Allocate Requests containing both GUARANTEED and OPPORTUNISTIC
   * container requests works as expected.
   *
   * @throws Exception
   */
@Test(timeout = 60000)
public void testMixedExecutionTypeRequestE2E() throws Exception {
    LOG.info("testDistributedSchedulingE2E - Register");
    RegisterApplicationMasterResponse responseRegister = client.registerApplicationMaster(RegisterApplicationMasterRequest.newInstance(NetUtils.getHostname(), 1024, ""));
    Assert.assertNotNull(responseRegister);
    Assert.assertNotNull(responseRegister.getQueue());
    Assert.assertNotNull(responseRegister.getApplicationACLs());
    Assert.assertNotNull(responseRegister.getClientToAMTokenMasterKey());
    Assert.assertNotNull(responseRegister.getContainersFromPreviousAttempts());
    Assert.assertNotNull(responseRegister.getSchedulerResourceTypes());
    Assert.assertNotNull(responseRegister.getMaximumResourceCapability());
    RMApp rmApp = cluster.getResourceManager().getRMContext().getRMApps().get(appId);
    Assert.assertEquals(RMAppState.RUNNING, rmApp.getState());
    LOG.info("testDistributedSchedulingE2E - Allocate");
    AllocateRequest request = createAllocateRequest(rmClient.getNodeReports(NodeState.RUNNING));
    List<ResourceRequest> askList = request.getAskList();
    List<ResourceRequest> newAskList = new ArrayList<>(askList);
    // Duplicate all ANY requests marking them as opportunistic
    for (ResourceRequest rr : askList) {
        if (ResourceRequest.ANY.equals(rr.getResourceName())) {
            ResourceRequest newRR = ResourceRequest.newInstance(rr.getPriority(), rr.getResourceName(), rr.getCapability(), rr.getNumContainers(), rr.getRelaxLocality(), rr.getNodeLabelExpression(), ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true));
            newAskList.add(newRR);
        }
    }
    request.setAskList(newAskList);
    AllocateResponse allocResponse = client.allocate(request);
    Assert.assertNotNull(allocResponse);
    // Ensure that all the requests are satisfied immediately
    Assert.assertEquals(2, allocResponse.getAllocatedContainers().size());
    // Verify that the allocated containers are OPPORTUNISTIC
    for (Container allocatedContainer : allocResponse.getAllocatedContainers()) {
        ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils.newContainerTokenIdentifier(allocatedContainer.getContainerToken());
        Assert.assertEquals(ExecutionType.OPPORTUNISTIC, containerTokenIdentifier.getExecutionType());
    }
    request.setAskList(new ArrayList<ResourceRequest>());
    request.setResponseId(request.getResponseId() + 1);
    Thread.sleep(1000);
    // RM should allocate GUARANTEED containers within 2 calls to allocate()
    allocResponse = client.allocate(request);
    Assert.assertNotNull(allocResponse);
    Assert.assertEquals(2, allocResponse.getAllocatedContainers().size());
    // Verify that the allocated containers are GUARANTEED
    for (Container allocatedContainer : allocResponse.getAllocatedContainers()) {
        ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils.newContainerTokenIdentifier(allocatedContainer.getContainerToken());
        Assert.assertEquals(ExecutionType.GUARANTEED, containerTokenIdentifier.getExecutionType());
    }
    LOG.info("testDistributedSchedulingE2E - Finish");
}

Example 27 with AllocateRequest

use of org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest in project hadoop by apache.

the class TestAMRMProxyService method releaseContainersAndAssert.

private void releaseContainersAndAssert(int appId, List<Container> containers) throws Exception {
    Assert.assertTrue(containers.size() > 0);
    AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
    allocateRequest.setResponseId(1);
    List<ContainerId> relList = new ArrayList<ContainerId>(containers.size());
    for (Container container : containers) {
        relList.add(container.getId());
    }
    allocateRequest.setReleaseList(relList);
    AllocateResponse allocateResponse = allocate(appId, allocateRequest);
    Assert.assertNotNull(allocateResponse);
    Assert.assertNull("new AMRMToken from RM should have been nulled by AMRMProxyService", allocateResponse.getAMRMToken());
    // The way the mock resource manager is setup, it will return the containers
    // that were released in the response. This is done because the UAMs run
    // asynchronously and we need to if all the resource managers received the
    // release it. The containers sent by the mock resource managers will be
    // aggregated and returned back to us and we can assert if all the release
    // lists reached the sub-clusters
    List<Container> containersForReleasedContainerIds = new ArrayList<Container>();
    containersForReleasedContainerIds.addAll(allocateResponse.getAllocatedContainers());
    // Send max 10 heart beats to receive all the containers. If not, we will
    // fail the test
    int numHeartbeat = 0;
    while (containersForReleasedContainerIds.size() < relList.size() && numHeartbeat++ < 10) {
        allocateResponse = allocate(appId, Records.newRecord(AllocateRequest.class));
        Assert.assertNotNull(allocateResponse);
        Assert.assertNull("new AMRMToken from RM should have been nulled by AMRMProxyService", allocateResponse.getAMRMToken());
        containersForReleasedContainerIds.addAll(allocateResponse.getAllocatedContainers());
        LOG.info("Number of containers received in this request: " + Integer.toString(allocateResponse.getAllocatedContainers().size()));
        LOG.info("Total number of containers received: " + Integer.toString(containersForReleasedContainerIds.size()));
        Thread.sleep(10);
    }
    Assert.assertEquals(relList.size(), containersForReleasedContainerIds.size());
}