Examples with ResourceRequest org.apache.hadoop.yarn.api.records.ResourceRequest used on opensource projects

Example 26 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class TestSchedulerUtils method testNormalizeRequestWithDominantResourceCalculator.

@Test(timeout = 30000)
public void testNormalizeRequestWithDominantResourceCalculator() {
    ResourceCalculator resourceCalculator = new DominantResourceCalculator();
    Resource minResource = Resources.createResource(1024, 1);
    Resource maxResource = Resources.createResource(10240, 10);
    Resource clusterResource = Resources.createResource(10 * 1024, 10);
    ResourceRequest ask = new ResourceRequestPBImpl();
    // case negative memory/vcores
    ask.setCapability(Resources.createResource(-1024, -1));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(minResource, ask.getCapability());
    // case zero memory/vcores
    ask.setCapability(Resources.createResource(0, 0));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(minResource, ask.getCapability());
    assertEquals(1, ask.getCapability().getVirtualCores());
    assertEquals(1024, ask.getCapability().getMemorySize());
    // case non-zero memory & zero cores
    ask.setCapability(Resources.createResource(1536, 0));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource, maxResource);
    assertEquals(Resources.createResource(2048, 1), ask.getCapability());
    assertEquals(1, ask.getCapability().getVirtualCores());
    assertEquals(2048, ask.getCapability().getMemorySize());
}

Example 27 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class TestSchedulerUtils method testValidateResourceRequest.

@Test(timeout = 30000)
public void testValidateResourceRequest() {
    YarnScheduler mockScheduler = mock(YarnScheduler.class);
    Resource maxResource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
    // zero memory
    try {
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, mockScheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        fail("Zero memory should be accepted");
    }
    // zero vcores
    try {
        Resource resource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, mockScheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        fail("Zero vcores should be accepted");
    }
    // max memory
    try {
        Resource resource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, mockScheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        fail("Max memory should be accepted");
    }
    // max vcores
    try {
        Resource resource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, mockScheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        fail("Max vcores should not be accepted");
    }
    // negative memory
    try {
        Resource resource = Resources.createResource(-1, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, mockScheduler, rmContext);
        fail("Negative memory should not be accepted");
    } catch (InvalidResourceRequestException e) {
    // expected
    }
    // negative vcores
    try {
        Resource resource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, -1);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, mockScheduler, rmContext);
        fail("Negative vcores should not be accepted");
    } catch (InvalidResourceRequestException e) {
    // expected
    }
    // more than max memory
    try {
        Resource resource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 1, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, mockScheduler, rmContext);
        fail("More than max memory should not be accepted");
    } catch (InvalidResourceRequestException e) {
    // expected
    }
    // more than max vcores
    try {
        Resource resource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + 1);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, mockScheduler, rmContext);
        fail("More than max vcores should not be accepted");
    } catch (InvalidResourceRequestException e) {
    // expected
    }
}

Example 28 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class TestCapacityScheduler method testRecoverRequestAfterPreemption.

@Test(timeout = 300000)
public void testRecoverRequestAfterPreemption() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm1 = new MockRM(conf);
    rm1.start();
    MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000);
    RMApp app1 = rm1.submitApp(1024);
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
    // request a container.
    am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
    ContainerId containerId1 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
    rm1.waitForState(nm1, containerId1, RMContainerState.ALLOCATED);
    RMContainer rmContainer = cs.getRMContainer(containerId1);
    List<ResourceRequest> requests = rmContainer.getResourceRequests();
    FiCaSchedulerApp app = cs.getApplicationAttempt(am1.getApplicationAttemptId());
    FiCaSchedulerNode node = cs.getNode(rmContainer.getAllocatedNode());
    for (ResourceRequest request : requests) {
        // Skip the OffRack and RackLocal resource requests.
        if (request.getResourceName().equals(node.getRackName()) || request.getResourceName().equals(ResourceRequest.ANY)) {
            continue;
        }
        // Already the node local resource request is cleared from RM after
        // allocation.
        Assert.assertEquals(0, app.getOutstandingAsksCount(SchedulerRequestKey.create(request), request.getResourceName()));
    }
    // Call killContainer to preempt the container
    cs.markContainerForKillable(rmContainer);
    Assert.assertEquals(3, requests.size());
    for (ResourceRequest request : requests) {
        // Resource request must have added back in RM after preempt event
        // handling.
        Assert.assertEquals(1, app.getOutstandingAsksCount(SchedulerRequestKey.create(request), request.getResourceName()));
    }
    // New container will be allocated and will move to ALLOCATED state
    ContainerId containerId2 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
    rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);
    // allocate container
    List<Container> containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
    // Now with updated ResourceRequest, a container is allocated for AM.
    Assert.assertTrue(containers.size() == 1);
}

Example 29 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class TestRMWebServicesApps method verifyResourceRequestsGeneric.

public void verifyResourceRequestsGeneric(RMApp app, String nodeLabelExpression, int numContainers, boolean relaxLocality, int priority, String resourceName, long memory, long vCores, String executionType, boolean enforceExecutionType) {
    ResourceRequest request = app.getAMResourceRequest();
    assertEquals("nodeLabelExpression doesn't match", request.getNodeLabelExpression(), nodeLabelExpression);
    assertEquals("numContainers doesn't match", request.getNumContainers(), numContainers);
    assertEquals("relaxLocality doesn't match", request.getRelaxLocality(), relaxLocality);
    assertEquals("priority does not match", request.getPriority().getPriority(), priority);
    assertEquals("resourceName does not match", request.getResourceName(), resourceName);
    assertEquals("memory does not match", request.getCapability().getMemorySize(), memory);
    assertEquals("vCores does not match", request.getCapability().getVirtualCores(), vCores);
    assertEquals("executionType does not match", request.getExecutionTypeRequest().getExecutionType().name(), executionType);
    assertEquals("enforceExecutionType does not match", request.getExecutionTypeRequest().getEnforceExecutionType(), enforceExecutionType);
}

Example 30 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class BaseAMRMProxyE2ETest method createAllocateRequest.

protected AllocateRequest createAllocateRequest(List<NodeReport> listNode) {
    // The test needs AMRMClient to create a real allocate request
    AMRMClientImpl<AMRMClient.ContainerRequest> amClient = new AMRMClientImpl<>();
    Resource capability = Resource.newInstance(1024, 2);
    Priority priority = Priority.newInstance(1);
    List<NodeReport> nodeReports = listNode;
    String node = nodeReports.get(0).getNodeId().getHost();
    String[] nodes = new String[] { node };
    AMRMClient.ContainerRequest storedContainer1 = new AMRMClient.ContainerRequest(capability, nodes, null, priority);
    amClient.addContainerRequest(storedContainer1);
    amClient.addContainerRequest(storedContainer1);
    List<ResourceRequest> resourceAsk = new ArrayList<>();
    for (ResourceRequest rr : amClient.ask) {
        resourceAsk.add(rr);
    }
    ResourceBlacklistRequest resourceBlacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<>(), new ArrayList<>());
    int responseId = 1;
    return AllocateRequest.newInstance(responseId, 0, resourceAsk, new ArrayList<>(), resourceBlacklistRequest);
}