Examples with InvalidResourceRequestException org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException used on opensource projects

Example 1 with InvalidResourceRequestException

use of org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException in project hadoop by apache.

the class RMServerUtils method checkSchedContainerChangeRequest.

/**
   * Validate increase/decrease request.
   * <pre>
   * - Throw exception when any other error happens
   * </pre>
   */
public static void checkSchedContainerChangeRequest(SchedContainerChangeRequest request, boolean increase) throws InvalidResourceRequestException {
    RMContext rmContext = request.getRmContext();
    ContainerId containerId = request.getContainerId();
    RMContainer rmContainer = request.getRMContainer();
    Resource targetResource = request.getTargetCapacity();
    // Compare targetResource and original resource
    Resource originalResource = rmContainer.getAllocatedResource();
    // <20G, 8>
    if (increase) {
        if (originalResource.getMemorySize() > targetResource.getMemorySize() || originalResource.getVirtualCores() > targetResource.getVirtualCores()) {
            String msg = "Trying to increase a container, but target resource has some" + " resource < original resource, target=" + targetResource + " original=" + originalResource + " containerId=" + containerId;
            throw new InvalidResourceRequestException(msg);
        }
    } else {
        if (originalResource.getMemorySize() < targetResource.getMemorySize() || originalResource.getVirtualCores() < targetResource.getVirtualCores()) {
            String msg = "Trying to decrease a container, but target resource has " + "some resource > original resource, target=" + targetResource + " original=" + originalResource + " containerId=" + containerId;
            throw new InvalidResourceRequestException(msg);
        }
    }
    // Target resource of the increase request is more than NM can offer
    ResourceScheduler scheduler = rmContext.getScheduler();
    RMNode rmNode = request.getSchedulerNode().getRMNode();
    if (!Resources.fitsIn(scheduler.getResourceCalculator(), scheduler.getClusterResource(), targetResource, rmNode.getTotalCapability())) {
        String msg = "Target resource=" + targetResource + " of containerId=" + containerId + " is more than node's total resource=" + rmNode.getTotalCapability();
        throw new InvalidResourceRequestException(msg);
    }
}

Example 2 with InvalidResourceRequestException

use of org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException in project hadoop by apache.

the class RMAppManager method validateAndCreateResourceRequest.

private ResourceRequest validateAndCreateResourceRequest(ApplicationSubmissionContext submissionContext, boolean isRecovery) throws InvalidResourceRequestException {
    // Check whether AM resource requirements are within required limits
    if (!submissionContext.getUnmanagedAM()) {
        ResourceRequest amReq = submissionContext.getAMContainerResourceRequest();
        if (amReq == null) {
            amReq = BuilderUtils.newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1);
        }
        // set label expression for AM container
        if (null == amReq.getNodeLabelExpression()) {
            amReq.setNodeLabelExpression(submissionContext.getNodeLabelExpression());
        }
        try {
            SchedulerUtils.normalizeAndValidateRequest(amReq, scheduler.getMaximumResourceCapability(), submissionContext.getQueue(), scheduler, isRecovery, rmContext);
        } catch (InvalidResourceRequestException e) {
            LOG.warn("RM app submission failed in validating AM resource request" + " for application " + submissionContext.getApplicationId(), e);
            throw e;
        }
        amReq.setCapability(scheduler.getNormalizedResource(amReq.getCapability()));
        return amReq;
    }
    return null;
}

Example 3 with InvalidResourceRequestException

use of org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException in project hadoop by apache.

the class AbstractYarnScheduler method createSchedContainerChangeRequest.

/**
   * Sanity check increase/decrease request, and return
   * SchedulerContainerResourceChangeRequest according to given
   * UpdateContainerRequest.
   * 
   * <pre>
   * - Returns non-null value means validation succeeded
   * - Throw exception when any other error happens
   * </pre>
   */
private SchedContainerChangeRequest createSchedContainerChangeRequest(UpdateContainerRequest request, boolean increase) throws YarnException {
    ContainerId containerId = request.getContainerId();
    RMContainer rmContainer = getRMContainer(containerId);
    if (null == rmContainer) {
        String msg = "Failed to get rmContainer for " + (increase ? "increase" : "decrease") + " request, with container-id=" + containerId;
        throw new InvalidResourceRequestException(msg);
    }
    SchedulerNode schedulerNode = getSchedulerNode(rmContainer.getAllocatedNode());
    return new SchedContainerChangeRequest(this.rmContext, schedulerNode, rmContainer, request.getCapability());
}

Example 4 with InvalidResourceRequestException

use of org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException in project hadoop by apache.

the class TestSchedulerUtils method testValidateResourceRequest.

@Test(timeout = 30000)
public void testValidateResourceRequest() {
    YarnScheduler mockScheduler = mock(YarnScheduler.class);
    Resource maxResource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
    // zero memory
    try {
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, mockScheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        fail("Zero memory should be accepted");
    }
    // zero vcores
    try {
        Resource resource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, mockScheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        fail("Zero vcores should be accepted");
    }
    // max memory
    try {
        Resource resource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, mockScheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        fail("Max memory should be accepted");
    }
    // max vcores
    try {
        Resource resource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, mockScheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        fail("Max vcores should not be accepted");
    }
    // negative memory
    try {
        Resource resource = Resources.createResource(-1, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, mockScheduler, rmContext);
        fail("Negative memory should not be accepted");
    } catch (InvalidResourceRequestException e) {
    // expected
    }
    // negative vcores
    try {
        Resource resource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, -1);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, mockScheduler, rmContext);
        fail("Negative vcores should not be accepted");
    } catch (InvalidResourceRequestException e) {
    // expected
    }
    // more than max memory
    try {
        Resource resource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB + 1, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, mockScheduler, rmContext);
        fail("More than max memory should not be accepted");
    } catch (InvalidResourceRequestException e) {
    // expected
    }
    // more than max vcores
    try {
        Resource resource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + 1);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null, mockScheduler, rmContext);
        fail("More than max vcores should not be accepted");
    } catch (InvalidResourceRequestException e) {
    // expected
    }
}

Example 5 with InvalidResourceRequestException

use of org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException in project hadoop by apache.

the class TestSchedulerUtils method testNormalizeNodeLabelExpression.

@Test(timeout = 30000)
public void testNormalizeNodeLabelExpression() throws IOException {
    // mock queue and scheduler
    YarnScheduler scheduler = mock(YarnScheduler.class);
    Set<String> queueAccessibleNodeLabels = Sets.newHashSet();
    QueueInfo queueInfo = mock(QueueInfo.class);
    when(queueInfo.getQueueName()).thenReturn("queue");
    when(queueInfo.getAccessibleNodeLabels()).thenReturn(queueAccessibleNodeLabels);
    when(queueInfo.getDefaultNodeLabelExpression()).thenReturn(" x ");
    when(scheduler.getQueueInfo(any(String.class), anyBoolean(), anyBoolean())).thenReturn(queueInfo);
    Resource maxResource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
    // queue has labels, success cases
    try {
        // set queue accessible node labels to [x, y]
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
        rmContext.getNodeLabelManager().addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y")));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        Assert.assertTrue(resReq.getNodeLabelExpression().equals("x"));
        resReq.setNodeLabelExpression(" y ");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        Assert.assertTrue(resReq.getNodeLabelExpression().equals("y"));
    } catch (InvalidResourceRequestException e) {
        e.printStackTrace();
        fail("Should be valid when request labels is a subset of queue labels");
    } finally {
        rmContext.getNodeLabelManager().removeFromClusterNodeLabels(Arrays.asList("x", "y"));
    }
}