Examples with InvalidResourceRequestException org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException used on opensource projects

Example 6 with InvalidResourceRequestException

use of org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException in project hadoop by apache.

the class TestSchedulerUtils method testValidateResourceRequestWithErrorLabelsPermission.

@Test(timeout = 30000)
public void testValidateResourceRequestWithErrorLabelsPermission() throws IOException {
    // mock queue and scheduler
    YarnScheduler scheduler = mock(YarnScheduler.class);
    Set<String> queueAccessibleNodeLabels = Sets.newHashSet();
    QueueInfo queueInfo = mock(QueueInfo.class);
    when(queueInfo.getQueueName()).thenReturn("queue");
    when(queueInfo.getAccessibleNodeLabels()).thenReturn(queueAccessibleNodeLabels);
    when(scheduler.getQueueInfo(any(String.class), anyBoolean(), anyBoolean())).thenReturn(queueInfo);
    Resource maxResource = Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
    // queue has labels, success cases
    try {
        // set queue accessible node labesl to [x, y]
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
        rmContext.getNodeLabelManager().addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y")));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        resReq.setNodeLabelExpression("x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        resReq.setNodeLabelExpression("y");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        resReq.setNodeLabelExpression("");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        resReq.setNodeLabelExpression(" ");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        e.printStackTrace();
        fail("Should be valid when request labels is a subset of queue labels");
    } finally {
        rmContext.getNodeLabelManager().removeFromClusterNodeLabels(Arrays.asList("x", "y"));
    }
    // requested. should fail
    try {
        // set queue accessible node labesl to [x, y]
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        resReq.setNodeLabelExpression("x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        fail("Should fail");
    } catch (InvalidResourceRequestException e) {
    }
    // queue has labels, failed cases (when ask a label not included by queue)
    try {
        // set queue accessible node labesl to [x, y]
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
        rmContext.getNodeLabelManager().addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y")));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        resReq.setNodeLabelExpression("z");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        fail("Should fail");
    } catch (InvalidResourceRequestException e) {
    } finally {
        rmContext.getNodeLabelManager().removeFromClusterNodeLabels(Arrays.asList("x", "y"));
    }
    // now
    try {
        // set queue accessible node labesl to [x, y]
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
        rmContext.getNodeLabelManager().addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y")));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        resReq.setNodeLabelExpression("x && y");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        fail("Should fail");
    } catch (InvalidResourceRequestException e) {
    } finally {
        rmContext.getNodeLabelManager().removeFromClusterNodeLabels(Arrays.asList("x", "y"));
    }
    // queue doesn't have label, succeed (when request no label)
    queueAccessibleNodeLabels.clear();
    try {
        // set queue accessible node labels to empty
        queueAccessibleNodeLabels.clear();
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        resReq.setNodeLabelExpression("");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        resReq.setNodeLabelExpression("  ");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        e.printStackTrace();
        fail("Should be valid when request labels is empty");
    }
    boolean invalidlabelexception = false;
    // queue doesn't have label, failed (when request any label)
    try {
        // set queue accessible node labels to empty
        queueAccessibleNodeLabels.clear();
        rmContext.getNodeLabelManager().addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x")));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        resReq.setNodeLabelExpression("x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        fail("Should fail");
    } catch (InvalidLabelResourceRequestException e) {
        invalidlabelexception = true;
    } catch (InvalidResourceRequestException e) {
    } finally {
        rmContext.getNodeLabelManager().removeFromClusterNodeLabels(Arrays.asList("x"));
    }
    Assert.assertTrue("InvalidLabelResourceRequestException excpeted", invalidlabelexception);
    // queue is "*", always succeeded
    try {
        // set queue accessible node labels to empty
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.add(RMNodeLabelsManager.ANY);
        rmContext.getNodeLabelManager().addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y"), NodeLabel.newInstance("z")));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        resReq.setNodeLabelExpression("x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        resReq.setNodeLabelExpression("y");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        resReq.setNodeLabelExpression("z");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        e.printStackTrace();
        fail("Should be valid when queue can access any labels");
    } finally {
        rmContext.getNodeLabelManager().removeFromClusterNodeLabels(Arrays.asList("x", "y", "z"));
    }
    // same as above, but cluster node labels don't contains label, should fail
    try {
        // set queue accessible node labels to empty
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.add(RMNodeLabelsManager.ANY);
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), ResourceRequest.ANY, resource, 1);
        resReq.setNodeLabelExpression("x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        fail("Should fail");
    } catch (InvalidResourceRequestException e) {
    }
    // we don't allow resource name other than ANY and specify label
    try {
        // set queue accessible node labesl to [x, y]
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
        rmContext.getNodeLabelManager().addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x"), NodeLabel.newInstance("y")));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), "rack", resource, 1);
        resReq.setNodeLabelExpression("x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        fail("Should fail");
    } catch (InvalidResourceRequestException e) {
    } finally {
        rmContext.getNodeLabelManager().removeFromClusterNodeLabels(Arrays.asList("x", "y"));
    }
    // queue has accessible label = *
    try {
        // set queue accessible node labesl to *
        queueAccessibleNodeLabels.clear();
        queueAccessibleNodeLabels.addAll(Arrays.asList(CommonNodeLabelsManager.ANY));
        rmContext.getNodeLabelManager().addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x")));
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq = BuilderUtils.newResourceRequest(mock(Priority.class), "rack", resource, 1);
        resReq.setNodeLabelExpression("x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue", scheduler, rmContext);
        fail("Should fail");
    } catch (InvalidResourceRequestException e) {
    } finally {
        rmContext.getNodeLabelManager().removeFromClusterNodeLabels(Arrays.asList("x"));
    }
    try {
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq1 = BuilderUtils.newResourceRequest(mock(Priority.class), "*", resource, 1, "x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq1, maxResource, "queue", scheduler, rmContext);
        fail("Should fail");
    } catch (InvalidResourceRequestException e) {
        assertEquals("Invalid label resource request, cluster do not contain , " + "label= x", e.getMessage());
    }
    try {
        rmContext.getYarnConfiguration().set(YarnConfiguration.NODE_LABELS_ENABLED, "false");
        Resource resource = Resources.createResource(0, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
        ResourceRequest resReq1 = BuilderUtils.newResourceRequest(mock(Priority.class), "*", resource, 1, "x");
        SchedulerUtils.normalizeAndvalidateRequest(resReq1, maxResource, "queue", scheduler, rmContext);
        Assert.assertEquals(RMNodeLabelsManager.NO_LABEL, resReq1.getNodeLabelExpression());
    } catch (InvalidResourceRequestException e) {
        assertEquals("Invalid resource request, node label not enabled but " + "request contains label expression", e.getMessage());
    }
}

Example 7 with InvalidResourceRequestException

use of org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException in project hadoop by apache.

the class ApplicationMasterService method allocateInternal.

protected void allocateInternal(ApplicationAttemptId appAttemptId, AllocateRequest request, AllocateResponse allocateResponse) throws YarnException {
    //filter illegal progress values
    float filteredProgress = request.getProgress();
    if (Float.isNaN(filteredProgress) || filteredProgress == Float.NEGATIVE_INFINITY || filteredProgress < 0) {
        request.setProgress(0);
    } else if (filteredProgress > 1 || filteredProgress == Float.POSITIVE_INFINITY) {
        request.setProgress(1);
    }
    // Send the status update to the appAttempt.
    this.rmContext.getDispatcher().getEventHandler().handle(new RMAppAttemptStatusupdateEvent(appAttemptId, request.getProgress()));
    List<ResourceRequest> ask = request.getAskList();
    List<ContainerId> release = request.getReleaseList();
    ResourceBlacklistRequest blacklistRequest = request.getResourceBlacklistRequest();
    List<String> blacklistAdditions = (blacklistRequest != null) ? blacklistRequest.getBlacklistAdditions() : Collections.EMPTY_LIST;
    List<String> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.getBlacklistRemovals() : Collections.EMPTY_LIST;
    RMApp app = this.rmContext.getRMApps().get(appAttemptId.getApplicationId());
    // set label expression for Resource Requests if resourceName=ANY
    ApplicationSubmissionContext asc = app.getApplicationSubmissionContext();
    for (ResourceRequest req : ask) {
        if (null == req.getNodeLabelExpression() && ResourceRequest.ANY.equals(req.getResourceName())) {
            req.setNodeLabelExpression(asc.getNodeLabelExpression());
        }
    }
    Resource maximumCapacity = rScheduler.getMaximumResourceCapability();
    // sanity check
    try {
        RMServerUtils.normalizeAndValidateRequests(ask, maximumCapacity, app.getQueue(), rScheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        LOG.warn("Invalid resource ask by application " + appAttemptId, e);
        throw e;
    }
    try {
        RMServerUtils.validateBlacklistRequest(blacklistRequest);
    } catch (InvalidResourceBlacklistRequestException e) {
        LOG.warn("Invalid blacklist request by application " + appAttemptId, e);
        throw e;
    }
    // AM to release containers from the earlier attempt.
    if (!app.getApplicationSubmissionContext().getKeepContainersAcrossApplicationAttempts()) {
        try {
            RMServerUtils.validateContainerReleaseRequest(release, appAttemptId);
        } catch (InvalidContainerReleaseException e) {
            LOG.warn("Invalid container release by application " + appAttemptId, e);
            throw e;
        }
    }
    // Split Update Resource Requests into increase and decrease.
    // No Exceptions are thrown here. All update errors are aggregated
    // and returned to the AM.
    List<UpdateContainerError> updateErrors = new ArrayList<>();
    ContainerUpdates containerUpdateRequests = RMServerUtils.validateAndSplitUpdateResourceRequests(rmContext, request, maximumCapacity, updateErrors);
    // Send new requests to appAttempt.
    Allocation allocation;
    RMAppAttemptState state = app.getRMAppAttempt(appAttemptId).getAppAttemptState();
    if (state.equals(RMAppAttemptState.FINAL_SAVING) || state.equals(RMAppAttemptState.FINISHING) || app.isAppFinalStateStored()) {
        LOG.warn(appAttemptId + " is in " + state + " state, ignore container allocate request.");
        allocation = EMPTY_ALLOCATION;
    } else {
        allocation = this.rScheduler.allocate(appAttemptId, ask, release, blacklistAdditions, blacklistRemovals, containerUpdateRequests);
    }
    if (!blacklistAdditions.isEmpty() || !blacklistRemovals.isEmpty()) {
        LOG.info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions + ", " + "blacklistRemovals: " + blacklistRemovals);
    }
    RMAppAttempt appAttempt = app.getRMAppAttempt(appAttemptId);
    if (allocation.getNMTokens() != null && !allocation.getNMTokens().isEmpty()) {
        allocateResponse.setNMTokens(allocation.getNMTokens());
    }
    // Notify the AM of container update errors
    addToUpdateContainerErrors(allocateResponse, updateErrors);
    // update the response with the deltas of node status changes
    List<RMNode> updatedNodes = new ArrayList<RMNode>();
    if (app.pullRMNodeUpdates(updatedNodes) > 0) {
        List<NodeReport> updatedNodeReports = new ArrayList<NodeReport>();
        for (RMNode rmNode : updatedNodes) {
            SchedulerNodeReport schedulerNodeReport = rScheduler.getNodeReport(rmNode.getNodeID());
            Resource used = BuilderUtils.newResource(0, 0);
            int numContainers = 0;
            if (schedulerNodeReport != null) {
                used = schedulerNodeReport.getUsedResource();
                numContainers = schedulerNodeReport.getNumContainers();
            }
            NodeId nodeId = rmNode.getNodeID();
            NodeReport report = BuilderUtils.newNodeReport(nodeId, rmNode.getState(), rmNode.getHttpAddress(), rmNode.getRackName(), used, rmNode.getTotalCapability(), numContainers, rmNode.getHealthReport(), rmNode.getLastHealthReportTime(), rmNode.getNodeLabels());
            updatedNodeReports.add(report);
        }
        allocateResponse.setUpdatedNodes(updatedNodeReports);
    }
    addToAllocatedContainers(allocateResponse, allocation.getContainers());
    allocateResponse.setCompletedContainersStatuses(appAttempt.pullJustFinishedContainers());
    allocateResponse.setAvailableResources(allocation.getResourceLimit());
    addToContainerUpdates(appAttemptId, allocateResponse, allocation);
    allocateResponse.setNumClusterNodes(this.rScheduler.getNumClusterNodes());
    // add collector address for this application
    if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
        allocateResponse.setCollectorAddr(this.rmContext.getRMApps().get(appAttemptId.getApplicationId()).getCollectorAddr());
    }
    // add preemption to the allocateResponse message (if any)
    allocateResponse.setPreemptionMessage(generatePreemptionMessage(allocation));
    // Set application priority
    allocateResponse.setApplicationPriority(app.getApplicationPriority());
}