Examples with ResourceRequest org.apache.hadoop.yarn.api.records.ResourceRequest used on opensource projects

Example 31 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class TestYarnApiClasses method testResourceRequest.

/**
   * Simple test Resource request.
   * Test hashCode, equals and compare.
   */
@Test
public void testResourceRequest() {
    Resource resource = recordFactory.newRecordInstance(Resource.class);
    Priority priority = recordFactory.newRecordInstance(Priority.class);
    ResourceRequest original = ResourceRequest.newInstance(priority, "localhost", resource, 2);
    ResourceRequest copy = ResourceRequest.newInstance(priority, "localhost", resource, 2);
    assertTrue(original.equals(copy));
    assertEquals(0, original.compareTo(copy));
    assertTrue(original.hashCode() == copy.hashCode());
    copy.setNumContainers(1);
    assertFalse(original.equals(copy));
    assertNotSame(0, original.compareTo(copy));
    assertFalse(original.hashCode() == copy.hashCode());
}

Example 32 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class RMAppManager method recoverApplication.

protected void recoverApplication(ApplicationStateData appState, RMState rmState) throws Exception {
    ApplicationSubmissionContext appContext = appState.getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();
    // create and recover app.
    RMAppImpl application = createAndPopulateNewRMApp(appContext, appState.getSubmitTime(), appState.getUser(), true, appState.getStartTime());
    // is true and give clear message so that user can react properly.
    if (!appContext.getUnmanagedAM() && application.getAMResourceRequest() == null && !YarnConfiguration.areNodeLabelsEnabled(this.conf)) {
        // check application submission context and see if am resource request
        // or application itself contains any node label expression.
        ResourceRequest amReqFromAppContext = appContext.getAMContainerResourceRequest();
        String labelExp = (amReqFromAppContext != null) ? amReqFromAppContext.getNodeLabelExpression() : null;
        if (labelExp == null) {
            labelExp = appContext.getNodeLabelExpression();
        }
        if (labelExp != null && !labelExp.equals(RMNodeLabelsManager.NO_LABEL)) {
            String message = "Failed to recover application " + appId + ". NodeLabel is not enabled in cluster, but AM resource request " + "contains a label expression.";
            LOG.warn(message);
            application.handle(new RMAppEvent(appId, RMAppEventType.APP_REJECTED, message));
            return;
        }
    }
    application.handle(new RMAppRecoverEvent(appId, rmState));
}

Example 33 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class RMAppManager method createAndPopulateNewRMApp.

private RMAppImpl createAndPopulateNewRMApp(ApplicationSubmissionContext submissionContext, long submitTime, String user, boolean isRecovery, long startTime) throws YarnException {
    if (!isRecovery) {
        // Do queue mapping
        if (rmContext.getQueuePlacementManager() != null) {
            // We only do queue mapping when it's a new application
            rmContext.getQueuePlacementManager().placeApplication(submissionContext, user);
        }
        // fail the submission if configured application timeout value is invalid
        RMServerUtils.validateApplicationTimeouts(submissionContext.getApplicationTimeouts());
    }
    ApplicationId applicationId = submissionContext.getApplicationId();
    ResourceRequest amReq = null;
    try {
        amReq = validateAndCreateResourceRequest(submissionContext, isRecovery);
    } catch (InvalidLabelResourceRequestException e) {
        // after recovery and user can see what's going on and react accordingly.
        if (isRecovery && !YarnConfiguration.areNodeLabelsEnabled(this.conf)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("AMResourceRequest is not created for " + applicationId + ". NodeLabel is not enabled in cluster, but AM resource " + "request contains a label expression.");
            }
        } else {
            throw e;
        }
    }
    // Verify and get the update application priority and set back to
    // submissionContext
    UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user);
    Priority appPriority = scheduler.checkAndGetApplicationPriority(submissionContext.getPriority(), userUgi, submissionContext.getQueue(), applicationId);
    submissionContext.setPriority(appPriority);
    // For now, exclude FS for the acl check.
    if (!isRecovery && YarnConfiguration.isAclEnabled(conf) && scheduler instanceof CapacityScheduler) {
        String queueName = submissionContext.getQueue();
        String appName = submissionContext.getApplicationName();
        CSQueue csqueue = ((CapacityScheduler) scheduler).getQueue(queueName);
        if (null != csqueue && !authorizer.checkPermission(new AccessRequest(csqueue.getPrivilegedEntity(), userUgi, SchedulerUtils.toAccessType(QueueACL.SUBMIT_APPLICATIONS), applicationId.toString(), appName, Server.getRemoteAddress(), null)) && !authorizer.checkPermission(new AccessRequest(csqueue.getPrivilegedEntity(), userUgi, SchedulerUtils.toAccessType(QueueACL.ADMINISTER_QUEUE), applicationId.toString(), appName, Server.getRemoteAddress(), null))) {
            throw RPCUtil.getRemoteException(new AccessControlException("User " + user + " does not have permission to submit " + applicationId + " to queue " + submissionContext.getQueue()));
        }
    }
    // Create RMApp
    RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf, submissionContext.getApplicationName(), user, submissionContext.getQueue(), submissionContext, this.scheduler, this.masterService, submitTime, submissionContext.getApplicationType(), submissionContext.getApplicationTags(), amReq, startTime);
    // influence each other
    if (rmContext.getRMApps().putIfAbsent(applicationId, application) != null) {
        String message = "Application with id " + applicationId + " is already present! Cannot add a duplicate!";
        LOG.warn(message);
        throw new YarnException(message);
    }
    if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
        // Start timeline collector for the submitted app
        application.startTimelineCollector();
    }
    // Inform the ACLs Manager
    this.applicationACLsManager.addApplication(applicationId, submissionContext.getAMContainerSpec().getApplicationACLs());
    String appViewACLs = submissionContext.getAMContainerSpec().getApplicationACLs().get(ApplicationAccessType.VIEW_APP);
    rmContext.getSystemMetricsPublisher().appACLsUpdated(application, appViewACLs, System.currentTimeMillis());
    return application;
}

Example 34 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class RMAppManager method validateAndCreateResourceRequest.

private ResourceRequest validateAndCreateResourceRequest(ApplicationSubmissionContext submissionContext, boolean isRecovery) throws InvalidResourceRequestException {
    // Check whether AM resource requirements are within required limits
    if (!submissionContext.getUnmanagedAM()) {
        ResourceRequest amReq = submissionContext.getAMContainerResourceRequest();
        if (amReq == null) {
            amReq = BuilderUtils.newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1);
        }
        // set label expression for AM container
        if (null == amReq.getNodeLabelExpression()) {
            amReq.setNodeLabelExpression(submissionContext.getNodeLabelExpression());
        }
        try {
            SchedulerUtils.normalizeAndValidateRequest(amReq, scheduler.getMaximumResourceCapability(), submissionContext.getQueue(), scheduler, isRecovery, rmContext);
        } catch (InvalidResourceRequestException e) {
            LOG.warn("RM app submission failed in validating AM resource request" + " for application " + submissionContext.getApplicationId(), e);
            throw e;
        }
        amReq.setCapability(scheduler.getNormalizedResource(amReq.getCapability()));
        return amReq;
    }
    return null;
}

Example 35 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class Application method addTask.

public synchronized void addTask(Task task) {
    SchedulerRequestKey schedulerKey = task.getSchedulerKey();
    Map<String, ResourceRequest> requests = this.requests.get(schedulerKey);
    if (requests == null) {
        requests = new HashMap<String, ResourceRequest>();
        this.requests.put(schedulerKey, requests);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Added priority=" + schedulerKey.getPriority() + " application=" + applicationId);
        }
    }
    final Resource capability = requestSpec.get(schedulerKey);
    // Note down the task
    Set<Task> tasks = this.tasks.get(schedulerKey);
    if (tasks == null) {
        tasks = new HashSet<Task>();
        this.tasks.put(schedulerKey, tasks);
    }
    tasks.add(task);
    LOG.info("Added task " + task.getTaskId() + " to application " + applicationId + " at priority " + schedulerKey.getPriority());
    if (LOG.isDebugEnabled()) {
        LOG.debug("addTask: application=" + applicationId + " #asks=" + ask.size());
    }
    // Create resource requests
    for (String host : task.getHosts()) {
        // Data-local
        addResourceRequest(schedulerKey, requests, host, capability);
    }
    // Rack-local
    for (String rack : task.getRacks()) {
        addResourceRequest(schedulerKey, requests, rack, capability);
    }
    // Off-switch
    addResourceRequest(schedulerKey, requests, ResourceRequest.ANY, capability);
}