Examples with ApplicationSubmissionContext org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext used on opensource projects

Example 61 with ApplicationSubmissionContext

use of org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext in project hadoop by apache.

the class ApplicationMasterService method allocateInternal.

protected void allocateInternal(ApplicationAttemptId appAttemptId, AllocateRequest request, AllocateResponse allocateResponse) throws YarnException {
    //filter illegal progress values
    float filteredProgress = request.getProgress();
    if (Float.isNaN(filteredProgress) || filteredProgress == Float.NEGATIVE_INFINITY || filteredProgress < 0) {
        request.setProgress(0);
    } else if (filteredProgress > 1 || filteredProgress == Float.POSITIVE_INFINITY) {
        request.setProgress(1);
    }
    // Send the status update to the appAttempt.
    this.rmContext.getDispatcher().getEventHandler().handle(new RMAppAttemptStatusupdateEvent(appAttemptId, request.getProgress()));
    List<ResourceRequest> ask = request.getAskList();
    List<ContainerId> release = request.getReleaseList();
    ResourceBlacklistRequest blacklistRequest = request.getResourceBlacklistRequest();
    List<String> blacklistAdditions = (blacklistRequest != null) ? blacklistRequest.getBlacklistAdditions() : Collections.EMPTY_LIST;
    List<String> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.getBlacklistRemovals() : Collections.EMPTY_LIST;
    RMApp app = this.rmContext.getRMApps().get(appAttemptId.getApplicationId());
    // set label expression for Resource Requests if resourceName=ANY
    ApplicationSubmissionContext asc = app.getApplicationSubmissionContext();
    for (ResourceRequest req : ask) {
        if (null == req.getNodeLabelExpression() && ResourceRequest.ANY.equals(req.getResourceName())) {
            req.setNodeLabelExpression(asc.getNodeLabelExpression());
        }
    }
    Resource maximumCapacity = rScheduler.getMaximumResourceCapability();
    // sanity check
    try {
        RMServerUtils.normalizeAndValidateRequests(ask, maximumCapacity, app.getQueue(), rScheduler, rmContext);
    } catch (InvalidResourceRequestException e) {
        LOG.warn("Invalid resource ask by application " + appAttemptId, e);
        throw e;
    }
    try {
        RMServerUtils.validateBlacklistRequest(blacklistRequest);
    } catch (InvalidResourceBlacklistRequestException e) {
        LOG.warn("Invalid blacklist request by application " + appAttemptId, e);
        throw e;
    }
    // AM to release containers from the earlier attempt.
    if (!app.getApplicationSubmissionContext().getKeepContainersAcrossApplicationAttempts()) {
        try {
            RMServerUtils.validateContainerReleaseRequest(release, appAttemptId);
        } catch (InvalidContainerReleaseException e) {
            LOG.warn("Invalid container release by application " + appAttemptId, e);
            throw e;
        }
    }
    // Split Update Resource Requests into increase and decrease.
    // No Exceptions are thrown here. All update errors are aggregated
    // and returned to the AM.
    List<UpdateContainerError> updateErrors = new ArrayList<>();
    ContainerUpdates containerUpdateRequests = RMServerUtils.validateAndSplitUpdateResourceRequests(rmContext, request, maximumCapacity, updateErrors);
    // Send new requests to appAttempt.
    Allocation allocation;
    RMAppAttemptState state = app.getRMAppAttempt(appAttemptId).getAppAttemptState();
    if (state.equals(RMAppAttemptState.FINAL_SAVING) || state.equals(RMAppAttemptState.FINISHING) || app.isAppFinalStateStored()) {
        LOG.warn(appAttemptId + " is in " + state + " state, ignore container allocate request.");
        allocation = EMPTY_ALLOCATION;
    } else {
        allocation = this.rScheduler.allocate(appAttemptId, ask, release, blacklistAdditions, blacklistRemovals, containerUpdateRequests);
    }
    if (!blacklistAdditions.isEmpty() || !blacklistRemovals.isEmpty()) {
        LOG.info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions + ", " + "blacklistRemovals: " + blacklistRemovals);
    }
    RMAppAttempt appAttempt = app.getRMAppAttempt(appAttemptId);
    if (allocation.getNMTokens() != null && !allocation.getNMTokens().isEmpty()) {
        allocateResponse.setNMTokens(allocation.getNMTokens());
    }
    // Notify the AM of container update errors
    addToUpdateContainerErrors(allocateResponse, updateErrors);
    // update the response with the deltas of node status changes
    List<RMNode> updatedNodes = new ArrayList<RMNode>();
    if (app.pullRMNodeUpdates(updatedNodes) > 0) {
        List<NodeReport> updatedNodeReports = new ArrayList<NodeReport>();
        for (RMNode rmNode : updatedNodes) {
            SchedulerNodeReport schedulerNodeReport = rScheduler.getNodeReport(rmNode.getNodeID());
            Resource used = BuilderUtils.newResource(0, 0);
            int numContainers = 0;
            if (schedulerNodeReport != null) {
                used = schedulerNodeReport.getUsedResource();
                numContainers = schedulerNodeReport.getNumContainers();
            }
            NodeId nodeId = rmNode.getNodeID();
            NodeReport report = BuilderUtils.newNodeReport(nodeId, rmNode.getState(), rmNode.getHttpAddress(), rmNode.getRackName(), used, rmNode.getTotalCapability(), numContainers, rmNode.getHealthReport(), rmNode.getLastHealthReportTime(), rmNode.getNodeLabels());
            updatedNodeReports.add(report);
        }
        allocateResponse.setUpdatedNodes(updatedNodeReports);
    }
    addToAllocatedContainers(allocateResponse, allocation.getContainers());
    allocateResponse.setCompletedContainersStatuses(appAttempt.pullJustFinishedContainers());
    allocateResponse.setAvailableResources(allocation.getResourceLimit());
    addToContainerUpdates(appAttemptId, allocateResponse, allocation);
    allocateResponse.setNumClusterNodes(this.rScheduler.getNumClusterNodes());
    // add collector address for this application
    if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
        allocateResponse.setCollectorAddr(this.rmContext.getRMApps().get(appAttemptId.getApplicationId()).getCollectorAddr());
    }
    // add preemption to the allocateResponse message (if any)
    allocateResponse.setPreemptionMessage(generatePreemptionMessage(allocation));
    // Set application priority
    allocateResponse.setApplicationPriority(app.getApplicationPriority());
}

Example 62 with ApplicationSubmissionContext

use of org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext in project hadoop by apache.

the class ClientRMService method submitApplication.

@Override
public SubmitApplicationResponse submitApplication(SubmitApplicationRequest request) throws YarnException, IOException {
    ApplicationSubmissionContext submissionContext = request.getApplicationSubmissionContext();
    ApplicationId applicationId = submissionContext.getApplicationId();
    CallerContext callerContext = CallerContext.getCurrent();
    // ApplicationSubmissionContext needs to be validated for safety - only
    // those fields that are independent of the RM's configuration will be
    // checked here, those that are dependent on RM configuration are validated
    // in RMAppManager.
    String user = null;
    try {
        // Safety
        user = UserGroupInformation.getCurrentUser().getShortUserName();
    } catch (IOException ie) {
        LOG.warn("Unable to get the current user.", ie);
        RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, ie.getMessage(), "ClientRMService", "Exception in submitting application", applicationId, callerContext);
        throw RPCUtil.getRemoteException(ie);
    }
    if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
        // Sanity check for flow run
        String value = null;
        try {
            for (String tag : submissionContext.getApplicationTags()) {
                if (tag.startsWith(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX + ":") || tag.startsWith(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX.toLowerCase() + ":")) {
                    value = tag.substring(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX.length() + 1);
                    Long.valueOf(value);
                }
            }
        } catch (NumberFormatException e) {
            LOG.warn("Invalid to flow run: " + value + ". Flow run should be a long integer", e);
            RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, e.getMessage(), "ClientRMService", "Exception in submitting application", applicationId);
            throw RPCUtil.getRemoteException(e);
        }
    }
    // If it is, simply return the response
    if (rmContext.getRMApps().get(applicationId) != null) {
        LOG.info("This is an earlier submitted application: " + applicationId);
        return SubmitApplicationResponse.newInstance();
    }
    ByteBuffer tokenConf = submissionContext.getAMContainerSpec().getTokensConf();
    if (tokenConf != null) {
        int maxSize = getConfig().getInt(YarnConfiguration.RM_DELEGATION_TOKEN_MAX_CONF_SIZE, YarnConfiguration.DEFAULT_RM_DELEGATION_TOKEN_MAX_CONF_SIZE_BYTES);
        LOG.info("Using app provided configurations for delegation token renewal," + " total size = " + tokenConf.capacity());
        if (tokenConf.capacity() > maxSize) {
            throw new YarnException("Exceed " + YarnConfiguration.RM_DELEGATION_TOKEN_MAX_CONF_SIZE + " = " + maxSize + " bytes, current conf size = " + tokenConf.capacity() + " bytes.");
        }
    }
    if (submissionContext.getQueue() == null) {
        submissionContext.setQueue(YarnConfiguration.DEFAULT_QUEUE_NAME);
    }
    if (submissionContext.getApplicationName() == null) {
        submissionContext.setApplicationName(YarnConfiguration.DEFAULT_APPLICATION_NAME);
    }
    if (submissionContext.getApplicationType() == null) {
        submissionContext.setApplicationType(YarnConfiguration.DEFAULT_APPLICATION_TYPE);
    } else {
        if (submissionContext.getApplicationType().length() > YarnConfiguration.APPLICATION_TYPE_LENGTH) {
            submissionContext.setApplicationType(submissionContext.getApplicationType().substring(0, YarnConfiguration.APPLICATION_TYPE_LENGTH));
        }
    }
    ReservationId reservationId = request.getApplicationSubmissionContext().getReservationID();
    checkReservationACLs(submissionContext.getQueue(), AuditConstants.SUBMIT_RESERVATION_REQUEST, reservationId);
    try {
        // call RMAppManager to submit application directly
        rmAppManager.submitApplication(submissionContext, System.currentTimeMillis(), user);
        LOG.info("Application with id " + applicationId.getId() + " submitted by user " + user);
        RMAuditLogger.logSuccess(user, AuditConstants.SUBMIT_APP_REQUEST, "ClientRMService", applicationId, callerContext);
    } catch (YarnException e) {
        LOG.info("Exception in submitting " + applicationId, e);
        RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, e.getMessage(), "ClientRMService", "Exception in submitting application", applicationId, callerContext);
        throw e;
    }
    SubmitApplicationResponse response = recordFactory.newRecordInstance(SubmitApplicationResponse.class);
    return response;
}

Example 63 with ApplicationSubmissionContext

use of org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext in project hadoop by apache.

the class YARNRunner method submitJob.

@Override
public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts) throws IOException, InterruptedException {
    addHistoryToken(ts);
    ApplicationSubmissionContext appContext = createApplicationSubmissionContext(conf, jobSubmitDir, ts);
    // Submit to ResourceManager
    try {
        ApplicationId applicationId = resMgrDelegate.submitApplication(appContext);
        ApplicationReport appMaster = resMgrDelegate.getApplicationReport(applicationId);
        String diagnostics = (appMaster == null ? "application report is null" : appMaster.getDiagnostics());
        if (appMaster == null || appMaster.getYarnApplicationState() == YarnApplicationState.FAILED || appMaster.getYarnApplicationState() == YarnApplicationState.KILLED) {
            throw new IOException("Failed to run job : " + diagnostics);
        }
        return clientCache.getClient(jobId).getJobStatus(jobId);
    } catch (YarnException e) {
        throw new IOException(e);
    }
}

Example 64 with ApplicationSubmissionContext

use of org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext in project hadoop by apache.

the class YARNRunner method createApplicationSubmissionContext.

/**
   * Constructs all the necessary information to start the MR AM.
   * @param jobConf the configuration for the MR job
   * @param jobSubmitDir the directory path for the job
   * @param ts the security credentials for the job
   * @return ApplicationSubmissionContext
   * @throws IOException on IO error (e.g. path resolution)
   */
public ApplicationSubmissionContext createApplicationSubmissionContext(Configuration jobConf, String jobSubmitDir, Credentials ts) throws IOException {
    ApplicationId applicationId = resMgrDelegate.getApplicationId();
    // Setup resource requirements
    Resource capability = recordFactory.newRecordInstance(Resource.class);
    capability.setMemorySize(conf.getInt(MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB));
    capability.setVirtualCores(conf.getInt(MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES));
    LOG.debug("AppMaster capability = " + capability);
    // Setup LocalResources
    Map<String, LocalResource> localResources = setupLocalResources(jobConf, jobSubmitDir);
    // Setup security tokens
    DataOutputBuffer dob = new DataOutputBuffer();
    ts.writeTokenStorageToStream(dob);
    ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    // Setup ContainerLaunchContext for AM container
    List<String> vargs = setupAMCommand(jobConf);
    ContainerLaunchContext amContainer = setupContainerLaunchContextForAM(jobConf, localResources, securityTokens, vargs);
    String regex = conf.get(MRJobConfig.MR_JOB_SEND_TOKEN_CONF);
    if (regex != null && !regex.isEmpty()) {
        setTokenRenewerConf(amContainer, conf, regex);
    }
    Collection<String> tagsFromConf = jobConf.getTrimmedStringCollection(MRJobConfig.JOB_TAGS);
    // Set up the ApplicationSubmissionContext
    ApplicationSubmissionContext appContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
    // ApplicationId
    appContext.setApplicationId(applicationId);
    // Queue name
    appContext.setQueue(jobConf.get(JobContext.QUEUE_NAME, YarnConfiguration.DEFAULT_QUEUE_NAME));
    // add reservationID if present
    ReservationId reservationID = null;
    try {
        reservationID = ReservationId.parseReservationId(jobConf.get(JobContext.RESERVATION_ID));
    } catch (NumberFormatException e) {
        // throw exception as reservationid as is invalid
        String errMsg = "Invalid reservationId: " + jobConf.get(JobContext.RESERVATION_ID) + " specified for the app: " + applicationId;
        LOG.warn(errMsg);
        throw new IOException(errMsg);
    }
    if (reservationID != null) {
        appContext.setReservationID(reservationID);
        LOG.info("SUBMITTING ApplicationSubmissionContext app:" + applicationId + " to queue:" + appContext.getQueue() + " with reservationId:" + appContext.getReservationID());
    }
    // Job name
    appContext.setApplicationName(jobConf.get(JobContext.JOB_NAME, YarnConfiguration.DEFAULT_APPLICATION_NAME));
    appContext.setCancelTokensWhenComplete(conf.getBoolean(MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN, true));
    // AM Container
    appContext.setAMContainerSpec(amContainer);
    appContext.setMaxAppAttempts(conf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS));
    appContext.setResource(capability);
    // set labels for the AM container request if present
    String amNodelabelExpression = conf.get(MRJobConfig.AM_NODE_LABEL_EXP);
    if (null != amNodelabelExpression && amNodelabelExpression.trim().length() != 0) {
        ResourceRequest amResourceRequest = recordFactory.newRecordInstance(ResourceRequest.class);
        amResourceRequest.setPriority(AM_CONTAINER_PRIORITY);
        amResourceRequest.setResourceName(ResourceRequest.ANY);
        amResourceRequest.setCapability(capability);
        amResourceRequest.setNumContainers(1);
        amResourceRequest.setNodeLabelExpression(amNodelabelExpression.trim());
        appContext.setAMContainerResourceRequest(amResourceRequest);
    }
    // set labels for the Job containers
    appContext.setNodeLabelExpression(jobConf.get(JobContext.JOB_NODE_LABEL_EXP));
    appContext.setApplicationType(MRJobConfig.MR_APPLICATION_TYPE);
    if (tagsFromConf != null && !tagsFromConf.isEmpty()) {
        appContext.setApplicationTags(new HashSet<>(tagsFromConf));
    }
    String jobPriority = jobConf.get(MRJobConfig.PRIORITY);
    if (jobPriority != null) {
        int iPriority;
        try {
            iPriority = TypeConverter.toYarnApplicationPriority(jobPriority);
        } catch (IllegalArgumentException e) {
            iPriority = Integer.parseInt(jobPriority);
        }
        appContext.setPriority(Priority.newInstance(iPriority));
    }
    return appContext;
}

Example 65 with ApplicationSubmissionContext

use of org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext in project hadoop by apache.

the class TestYARNRunner method testJobPriority.

@Test
public void testJobPriority() throws Exception {
    JobConf jobConf = new JobConf();
    jobConf.set(MRJobConfig.PRIORITY, "LOW");
    YARNRunner yarnRunner = new YARNRunner(jobConf);
    ApplicationSubmissionContext appSubCtx = buildSubmitContext(yarnRunner, jobConf);
    // 2 corresponds to LOW
    assertEquals(appSubCtx.getPriority(), Priority.newInstance(2));
    // Set an integer explicitly
    jobConf.set(MRJobConfig.PRIORITY, "12");
    yarnRunner = new YARNRunner(jobConf);
    appSubCtx = buildSubmitContext(yarnRunner, jobConf);
    // Verify whether 12 is set to submission context
    assertEquals(appSubCtx.getPriority(), Priority.newInstance(12));
}