Examples with Priority org.apache.hadoop.yarn.api.records.Priority used on opensource projects

Example 11 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project hadoop by apache.

the class RMWebServices method updateApplicationPriority.

@PUT
@Path("/apps/{appid}/priority")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public Response updateApplicationPriority(AppPriority targetPriority, @Context HttpServletRequest hsr, @PathParam("appid") String appId) throws AuthorizationException, YarnException, InterruptedException, IOException {
    init();
    if (targetPriority == null) {
        throw new YarnException("Target Priority cannot be null");
    }
    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
    if (callerUGI == null) {
        throw new AuthorizationException("Unable to obtain user name, user not authenticated");
    }
    if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) {
        return Response.status(Status.FORBIDDEN).entity("The default static user cannot carry out this operation.").build();
    }
    String userName = callerUGI.getUserName();
    RMApp app = null;
    try {
        app = getRMAppForAppId(appId);
    } catch (NotFoundException e) {
        RMAuditLogger.logFailure(userName, AuditConstants.UPDATE_APP_PRIORITY, "UNKNOWN", "RMWebService", "Trying to update priority an absent application " + appId);
        throw e;
    }
    Priority priority = app.getApplicationPriority();
    if (priority == null || priority.getPriority() != targetPriority.getPriority()) {
        return modifyApplicationPriority(app, callerUGI, targetPriority.getPriority());
    }
    return Response.status(Status.OK).entity(targetPriority).build();
}

Example 12 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project hadoop by apache.

the class RMWebServices method modifyApplicationPriority.

private Response modifyApplicationPriority(final RMApp app, UserGroupInformation callerUGI, final int appPriority) throws IOException, InterruptedException {
    String userName = callerUGI.getUserName();
    try {
        callerUGI.doAs(new PrivilegedExceptionAction<Void>() {

            @Override
            public Void run() throws IOException, YarnException {
                Priority priority = Priority.newInstance(appPriority);
                UpdateApplicationPriorityRequest request = UpdateApplicationPriorityRequest.newInstance(app.getApplicationId(), priority);
                rm.getClientRMService().updateApplicationPriority(request);
                return null;
            }
        });
    } catch (UndeclaredThrowableException ue) {
        // bubble that up to the user
        if (ue.getCause() instanceof YarnException) {
            YarnException ye = (YarnException) ue.getCause();
            if (ye.getCause() instanceof AccessControlException) {
                String appId = app.getApplicationId().toString();
                String msg = "Unauthorized attempt to change priority of appid " + appId + " by remote user " + userName;
                return Response.status(Status.FORBIDDEN).entity(msg).build();
            } else if (ye.getMessage().startsWith("Application in") && ye.getMessage().endsWith("state cannot be update priority.")) {
                return Response.status(Status.BAD_REQUEST).entity(ye.getMessage()).build();
            } else {
                throw ue;
            }
        } else {
            throw ue;
        }
    }
    AppPriority ret = new AppPriority(app.getApplicationPriority().getPriority());
    return Response.status(Status.OK).entity(ret).build();
}

Example 13 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project hadoop by apache.

the class MockRM method submitApp.

public RMApp submitApp(int masterMemory, long attemptFailuresValidityInterval, boolean keepContainers) throws Exception {
    Resource resource = Records.newRecord(Resource.class);
    resource.setMemorySize(masterMemory);
    Priority priority = Priority.newInstance(0);
    return submitApp(resource, "", UserGroupInformation.getCurrentUser().getShortUserName(), null, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, keepContainers, false, null, attemptFailuresValidityInterval, null, true, priority);
}

Example 14 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project hadoop by apache.

the class AppPriorityACLConfigurationParser method parsePriorityACLType.

/*
   * Parse different types of ACLs sub parts for on priority group and store in
   * a map for later processing.
   */
private void parsePriorityACLType(AppPriorityACLGroup userPriorityACL, String[] splits, List<StringBuilder> userAndGroupName) {
    // Here splits will have the key value pair at index 0 and 1 respectively.
    // To parse all keys, its better to convert to PriorityACLConfig enum.
    AppPriorityACLKeyType aclType = AppPriorityACLKeyType.valueOf(StringUtils.toUpperCase(splits[0].trim()));
    switch(aclType) {
        case MAX_PRIORITY:
            userPriorityACL.setMaxPriority(Priority.newInstance(Integer.parseInt(splits[1])));
            break;
        case USER:
            userAndGroupName.add(getUserOrGroupACLStringFromConfig(splits[1]));
            break;
        case GROUP:
            userAndGroupName.add(getUserOrGroupACLStringFromConfig(splits[1]));
            break;
        case DEFAULT_PRIORITY:
            int defaultPriority = Integer.parseInt(splits[1]);
            Priority priority = (defaultPriority < 0) ? Priority.newInstance(0) : Priority.newInstance(defaultPriority);
            userPriorityACL.setDefaultPriority(priority);
            break;
        default:
            break;
    }
}

Example 15 with Priority

use of org.apache.hadoop.yarn.api.records.Priority in project hadoop by apache.

the class RegularContainerAllocator method preCheckForPlacementSet.

/*
   * Pre-check if we can allocate a pending resource request
   * (given schedulerKey) to a given PlacementSet.
   * We will consider stuffs like exclusivity, pending resource, node partition,
   * headroom, etc.
   */
private ContainerAllocation preCheckForPlacementSet(Resource clusterResource, PlacementSet<FiCaSchedulerNode> ps, SchedulingMode schedulingMode, ResourceLimits resourceLimits, SchedulerRequestKey schedulerKey) {
    Priority priority = schedulerKey.getPriority();
    FiCaSchedulerNode node = PlacementSetUtils.getSingleNode(ps);
    PendingAsk offswitchPendingAsk = application.getPendingAsk(schedulerKey, ResourceRequest.ANY);
    if (offswitchPendingAsk.getCount() <= 0) {
        ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.PRIORITY_SKIPPED_BECAUSE_NULL_ANY_REQUEST);
        return ContainerAllocation.PRIORITY_SKIPPED;
    }
    // Required resource
    Resource required = offswitchPendingAsk.getPerAllocationResource();
    // Do we need containers at this 'priority'?
    if (application.getOutstandingAsksCount(schedulerKey) <= 0) {
        ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.APPLICATION_PRIORITY_DO_NOT_NEED_RESOURCE);
        return ContainerAllocation.PRIORITY_SKIPPED;
    }
    // avoid painful of preempt an AM container
    if (schedulingMode == SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY) {
        if (application.isWaitingForAMContainer()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Skip allocating AM container to app_attempt=" + application.getApplicationAttemptId() + ", don't allow to allocate AM container in non-exclusive mode");
            }
            application.updateAppSkipNodeDiagnostics("Skipping assigning to Node in Ignore Exclusivity mode. ");
            ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.SKIP_IN_IGNORE_EXCLUSIVITY_MODE);
            return ContainerAllocation.APP_SKIPPED;
        }
    }
    // If not match, jump to next priority.
    if (!appInfo.acceptNodePartition(schedulerKey, node.getPartition(), schedulingMode)) {
        ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.PRIORITY_SKIPPED_BECAUSE_NODE_PARTITION_DOES_NOT_MATCH_REQUEST);
        return ContainerAllocation.PRIORITY_SKIPPED;
    }
    if (!application.getCSLeafQueue().getReservationContinueLooking()) {
        if (!shouldAllocOrReserveNewContainer(schedulerKey, required)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("doesn't need containers based on reservation algo!");
            }
            ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.DO_NOT_NEED_ALLOCATIONATTEMPTINFOS);
            return ContainerAllocation.PRIORITY_SKIPPED;
        }
    }
    if (!checkHeadroom(clusterResource, resourceLimits, required, ps.getPartition())) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("cannot allocate required resource=" + required + " because of headroom");
        }
        ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.QUEUE_SKIPPED_HEADROOM);
        return ContainerAllocation.QUEUE_SKIPPED;
    }
    // Increase missed-non-partitioned-resource-request-opportunity.
    // This is to make sure non-partitioned-resource-request will prefer
    // to be allocated to non-partitioned nodes
    int missedNonPartitionedRequestSchedulingOpportunity = 0;
    // NO_LABEL under RESPECT_EXCLUSIVITY mode
    if (StringUtils.equals(RMNodeLabelsManager.NO_LABEL, appInfo.getSchedulingPlacementSet(schedulerKey).getPrimaryRequestedNodePartition())) {
        missedNonPartitionedRequestSchedulingOpportunity = application.addMissedNonPartitionedRequestSchedulingOpportunity(schedulerKey);
    }
    if (schedulingMode == SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY) {
        // non-partitioned partition first.
        if (missedNonPartitionedRequestSchedulingOpportunity < rmContext.getScheduler().getNumClusterNodes()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Skip app_attempt=" + application.getApplicationAttemptId() + " priority=" + schedulerKey.getPriority() + " because missed-non-partitioned-resource-request" + " opportunity under required:" + " Now=" + missedNonPartitionedRequestSchedulingOpportunity + " required=" + rmContext.getScheduler().getNumClusterNodes());
            }
            ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(activitiesManager, node, application, priority, ActivityDiagnosticConstant.NON_PARTITIONED_PARTITION_FIRST);
            return ContainerAllocation.APP_SKIPPED;
        }
    }
    return null;
}