Examples with CSQueue org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue used on opensource projects

Example 1 with CSQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue in project hadoop by apache.

the class RMAppManager method createAndPopulateNewRMApp.

private RMAppImpl createAndPopulateNewRMApp(ApplicationSubmissionContext submissionContext, long submitTime, String user, boolean isRecovery, long startTime) throws YarnException {
    if (!isRecovery) {
        // Do queue mapping
        if (rmContext.getQueuePlacementManager() != null) {
            // We only do queue mapping when it's a new application
            rmContext.getQueuePlacementManager().placeApplication(submissionContext, user);
        }
        // fail the submission if configured application timeout value is invalid
        RMServerUtils.validateApplicationTimeouts(submissionContext.getApplicationTimeouts());
    }
    ApplicationId applicationId = submissionContext.getApplicationId();
    ResourceRequest amReq = null;
    try {
        amReq = validateAndCreateResourceRequest(submissionContext, isRecovery);
    } catch (InvalidLabelResourceRequestException e) {
        // after recovery and user can see what's going on and react accordingly.
        if (isRecovery && !YarnConfiguration.areNodeLabelsEnabled(this.conf)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("AMResourceRequest is not created for " + applicationId + ". NodeLabel is not enabled in cluster, but AM resource " + "request contains a label expression.");
            }
        } else {
            throw e;
        }
    }
    // Verify and get the update application priority and set back to
    // submissionContext
    UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user);
    Priority appPriority = scheduler.checkAndGetApplicationPriority(submissionContext.getPriority(), userUgi, submissionContext.getQueue(), applicationId);
    submissionContext.setPriority(appPriority);
    // For now, exclude FS for the acl check.
    if (!isRecovery && YarnConfiguration.isAclEnabled(conf) && scheduler instanceof CapacityScheduler) {
        String queueName = submissionContext.getQueue();
        String appName = submissionContext.getApplicationName();
        CSQueue csqueue = ((CapacityScheduler) scheduler).getQueue(queueName);
        if (null != csqueue && !authorizer.checkPermission(new AccessRequest(csqueue.getPrivilegedEntity(), userUgi, SchedulerUtils.toAccessType(QueueACL.SUBMIT_APPLICATIONS), applicationId.toString(), appName, Server.getRemoteAddress(), null)) && !authorizer.checkPermission(new AccessRequest(csqueue.getPrivilegedEntity(), userUgi, SchedulerUtils.toAccessType(QueueACL.ADMINISTER_QUEUE), applicationId.toString(), appName, Server.getRemoteAddress(), null))) {
            throw RPCUtil.getRemoteException(new AccessControlException("User " + user + " does not have permission to submit " + applicationId + " to queue " + submissionContext.getQueue()));
        }
    }
    // Create RMApp
    RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf, submissionContext.getApplicationName(), user, submissionContext.getQueue(), submissionContext, this.scheduler, this.masterService, submitTime, submissionContext.getApplicationType(), submissionContext.getApplicationTags(), amReq, startTime);
    // influence each other
    if (rmContext.getRMApps().putIfAbsent(applicationId, application) != null) {
        String message = "Application with id " + applicationId + " is already present! Cannot add a duplicate!";
        LOG.warn(message);
        throw new YarnException(message);
    }
    if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
        // Start timeline collector for the submitted app
        application.startTimelineCollector();
    }
    // Inform the ACLs Manager
    this.applicationACLsManager.addApplication(applicationId, submissionContext.getAMContainerSpec().getApplicationACLs());
    String appViewACLs = submissionContext.getAMContainerSpec().getApplicationACLs().get(ApplicationAccessType.VIEW_APP);
    rmContext.getSystemMetricsPublisher().appACLsUpdated(application, appViewACLs, System.currentTimeMillis());
    return application;
}

Example 2 with CSQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue in project hadoop by apache.

the class RMWebServices method getSchedulerInfo.

@GET
@Path("/scheduler")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public SchedulerTypeInfo getSchedulerInfo() {
    init();
    ResourceScheduler rs = rm.getResourceScheduler();
    SchedulerInfo sinfo;
    if (rs instanceof CapacityScheduler) {
        CapacityScheduler cs = (CapacityScheduler) rs;
        CSQueue root = cs.getRootQueue();
        sinfo = new CapacitySchedulerInfo(root, cs);
    } else if (rs instanceof FairScheduler) {
        FairScheduler fs = (FairScheduler) rs;
        sinfo = new FairSchedulerInfo(fs);
    } else if (rs instanceof FifoScheduler) {
        sinfo = new FifoSchedulerInfo(this.rm);
    } else {
        throw new NotFoundException("Unknown scheduler configured");
    }
    return new SchedulerTypeInfo(sinfo);
}

Example 3 with CSQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue in project hadoop by apache.

the class CapacityReservationSystem method getPlanQueueCapacity.

@Override
protected Resource getPlanQueueCapacity(String planQueueName) {
    Resource minAllocation = getMinAllocation();
    ResourceCalculator rescCalc = getResourceCalculator();
    CSQueue planQueue = capScheduler.getQueue(planQueueName);
    return rescCalc.multiplyAndNormalizeDown(capScheduler.getClusterResource(), planQueue.getAbsoluteCapacity(), minAllocation);
}

Example 4 with CSQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method mockQueueHierarchy.

/**
   * Format is:
   * <pre>
   * root (<partition-name-1>=[guaranteed max used pending (reserved)],<partition-name-2>=..);
   * -A(...);
   * --A1(...);
   * --A2(...);
   * -B...
   * </pre>
   * ";" splits queues, and there should no empty lines, no extra spaces
   *
   * For each queue, it has configurations to specify capacities (to each
   * partition), format is:
   * <pre>
   * -<queueName> (<labelName1>=[guaranteed max used pending], \
   *               <labelName2>=[guaranteed max used pending])
   *              {key1=value1,key2=value2};  // Additional configs
   * </pre>
   */
@SuppressWarnings({ "unchecked", "rawtypes" })
private ParentQueue mockQueueHierarchy(String queueExprs) {
    String[] queueExprArray = queueExprs.split(";");
    ParentQueue rootQueue = null;
    for (int idx = 0; idx < queueExprArray.length; idx++) {
        String q = queueExprArray[idx];
        CSQueue queue;
        // Initialize queue
        if (isParent(queueExprArray, idx)) {
            ParentQueue parentQueue = mock(ParentQueue.class);
            queue = parentQueue;
            List<CSQueue> children = new ArrayList<CSQueue>();
            when(parentQueue.getChildQueues()).thenReturn(children);
            QueueOrderingPolicy policy = mock(QueueOrderingPolicy.class);
            when(policy.getConfigName()).thenReturn(CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY);
            when(parentQueue.getQueueOrderingPolicy()).thenReturn(policy);
        } else {
            LeafQueue leafQueue = mock(LeafQueue.class);
            final TreeSet<FiCaSchedulerApp> apps = new TreeSet<>(new Comparator<FiCaSchedulerApp>() {

                @Override
                public int compare(FiCaSchedulerApp a1, FiCaSchedulerApp a2) {
                    if (a1.getPriority() != null && !a1.getPriority().equals(a2.getPriority())) {
                        return a1.getPriority().compareTo(a2.getPriority());
                    }
                    int res = a1.getApplicationId().compareTo(a2.getApplicationId());
                    return res;
                }
            });
            when(leafQueue.getApplications()).thenReturn(apps);
            when(leafQueue.getAllApplications()).thenReturn(apps);
            OrderingPolicy<FiCaSchedulerApp> so = mock(OrderingPolicy.class);
            when(so.getPreemptionIterator()).thenAnswer(new Answer() {

                public Object answer(InvocationOnMock invocation) {
                    return apps.descendingIterator();
                }
            });
            when(leafQueue.getOrderingPolicy()).thenReturn(so);
            Map<String, TreeSet<RMContainer>> ignorePartitionContainers = new HashMap<>();
            when(leafQueue.getIgnoreExclusivityRMContainers()).thenReturn(ignorePartitionContainers);
            queue = leafQueue;
        }
        ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
        when(queue.getReadLock()).thenReturn(lock.readLock());
        setupQueue(queue, q, queueExprArray, idx);
        if (queue.getQueueName().equals(ROOT)) {
            rootQueue = (ParentQueue) queue;
        }
    }
    return rootQueue;
}

Example 5 with CSQueue

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue in project hadoop by apache.

the class TestCapacitySchedulerPlanFollower method assertReservationQueueDoesNotExist.

@Override
protected void assertReservationQueueDoesNotExist(ReservationId r2) {
    CSQueue q2 = cs.getQueue(r2.toString());
    assertNull(q2);
}