Examples with ResourceRequest org.apache.hadoop.yarn.api.records.ResourceRequest used on opensource projects

Example 91 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class TestFairScheduler method testCancelStrictLocality.

@Test
public void testCancelStrictLocality() throws IOException {
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
    scheduler.handle(nodeEvent1);
    RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 2, "127.0.0.2");
    NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
    scheduler.handle(nodeEvent2);
    ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1", 0);
    ResourceRequest nodeRequest = createResourceRequest(1024, node1.getHostName(), 1, 1, true);
    ResourceRequest rackRequest = createResourceRequest(1024, "rack1", 1, 1, false);
    ResourceRequest anyRequest = createResourceRequest(1024, ResourceRequest.ANY, 1, 1, false);
    createSchedulingRequestExistingApplication(nodeRequest, attId1);
    createSchedulingRequestExistingApplication(rackRequest, attId1);
    createSchedulingRequestExistingApplication(anyRequest, attId1);
    scheduler.update();
    NodeUpdateSchedulerEvent node2UpdateEvent = new NodeUpdateSchedulerEvent(node2);
    // no matter how many heartbeats, node2 should never get a container
    FSAppAttempt app = scheduler.getSchedulerApp(attId1);
    for (int i = 0; i < 10; i++) {
        scheduler.handle(node2UpdateEvent);
        assertEquals(0, app.getLiveContainers().size());
    }
    // relax locality
    List<ResourceRequest> update = Arrays.asList(createResourceRequest(1024, node1.getHostName(), 1, 0, true), createResourceRequest(1024, "rack1", 1, 0, true), createResourceRequest(1024, ResourceRequest.ANY, 1, 1, true));
    scheduler.allocate(attId1, update, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
    // then node2 should get the container
    scheduler.handle(node2UpdateEvent);
    assertEquals(1, app.getLiveContainers().size());
}

Example 92 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class AppSchedulingInfo method updateResourceRequests.

/**
   * The ApplicationMaster is updating resource requirements for the
   * application, by asking for more resources and releasing resources acquired
   * by the application.
   *
   * @param requests
   *          resources to be acquired
   * @param recoverPreemptedRequestForAContainer
   *          recover ResourceRequest on preemption
   * @return true if any resource was updated, false otherwise
   */
public boolean updateResourceRequests(List<ResourceRequest> requests, boolean recoverPreemptedRequestForAContainer) {
    if (null == requests || requests.isEmpty()) {
        return false;
    }
    // Flag to track if any incoming requests update "ANY" requests
    boolean offswitchResourcesUpdated = false;
    try {
        this.writeLock.lock();
        // A map to group resource requests and dedup
        Map<SchedulerRequestKey, Map<String, ResourceRequest>> dedupRequests = new HashMap<>();
        // Group resource request by schedulerRequestKey and resourceName
        for (ResourceRequest request : requests) {
            SchedulerRequestKey schedulerKey = SchedulerRequestKey.create(request);
            if (!dedupRequests.containsKey(schedulerKey)) {
                dedupRequests.put(schedulerKey, new HashMap<>());
            }
            dedupRequests.get(schedulerKey).put(request.getResourceName(), request);
        }
        // Update scheduling placement set
        offswitchResourcesUpdated = addToPlacementSets(recoverPreemptedRequestForAContainer, dedupRequests);
        return offswitchResourcesUpdated;
    } finally {
        this.writeLock.unlock();
    }
}

Example 93 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class FSPreemptionThread method identifyContainersToPreempt.

/**
   * Given an app, identify containers to preempt to satisfy the app's
   * starvation.
   *
   * Mechanics:
   * 1. Fetch all {@link ResourceRequest}s corresponding to the amount of
   * starvation.
   * 2. For each {@link ResourceRequest}, iterate through matching
   * nodes and identify containers to preempt all on one node, also
   * optimizing for least number of AM container preemptions.
   *
   * @param starvedApp starved application for which we are identifying
   *                   preemption targets
   * @return list of containers to preempt to satisfy starvedApp
   */
private List<RMContainer> identifyContainersToPreempt(FSAppAttempt starvedApp) {
    List<RMContainer> containersToPreempt = new ArrayList<>();
    // Iterate through enough RRs to address app's starvation
    for (ResourceRequest rr : starvedApp.getStarvedResourceRequests()) {
        for (int i = 0; i < rr.getNumContainers(); i++) {
            PreemptableContainers bestContainers = null;
            List<FSSchedulerNode> potentialNodes = scheduler.getNodeTracker().getNodesByResourceName(rr.getResourceName());
            for (FSSchedulerNode node : potentialNodes) {
                // TODO (YARN-5829): Attempt to reserve the node for starved app.
                if (isNodeAlreadyReserved(node, starvedApp)) {
                    continue;
                }
                int maxAMContainers = bestContainers == null ? Integer.MAX_VALUE : bestContainers.numAMContainers;
                PreemptableContainers preemptableContainers = identifyContainersToPreemptOnNode(rr.getCapability(), node, maxAMContainers);
                if (preemptableContainers != null) {
                    // This set is better than any previously identified set.
                    bestContainers = preemptableContainers;
                    if (preemptableContainers.numAMContainers == 0) {
                        break;
                    }
                }
            }
            if (bestContainers != null && bestContainers.containers.size() > 0) {
                containersToPreempt.addAll(bestContainers.containers);
                trackPreemptionsAgainstNode(bestContainers.containers);
            }
        }
    }
    // End of iteration over RRs
    return containersToPreempt;
}

Example 94 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class TestFairScheduler method testQueueDemandCalculation.

/**
   * Make allocation requests and ensure they are reflected in queue demand.
   */
@Test
public void testQueueDemandCalculation() throws Exception {
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    ApplicationAttemptId id11 = createAppAttemptId(1, 1);
    createMockRMApp(id11);
    scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1", false);
    scheduler.addApplicationAttempt(id11, false, false);
    ApplicationAttemptId id21 = createAppAttemptId(2, 1);
    createMockRMApp(id21);
    scheduler.addApplication(id21.getApplicationId(), "root.queue2", "user1", false);
    scheduler.addApplicationAttempt(id21, false, false);
    ApplicationAttemptId id22 = createAppAttemptId(2, 2);
    createMockRMApp(id22);
    scheduler.addApplication(id22.getApplicationId(), "root.queue2", "user1", false);
    scheduler.addApplicationAttempt(id22, false, false);
    int minReqSize = FairSchedulerConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB;
    // First ask, queue1 requests 1 large (minReqSize * 2).
    List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
    ResourceRequest request1 = createResourceRequest(minReqSize * 2, ResourceRequest.ANY, 1, 1, true);
    ask1.add(request1);
    scheduler.allocate(id11, ask1, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
    // Second ask, queue2 requests 1 large.
    List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
    ResourceRequest request2 = createResourceRequest(2 * minReqSize, "foo", 1, 1, false);
    ResourceRequest request3 = createResourceRequest(2 * minReqSize, ResourceRequest.ANY, 1, 1, false);
    ask2.add(request2);
    ask2.add(request3);
    scheduler.allocate(id21, ask2, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
    // Third ask, queue2 requests 2 small (minReqSize).
    List<ResourceRequest> ask3 = new ArrayList<ResourceRequest>();
    ResourceRequest request4 = createResourceRequest(minReqSize, "bar", 2, 2, true);
    ResourceRequest request5 = createResourceRequest(minReqSize, ResourceRequest.ANY, 2, 2, true);
    ask3.add(request4);
    ask3.add(request5);
    scheduler.allocate(id22, ask3, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
    scheduler.update();
    assertEquals(2 * minReqSize, scheduler.getQueueManager().getQueue("root.queue1").getDemand().getMemorySize());
    assertEquals(2 * minReqSize + 2 * minReqSize, scheduler.getQueueManager().getQueue("root.queue2").getDemand().getMemorySize());
}

Example 95 with ResourceRequest

use of org.apache.hadoop.yarn.api.records.ResourceRequest in project hadoop by apache.

the class TestFairScheduler method testRackLocalAppReservationThreshold.

@Test(timeout = 5000)
public void testRackLocalAppReservationThreshold() throws Exception {
    conf.setFloat(FairSchedulerConfiguration.RESERVABLE_NODES, 0.50f);
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    // Add four node
    RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(3072), 1, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
    scheduler.handle(nodeEvent1);
    // These 3 on different rack
    RMNode node2 = MockNodes.newNodeInfo(2, Resources.createResource(3072), 1, "127.0.0.2");
    NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
    scheduler.handle(nodeEvent2);
    RMNode node3 = MockNodes.newNodeInfo(2, Resources.createResource(3072), 1, "127.0.0.3");
    NodeAddedSchedulerEvent nodeEvent3 = new NodeAddedSchedulerEvent(node3);
    scheduler.handle(nodeEvent3);
    RMNode node4 = MockNodes.newNodeInfo(2, Resources.createResource(3072), 1, "127.0.0.4");
    NodeAddedSchedulerEvent nodeEvent4 = new NodeAddedSchedulerEvent(node4);
    scheduler.handle(nodeEvent4);
    // Ensure capacity on all nodes are allocated
    createSchedulingRequest(2048, "queue1", "user1", 1);
    scheduler.update();
    scheduler.handle(new NodeUpdateSchedulerEvent(node1));
    createSchedulingRequest(2048, "queue1", "user1", 1);
    scheduler.update();
    scheduler.handle(new NodeUpdateSchedulerEvent(node2));
    createSchedulingRequest(2048, "queue1", "user1", 1);
    scheduler.update();
    scheduler.handle(new NodeUpdateSchedulerEvent(node3));
    createSchedulingRequest(2048, "queue1", "user1", 1);
    scheduler.update();
    scheduler.handle(new NodeUpdateSchedulerEvent(node4));
    // Verify capacity allocation
    assertEquals(8192, scheduler.getQueueManager().getQueue("queue1").getResourceUsage().getMemorySize());
    // Create new app with a resource request that can be satisfied by any
    // node but would be
    ApplicationAttemptId attemptId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
    createMockRMApp(attemptId);
    scheduler.addApplication(attemptId.getApplicationId(), "queue1", "user1", false);
    scheduler.addApplicationAttempt(attemptId, false, false);
    List<ResourceRequest> asks = new ArrayList<ResourceRequest>();
    asks.add(createResourceRequest(2048, node2.getRackName(), 1, 1, false));
    scheduler.allocate(attemptId, asks, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
    ApplicationAttemptId attId = createSchedulingRequest(2048, "queue1", "user1", 1);
    scheduler.update();
    scheduler.handle(new NodeUpdateSchedulerEvent(node1));
    assertEquals(1, scheduler.getSchedulerApp(attId).getNumReservations(null, true));
    scheduler.update();
    scheduler.handle(new NodeUpdateSchedulerEvent(node2));
    assertEquals(2, scheduler.getSchedulerApp(attId).getNumReservations(null, true));
    scheduler.update();
    scheduler.handle(new NodeUpdateSchedulerEvent(node3));
    // No new reservations should happen since it exceeds threshold
    assertEquals(2, scheduler.getSchedulerApp(attId).getNumReservations(null, true));
    // Add 1 more node
    RMNode node5 = MockNodes.newNodeInfo(2, Resources.createResource(3072), 1, "127.0.0.4");
    NodeAddedSchedulerEvent nodeEvent5 = new NodeAddedSchedulerEvent(node5);
    scheduler.handle(nodeEvent5);
    // New node satisfies resource request
    scheduler.update();
    scheduler.handle(new NodeUpdateSchedulerEvent(node4));
    assertEquals(10240, scheduler.getQueueManager().getQueue("queue1").getResourceUsage().getMemorySize());
    scheduler.handle(new NodeUpdateSchedulerEvent(node1));
    scheduler.handle(new NodeUpdateSchedulerEvent(node2));
    scheduler.handle(new NodeUpdateSchedulerEvent(node3));
    scheduler.handle(new NodeUpdateSchedulerEvent(node4));
    scheduler.update();
    // Verify number of reservations have decremented
    assertEquals(0, scheduler.getSchedulerApp(attId).getNumReservations(null, true));
}