Examples with Allocation org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation used on opensource projects

Example 6 with Allocation

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation in project hadoop by apache.

the class TestFifoScheduler method testBlackListNodes.

@Test(timeout = 50000)
public void testBlackListNodes() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    rm.start();
    FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();
    int rack_num_0 = 0;
    int rack_num_1 = 1;
    // Add 4 nodes in 2 racks
    // host_0_0 in rack0
    String host_0_0 = "127.0.0.1";
    RMNode n1 = MockNodes.newNodeInfo(rack_num_0, MockNodes.newResource(4 * GB), 1, host_0_0);
    fs.handle(new NodeAddedSchedulerEvent(n1));
    // host_0_1 in rack0
    String host_0_1 = "127.0.0.2";
    RMNode n2 = MockNodes.newNodeInfo(rack_num_0, MockNodes.newResource(4 * GB), 1, host_0_1);
    fs.handle(new NodeAddedSchedulerEvent(n2));
    // host_1_0 in rack1
    String host_1_0 = "127.0.0.3";
    RMNode n3 = MockNodes.newNodeInfo(rack_num_1, MockNodes.newResource(4 * GB), 1, host_1_0);
    fs.handle(new NodeAddedSchedulerEvent(n3));
    // host_1_1 in rack1
    String host_1_1 = "127.0.0.4";
    RMNode n4 = MockNodes.newNodeInfo(rack_num_1, MockNodes.newResource(4 * GB), 1, host_1_1);
    fs.handle(new NodeAddedSchedulerEvent(n4));
    // Add one application
    ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1);
    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(appId1, 1);
    createMockRMApp(appAttemptId1, rm.getRMContext());
    SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user");
    fs.handle(appEvent);
    SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false);
    fs.handle(attemptEvent);
    List<ContainerId> emptyId = new ArrayList<ContainerId>();
    List<ResourceRequest> emptyAsk = new ArrayList<ResourceRequest>();
    // Allow rack-locality for rack_1, but blacklist host_1_0
    // Set up resource requests
    // Ask for a 1 GB container for app 1
    List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
    ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "rack1", BuilderUtils.newResource(GB, 1), 1, RMNodeLabelsManager.NO_LABEL));
    ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1, RMNodeLabelsManager.NO_LABEL));
    fs.allocate(appAttemptId1, ask1, emptyId, Collections.singletonList(host_1_0), null, NULL_UPDATE_REQUESTS);
    // Trigger container assignment
    fs.handle(new NodeUpdateSchedulerEvent(n3));
    // Get the allocation for the application and verify no allocation on
    // blacklist node
    Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation1", 0, allocation1.getContainers().size());
    // verify host_1_1 can get allocated as not in blacklist
    fs.handle(new NodeUpdateSchedulerEvent(n4));
    Allocation allocation2 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation2", 1, allocation2.getContainers().size());
    List<Container> containerList = allocation2.getContainers();
    for (Container container : containerList) {
        Assert.assertEquals("Container is allocated on n4", container.getNodeId(), n4.getNodeID());
    }
    // Ask for a 1 GB container again for app 1
    List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
    // this time, rack0 is also in blacklist, so only host_1_1 is available to
    // be assigned
    ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
    fs.allocate(appAttemptId1, ask2, emptyId, Collections.singletonList("rack0"), null, NULL_UPDATE_REQUESTS);
    // verify n1 is not qualified to be allocated
    fs.handle(new NodeUpdateSchedulerEvent(n1));
    Allocation allocation3 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation3", 0, allocation3.getContainers().size());
    // verify n2 is not qualified to be allocated
    fs.handle(new NodeUpdateSchedulerEvent(n2));
    Allocation allocation4 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation4", 0, allocation4.getContainers().size());
    // verify n3 is not qualified to be allocated
    fs.handle(new NodeUpdateSchedulerEvent(n3));
    Allocation allocation5 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation5", 0, allocation5.getContainers().size());
    fs.handle(new NodeUpdateSchedulerEvent(n4));
    Allocation allocation6 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null, NULL_UPDATE_REQUESTS);
    Assert.assertEquals("allocation6", 1, allocation6.getContainers().size());
    containerList = allocation6.getContainers();
    for (Container container : containerList) {
        Assert.assertEquals("Container is allocated on n4", container.getNodeId(), n4.getNodeID());
    }
    rm.stop();
}

Example 7 with Allocation

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation in project hadoop by apache.

the class TestCapacityScheduler method testApplicationHeadRoom.

// Verifies headroom passed to ApplicationMaster has been updated in
// RMAppAttemptMetrics
@Test
public void testApplicationHeadRoom() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    rm.start();
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    RMAppAttemptMetrics attemptMetric = new RMAppAttemptMetrics(appAttemptId, rm.getRMContext());
    RMAppImpl app = mock(RMAppImpl.class);
    when(app.getApplicationId()).thenReturn(appId);
    RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class);
    Container container = mock(Container.class);
    when(attempt.getMasterContainer()).thenReturn(container);
    ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class);
    when(attempt.getSubmissionContext()).thenReturn(submissionContext);
    when(attempt.getAppAttemptId()).thenReturn(appAttemptId);
    when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric);
    when(app.getCurrentAppAttempt()).thenReturn(attempt);
    rm.getRMContext().getRMApps().put(appId, app);
    SchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId, "default", "user");
    cs.handle(addAppEvent);
    SchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    cs.handle(addAttemptEvent);
    Allocation allocate = cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(), Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS);
    Assert.assertNotNull(attempt);
    Assert.assertEquals(Resource.newInstance(0, 0), allocate.getResourceLimit());
    Assert.assertEquals(Resource.newInstance(0, 0), attemptMetric.getApplicationAttemptHeadroom());
    // Add a node to cluster
    Resource newResource = Resource.newInstance(4 * GB, 1);
    RMNode node = MockNodes.newNodeInfo(0, newResource, 1, "127.0.0.1");
    cs.handle(new NodeAddedSchedulerEvent(node));
    allocate = cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(), Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS);
    // All resources should be sent as headroom
    Assert.assertEquals(newResource, allocate.getResourceLimit());
    Assert.assertEquals(newResource, attemptMetric.getApplicationAttemptHeadroom());
    rm.stop();
}

Example 8 with Allocation

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation in project hadoop by apache.

the class Application method getResources.

public synchronized List<Container> getResources() throws IOException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("getResources begin:" + " application=" + applicationId + " #ask=" + ask.size());
        for (ResourceRequest request : ask) {
            LOG.debug("getResources:" + " application=" + applicationId + " ask-request=" + request);
        }
    }
    // Get resources from the ResourceManager
    Allocation allocation = resourceManager.getResourceScheduler().allocate(applicationAttemptId, new ArrayList<ResourceRequest>(ask), new ArrayList<ContainerId>(), null, null, new ContainerUpdates());
    if (LOG.isInfoEnabled()) {
        LOG.info("-=======" + applicationAttemptId + System.lineSeparator() + "----------" + resourceManager.getRMContext().getRMApps().get(applicationId).getRMAppAttempt(applicationAttemptId));
    }
    List<Container> containers = allocation.getContainers();
    // Clear state for next interaction with ResourceManager
    ask.clear();
    if (LOG.isDebugEnabled()) {
        LOG.debug("getResources() for " + applicationId + ":" + " ask=" + ask.size() + " recieved=" + containers.size());
    }
    return containers;
}

Example 9 with Allocation

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation in project hadoop by apache.

the class SLSCapacityScheduler method allocate.

@Override
public Allocation allocate(ApplicationAttemptId attemptId, List<ResourceRequest> resourceRequests, List<ContainerId> containerIds, List<String> strings, List<String> strings2, ContainerUpdates updateRequests) {
    if (metricsON) {
        final Timer.Context context = schedulerAllocateTimer.time();
        Allocation allocation = null;
        try {
            allocation = super.allocate(attemptId, resourceRequests, containerIds, strings, strings2, updateRequests);
            return allocation;
        } finally {
            context.stop();
            schedulerAllocateCounter.inc();
            try {
                updateQueueWithAllocateRequest(allocation, attemptId, resourceRequests, containerIds);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    } else {
        return super.allocate(attemptId, resourceRequests, containerIds, strings, strings2, updateRequests);
    }
}

Example 10 with Allocation

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation in project hadoop by apache.

the class ResourceSchedulerWrapper method allocate.

@Override
public Allocation allocate(ApplicationAttemptId attemptId, List<ResourceRequest> resourceRequests, List<ContainerId> containerIds, List<String> strings, List<String> strings2, ContainerUpdates updateRequests) {
    if (metricsON) {
        final Timer.Context context = schedulerAllocateTimer.time();
        Allocation allocation = null;
        try {
            allocation = scheduler.allocate(attemptId, resourceRequests, containerIds, strings, strings2, updateRequests);
            return allocation;
        } finally {
            context.stop();
            schedulerAllocateCounter.inc();
            try {
                updateQueueWithAllocateRequest(allocation, attemptId, resourceRequests, containerIds);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    } else {
        return scheduler.allocate(attemptId, resourceRequests, containerIds, strings, strings2, updateRequests);
    }
}