Examples with Task org.apache.hadoop.yarn.server.resourcemanager.Task used on opensource projects

Example 6 with Task

use of org.apache.hadoop.yarn.server.resourcemanager.Task in project hadoop by apache.

the class TestCapacityScheduler method testMoveAppSuccess.

@Test
public void testMoveAppSuccess() throws Exception {
    ResourceScheduler scheduler = resourceManager.getResourceScheduler();
    // Register node1
    String host_0 = "host_0";
    NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(5 * GB, 1));
    // Register node2
    String host_1 = "host_1";
    NodeManager nm_1 = registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(5 * GB, 1));
    // ResourceRequest priorities
    Priority priority_0 = Priority.newInstance(0);
    Priority priority_1 = Priority.newInstance(1);
    // Submit application_0
    Application application_0 = new Application("user_0", "a1", resourceManager);
    // app + app attempt event sent to scheduler
    application_0.submit();
    application_0.addNodeManager(host_0, 1234, nm_0);
    application_0.addNodeManager(host_1, 1234, nm_1);
    Resource capability_0_0 = Resources.createResource(3 * GB, 1);
    application_0.addResourceRequestSpec(priority_1, capability_0_0);
    Resource capability_0_1 = Resources.createResource(2 * GB, 1);
    application_0.addResourceRequestSpec(priority_0, capability_0_1);
    Task task_0_0 = new Task(application_0, priority_1, new String[] { host_0, host_1 });
    application_0.addTask(task_0_0);
    // Submit application_1
    Application application_1 = new Application("user_1", "b2", resourceManager);
    // app + app attempt event sent to scheduler
    application_1.submit();
    application_1.addNodeManager(host_0, 1234, nm_0);
    application_1.addNodeManager(host_1, 1234, nm_1);
    Resource capability_1_0 = Resources.createResource(1 * GB, 1);
    application_1.addResourceRequestSpec(priority_1, capability_1_0);
    Resource capability_1_1 = Resources.createResource(2 * GB, 1);
    application_1.addResourceRequestSpec(priority_0, capability_1_1);
    Task task_1_0 = new Task(application_1, priority_1, new String[] { host_0, host_1 });
    application_1.addTask(task_1_0);
    // Send resource requests to the scheduler
    // allocate
    application_0.schedule();
    // allocate
    application_1.schedule();
    // b2 can only run 1 app at a time
    scheduler.moveApplication(application_0.getApplicationId(), "b2");
    nodeUpdate(nm_0);
    nodeUpdate(nm_1);
    // Get allocations from the scheduler
    // task_0_0
    application_0.schedule();
    checkApplicationResourceUsage(0 * GB, application_0);
    // task_1_0
    application_1.schedule();
    checkApplicationResourceUsage(1 * GB, application_1);
    // task_1_0 (1G) application_0 moved to b2 with max running app 1 so it is
    // not scheduled
    checkNodeResourceUsage(1 * GB, nm_0);
    checkNodeResourceUsage(0 * GB, nm_1);
    // lets move application_0 to a queue where it can run
    scheduler.moveApplication(application_0.getApplicationId(), "a2");
    application_0.schedule();
    nodeUpdate(nm_1);
    // Get allocations from the scheduler
    // task_0_0
    application_0.schedule();
    checkApplicationResourceUsage(3 * GB, application_0);
    checkNodeResourceUsage(1 * GB, nm_0);
    checkNodeResourceUsage(3 * GB, nm_1);
}

Example 7 with Task

use of org.apache.hadoop.yarn.server.resourcemanager.Task in project hadoop by apache.

the class TestCapacityScheduler method testMoveAppViolateQueueState.

@Test(expected = YarnException.class)
public void testMoveAppViolateQueueState() throws Exception {
    resourceManager = new ResourceManager() {

        @Override
        protected RMNodeLabelsManager createNodeLabelManager() {
            RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
            mgr.init(getConfig());
            return mgr;
        }
    };
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    setupQueueConfiguration(csConf);
    StringBuilder qState = new StringBuilder();
    qState.append(CapacitySchedulerConfiguration.PREFIX).append(B).append(CapacitySchedulerConfiguration.DOT).append(CapacitySchedulerConfiguration.STATE);
    csConf.set(qState.toString(), QueueState.STOPPED.name());
    YarnConfiguration conf = new YarnConfiguration(csConf);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    resourceManager.init(conf);
    resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
    resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
    ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
    mockContext = mock(RMContext.class);
    when(mockContext.getConfigurationProvider()).thenReturn(new LocalConfigurationProvider());
    ResourceScheduler scheduler = resourceManager.getResourceScheduler();
    // Register node1
    String host_0 = "host_0";
    NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(6 * GB, 1));
    // ResourceRequest priorities
    Priority priority_0 = Priority.newInstance(0);
    Priority priority_1 = Priority.newInstance(1);
    // Submit application_0
    Application application_0 = new Application("user_0", "a1", resourceManager);
    // app + app attempt event sent to scheduler
    application_0.submit();
    application_0.addNodeManager(host_0, 1234, nm_0);
    Resource capability_0_0 = Resources.createResource(3 * GB, 1);
    application_0.addResourceRequestSpec(priority_1, capability_0_0);
    Resource capability_0_1 = Resources.createResource(2 * GB, 1);
    application_0.addResourceRequestSpec(priority_0, capability_0_1);
    Task task_0_0 = new Task(application_0, priority_1, new String[] { host_0 });
    application_0.addTask(task_0_0);
    // Send resource requests to the scheduler
    // allocate
    application_0.schedule();
    // task_0_0 allocated
    nodeUpdate(nm_0);
    // Get allocations from the scheduler
    // task_0_0
    application_0.schedule();
    checkApplicationResourceUsage(3 * GB, application_0);
    checkNodeResourceUsage(3 * GB, nm_0);
    // b2 queue contains 3GB consumption app,
    // add another 3GB will hit max capacity limit on queue b
    scheduler.moveApplication(application_0.getApplicationId(), "b1");
}

Example 8 with Task

use of org.apache.hadoop.yarn.server.resourcemanager.Task in project hadoop by apache.

the class TestCapacityScheduler method testCapacityScheduler.

@Test
public void testCapacityScheduler() throws Exception {
    LOG.info("--- START: testCapacityScheduler ---");
    // Register node1
    String host_0 = "host_0";
    org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(4 * GB, 1));
    // Register node2
    String host_1 = "host_1";
    org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_1 = registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(2 * GB, 1));
    // ResourceRequest priorities
    Priority priority_0 = Priority.newInstance(0);
    Priority priority_1 = Priority.newInstance(1);
    // Submit an application
    Application application_0 = new Application("user_0", "a1", resourceManager);
    application_0.submit();
    application_0.addNodeManager(host_0, 1234, nm_0);
    application_0.addNodeManager(host_1, 1234, nm_1);
    Resource capability_0_0 = Resources.createResource(1 * GB, 1);
    application_0.addResourceRequestSpec(priority_1, capability_0_0);
    Resource capability_0_1 = Resources.createResource(2 * GB, 1);
    application_0.addResourceRequestSpec(priority_0, capability_0_1);
    Task task_0_0 = new Task(application_0, priority_1, new String[] { host_0, host_1 });
    application_0.addTask(task_0_0);
    // Submit another application
    Application application_1 = new Application("user_1", "b2", resourceManager);
    application_1.submit();
    application_1.addNodeManager(host_0, 1234, nm_0);
    application_1.addNodeManager(host_1, 1234, nm_1);
    Resource capability_1_0 = Resources.createResource(3 * GB, 1);
    application_1.addResourceRequestSpec(priority_1, capability_1_0);
    Resource capability_1_1 = Resources.createResource(2 * GB, 1);
    application_1.addResourceRequestSpec(priority_0, capability_1_1);
    Task task_1_0 = new Task(application_1, priority_1, new String[] { host_0, host_1 });
    application_1.addTask(task_1_0);
    // Send resource requests to the scheduler
    application_0.schedule();
    application_1.schedule();
    // Send a heartbeat to kick the tires on the Scheduler
    LOG.info("Kick!");
    // task_0_0 and task_1_0 allocated, used=4G
    nodeUpdate(nm_0);
    // nothing allocated
    nodeUpdate(nm_1);
    // Get allocations from the scheduler
    // task_0_0 
    application_0.schedule();
    checkApplicationResourceUsage(1 * GB, application_0);
    // task_1_0
    application_1.schedule();
    checkApplicationResourceUsage(3 * GB, application_1);
    // task_0_0 (1G) and task_1_0 (3G)
    checkNodeResourceUsage(4 * GB, nm_0);
    // no tasks, 2G available
    checkNodeResourceUsage(0 * GB, nm_1);
    LOG.info("Adding new tasks...");
    Task task_1_1 = new Task(application_1, priority_0, new String[] { ResourceRequest.ANY });
    application_1.addTask(task_1_1);
    application_1.schedule();
    Task task_0_1 = new Task(application_0, priority_0, new String[] { host_0, host_1 });
    application_0.addTask(task_0_1);
    application_0.schedule();
    // Send a heartbeat to kick the tires on the Scheduler
    LOG.info("Sending hb from " + nm_0.getHostName());
    // nothing new, used=4G
    nodeUpdate(nm_0);
    LOG.info("Sending hb from " + nm_1.getHostName());
    // task_0_1 is prefer as locality, used=2G
    nodeUpdate(nm_1);
    // Get allocations from the scheduler
    LOG.info("Trying to allocate...");
    application_0.schedule();
    checkApplicationResourceUsage(1 * GB, application_0);
    application_1.schedule();
    checkApplicationResourceUsage(5 * GB, application_1);
    nodeUpdate(nm_0);
    nodeUpdate(nm_1);
    checkNodeResourceUsage(4 * GB, nm_0);
    checkNodeResourceUsage(2 * GB, nm_1);
    LOG.info("--- END: testCapacityScheduler ---");
}

Example 9 with Task

use of org.apache.hadoop.yarn.server.resourcemanager.Task in project hadoop by apache.

the class TestCapacityScheduler method testResourceUpdateDecommissioningNode.

@Test
public void testResourceUpdateDecommissioningNode() throws Exception {
    // Mock the RMNodeResourceUpdate event handler to update SchedulerNode
    // to have 0 available resource
    RMContext spyContext = Mockito.spy(resourceManager.getRMContext());
    Dispatcher mockDispatcher = mock(AsyncDispatcher.class);
    when(mockDispatcher.getEventHandler()).thenReturn(new EventHandler<Event>() {

        @Override
        public void handle(Event event) {
            if (event instanceof RMNodeResourceUpdateEvent) {
                RMNodeResourceUpdateEvent resourceEvent = (RMNodeResourceUpdateEvent) event;
                resourceManager.getResourceScheduler().getSchedulerNode(resourceEvent.getNodeId()).updateTotalResource(resourceEvent.getResourceOption().getResource());
            }
        }
    });
    Mockito.doReturn(mockDispatcher).when(spyContext).getDispatcher();
    ((CapacityScheduler) resourceManager.getResourceScheduler()).setRMContext(spyContext);
    ((AsyncDispatcher) mockDispatcher).start();
    // Register node
    String host_0 = "host_0";
    org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4));
    // ResourceRequest priorities
    Priority priority_0 = Priority.newInstance(0);
    // Submit an application
    Application application_0 = new Application("user_0", "a1", resourceManager);
    application_0.submit();
    application_0.addNodeManager(host_0, 1234, nm_0);
    Resource capability_0_0 = Resources.createResource(1 * GB, 1);
    application_0.addResourceRequestSpec(priority_0, capability_0_0);
    Task task_0_0 = new Task(application_0, priority_0, new String[] { host_0 });
    application_0.addTask(task_0_0);
    // Send resource requests to the scheduler
    application_0.schedule();
    nodeUpdate(nm_0);
    // Kick off another heartbeat with the node state mocked to decommissioning
    // This should update the schedulernodes to have 0 available resource
    RMNode spyNode = Mockito.spy(resourceManager.getRMContext().getRMNodes().get(nm_0.getNodeId()));
    when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING);
    resourceManager.getResourceScheduler().handle(new NodeUpdateSchedulerEvent(spyNode));
    // Get allocations from the scheduler
    application_0.schedule();
    // Check the used resource is 1 GB 1 core
    Assert.assertEquals(1 * GB, nm_0.getUsed().getMemorySize());
    Resource usedResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getAllocatedResource();
    Assert.assertEquals(usedResource.getMemorySize(), 1 * GB);
    Assert.assertEquals(usedResource.getVirtualCores(), 1);
    // Check total resource of scheduler node is also changed to 1 GB 1 core
    Resource totalResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getTotalResource();
    Assert.assertEquals(totalResource.getMemorySize(), 1 * GB);
    Assert.assertEquals(totalResource.getVirtualCores(), 1);
    // Check the available resource is 0/0
    Resource availableResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource();
    Assert.assertEquals(availableResource.getMemorySize(), 0);
    Assert.assertEquals(availableResource.getVirtualCores(), 0);
}

Example 10 with Task

use of org.apache.hadoop.yarn.server.resourcemanager.Task in project hadoop by apache.

the class TestFifoScheduler method testFifoScheduler.

//  @Test
public void testFifoScheduler() throws Exception {
    LOG.info("--- START: testFifoScheduler ---");
    final int GB = 1024;
    // Register node1
    String host_0 = "host_0";
    org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(4 * GB, 1));
    nm_0.heartbeat();
    // Register node2
    String host_1 = "host_1";
    org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_1 = registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(2 * GB, 1));
    nm_1.heartbeat();
    // ResourceRequest priorities
    Priority priority_0 = Priority.newInstance(0);
    Priority priority_1 = Priority.newInstance(1);
    // Submit an application
    Application application_0 = new Application("user_0", resourceManager);
    application_0.submit();
    application_0.addNodeManager(host_0, 1234, nm_0);
    application_0.addNodeManager(host_1, 1234, nm_1);
    Resource capability_0_0 = Resources.createResource(GB);
    application_0.addResourceRequestSpec(priority_1, capability_0_0);
    Resource capability_0_1 = Resources.createResource(2 * GB);
    application_0.addResourceRequestSpec(priority_0, capability_0_1);
    Task task_0_0 = new Task(application_0, priority_1, new String[] { host_0, host_1 });
    application_0.addTask(task_0_0);
    // Submit another application
    Application application_1 = new Application("user_1", resourceManager);
    application_1.submit();
    application_1.addNodeManager(host_0, 1234, nm_0);
    application_1.addNodeManager(host_1, 1234, nm_1);
    Resource capability_1_0 = Resources.createResource(3 * GB);
    application_1.addResourceRequestSpec(priority_1, capability_1_0);
    Resource capability_1_1 = Resources.createResource(4 * GB);
    application_1.addResourceRequestSpec(priority_0, capability_1_1);
    Task task_1_0 = new Task(application_1, priority_1, new String[] { host_0, host_1 });
    application_1.addTask(task_1_0);
    // Send resource requests to the scheduler
    LOG.info("Send resource requests to the scheduler");
    application_0.schedule();
    application_1.schedule();
    // Send a heartbeat to kick the tires on the Scheduler
    LOG.info("Send a heartbeat to kick the tires on the Scheduler... " + "nm0 -> task_0_0 and task_1_0 allocated, used=4G " + "nm1 -> nothing allocated");
    // task_0_0 and task_1_0 allocated, used=4G
    nm_0.heartbeat();
    // nothing allocated
    nm_1.heartbeat();
    // Get allocations from the scheduler
    // task_0_0 
    application_0.schedule();
    checkApplicationResourceUsage(GB, application_0);
    // task_1_0
    application_1.schedule();
    checkApplicationResourceUsage(3 * GB, application_1);
    nm_0.heartbeat();
    nm_1.heartbeat();
    // task_0_0 (1G) and task_1_0 (3G)
    checkNodeResourceUsage(4 * GB, nm_0);
    // no tasks, 2G available
    checkNodeResourceUsage(0 * GB, nm_1);
    LOG.info("Adding new tasks...");
    Task task_1_1 = new Task(application_1, priority_1, new String[] { ResourceRequest.ANY });
    application_1.addTask(task_1_1);
    Task task_1_2 = new Task(application_1, priority_1, new String[] { ResourceRequest.ANY });
    application_1.addTask(task_1_2);
    Task task_1_3 = new Task(application_1, priority_0, new String[] { ResourceRequest.ANY });
    application_1.addTask(task_1_3);
    application_1.schedule();
    Task task_0_1 = new Task(application_0, priority_1, new String[] { host_0, host_1 });
    application_0.addTask(task_0_1);
    Task task_0_2 = new Task(application_0, priority_1, new String[] { host_0, host_1 });
    application_0.addTask(task_0_2);
    Task task_0_3 = new Task(application_0, priority_0, new String[] { ResourceRequest.ANY });
    application_0.addTask(task_0_3);
    application_0.schedule();
    // Send a heartbeat to kick the tires on the Scheduler
    LOG.info("Sending hb from " + nm_0.getHostName());
    // nothing new, used=4G
    nm_0.heartbeat();
    LOG.info("Sending hb from " + nm_1.getHostName());
    // task_0_3, used=2G
    nm_1.heartbeat();
    // Get allocations from the scheduler
    LOG.info("Trying to allocate...");
    application_0.schedule();
    checkApplicationResourceUsage(3 * GB, application_0);
    application_1.schedule();
    checkApplicationResourceUsage(3 * GB, application_1);
    nm_0.heartbeat();
    nm_1.heartbeat();
    checkNodeResourceUsage(4 * GB, nm_0);
    checkNodeResourceUsage(2 * GB, nm_1);
    // Complete tasks
    LOG.info("Finishing up task_0_0");
    // Now task_0_1
    application_0.finishTask(task_0_0);
    application_0.schedule();
    application_1.schedule();
    nm_0.heartbeat();
    nm_1.heartbeat();
    checkApplicationResourceUsage(3 * GB, application_0);
    checkApplicationResourceUsage(3 * GB, application_1);
    checkNodeResourceUsage(4 * GB, nm_0);
    checkNodeResourceUsage(2 * GB, nm_1);
    LOG.info("Finishing up task_1_0");
    // Now task_0_2
    application_1.finishTask(task_1_0);
    // final overcommit for app0 caused here
    application_0.schedule();
    application_1.schedule();
    // final overcommit for app0 occurs here
    nm_0.heartbeat();
    nm_1.heartbeat();
    checkApplicationResourceUsage(4 * GB, application_0);
    checkApplicationResourceUsage(0 * GB, application_1);
    //checkNodeResourceUsage(1*GB, nm_0);  // final over-commit -> rm.node->1G, test.node=2G
    checkNodeResourceUsage(2 * GB, nm_1);
    LOG.info("Finishing up task_0_3");
    // No more
    application_0.finishTask(task_0_3);
    application_0.schedule();
    application_1.schedule();
    nm_0.heartbeat();
    nm_1.heartbeat();
    checkApplicationResourceUsage(2 * GB, application_0);
    checkApplicationResourceUsage(0 * GB, application_1);
    //checkNodeResourceUsage(2*GB, nm_0);  // final over-commit, rm.node->1G, test.node->2G
    checkNodeResourceUsage(0 * GB, nm_1);
    LOG.info("Finishing up task_0_1");
    application_0.finishTask(task_0_1);
    application_0.schedule();
    application_1.schedule();
    nm_0.heartbeat();
    nm_1.heartbeat();
    checkApplicationResourceUsage(1 * GB, application_0);
    checkApplicationResourceUsage(0 * GB, application_1);
    LOG.info("Finishing up task_0_2");
    // now task_1_3 can go!
    application_0.finishTask(task_0_2);
    application_0.schedule();
    application_1.schedule();
    nm_0.heartbeat();
    nm_1.heartbeat();
    checkApplicationResourceUsage(0 * GB, application_0);
    checkApplicationResourceUsage(4 * GB, application_1);
    LOG.info("Finishing up task_1_3");
    // now task_1_1
    application_1.finishTask(task_1_3);
    application_0.schedule();
    application_1.schedule();
    nm_0.heartbeat();
    nm_1.heartbeat();
    checkApplicationResourceUsage(0 * GB, application_0);
    checkApplicationResourceUsage(3 * GB, application_1);
    LOG.info("Finishing up task_1_1");
    application_1.finishTask(task_1_1);
    application_0.schedule();
    application_1.schedule();
    nm_0.heartbeat();
    nm_1.heartbeat();
    checkApplicationResourceUsage(0 * GB, application_0);
    checkApplicationResourceUsage(3 * GB, application_1);
    LOG.info("--- END: testFifoScheduler ---");
}