Examples with CapacityScheduler org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler used on opensource projects

Example 26 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class TestFairOrderingPolicy method testSizeBasedWeightNotAffectAppActivation.

@Test
public void testSizeBasedWeightNotAffectAppActivation() throws Exception {
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    // Define top-level queues
    String queuePath = CapacitySchedulerConfiguration.ROOT + ".default";
    csConf.setOrderingPolicy(queuePath, CapacitySchedulerConfiguration.FAIR_APP_ORDERING_POLICY);
    csConf.setOrderingPolicyParameter(queuePath, FairOrderingPolicy.ENABLE_SIZE_BASED_WEIGHT, "true");
    csConf.setMaximumApplicationMasterResourcePerQueuePercent(queuePath, 0.1f);
    // inject node label manager
    MockRM rm = new MockRM(csConf);
    rm.start();
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    // Get LeafQueue
    LeafQueue lq = (LeafQueue) cs.getQueue("default");
    OrderingPolicy<FiCaSchedulerApp> policy = lq.getOrderingPolicy();
    Assert.assertTrue(policy instanceof FairOrderingPolicy);
    Assert.assertTrue(((FairOrderingPolicy<FiCaSchedulerApp>) policy).getSizeBasedWeight());
    rm.registerNode("h1:1234", 10 * GB);
    // Submit 4 apps
    rm.submitApp(1 * GB, "app", "user", null, "default");
    rm.submitApp(1 * GB, "app", "user", null, "default");
    rm.submitApp(1 * GB, "app", "user", null, "default");
    rm.submitApp(1 * GB, "app", "user", null, "default");
    Assert.assertEquals(1, lq.getNumActiveApplications());
    Assert.assertEquals(3, lq.getNumPendingApplications());
    // Try allocate once, #active-apps and #pending-apps should be still correct
    cs.handle(new NodeUpdateSchedulerEvent(rm.getRMContext().getRMNodes().get(NodeId.newInstance("h1", 1234))));
    Assert.assertEquals(1, lq.getNumActiveApplications());
    Assert.assertEquals(3, lq.getNumPendingApplications());
}

Example 27 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class TestCapacitySchedulerPlanFollower method setUp.

@Before
public void setUp() throws Exception {
    CapacityScheduler spyCs = new CapacityScheduler();
    cs = spy(spyCs);
    scheduler = cs;
    rmContext = TestUtils.getMockRMContext();
    spyRMContext = spy(rmContext);
    ConcurrentMap<ApplicationId, RMApp> spyApps = spy(new ConcurrentHashMap<ApplicationId, RMApp>());
    RMApp rmApp = mock(RMApp.class);
    RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
    when(rmApp.getRMAppAttempt((ApplicationAttemptId) Matchers.any())).thenReturn(rmAppAttempt);
    when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt);
    Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
    Mockito.doReturn(true).when(spyApps).containsKey((ApplicationId) Matchers.any());
    when(spyRMContext.getRMApps()).thenReturn(spyApps);
    when(spyRMContext.getScheduler()).thenReturn(scheduler);
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    ReservationSystemTestUtil.setupQueueConfiguration(csConf);
    cs.setConf(csConf);
    csContext = mock(CapacitySchedulerContext.class);
    when(csContext.getConfiguration()).thenReturn(csConf);
    when(csContext.getConf()).thenReturn(csConf);
    when(csContext.getMinimumResourceCapability()).thenReturn(minAlloc);
    when(csContext.getMaximumResourceCapability()).thenReturn(maxAlloc);
    when(csContext.getClusterResource()).thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32));
    when(scheduler.getClusterResource()).thenReturn(Resources.createResource(125 * GB, 125));
    when(csContext.getResourceCalculator()).thenReturn(new DefaultResourceCalculator());
    RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(csConf);
    containerTokenSecretManager.rollMasterKey();
    when(csContext.getContainerTokenSecretManager()).thenReturn(containerTokenSecretManager);
    cs.setRMContext(spyRMContext);
    cs.init(csConf);
    cs.start();
    setupPlanFollower();
}

Example 28 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class TestRMWebServicesSchedulerActivities method testAppReserveNewContainer.

@Test
public void testAppReserveNewContainer() throws Exception {
    //Start RM so that it accepts app submissions
    rm.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 4 * 1024, rm.getResourceTrackerService());
    MockNM nm2 = new MockNM("127.0.0.2:1234", 4 * 1024, rm.getResourceTrackerService());
    nm1.registerNode();
    nm2.registerNode();
    try {
        RMApp app1 = rm.submitApp(10, "app1", "user1", null, "b1");
        MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
        RMApp app2 = rm.submitApp(10, "app2", "user1", null, "b2");
        MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2);
        am1.allocate(Arrays.asList(ResourceRequest.newInstance(Priority.UNDEFINED, "*", Resources.createResource(4096), 10)), null);
        // Reserve new container
        WebResource r = resource();
        MultivaluedMapImpl params = new MultivaluedMapImpl();
        params.add("appId", app1.getApplicationId().toString());
        ClientResponse response = r.path("ws").path("v1").path("cluster").path("scheduler/app-activities").queryParams(params).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
        JSONObject json = response.getEntity(JSONObject.class);
        nm2.nodeHeartbeat(true);
        Thread.sleep(1000);
        response = r.path("ws").path("v1").path("cluster").path("scheduler/app-activities").queryParams(params).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
        json = response.getEntity(JSONObject.class);
        verifyNumberOfAllocations(json, 1);
        // Do a node heartbeat again without releasing container from app2
        r = resource();
        params = new MultivaluedMapImpl();
        params.add("appId", app1.getApplicationId().toString());
        response = r.path("ws").path("v1").path("cluster").path("scheduler/app-activities").queryParams(params).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
        json = response.getEntity(JSONObject.class);
        nm2.nodeHeartbeat(true);
        Thread.sleep(1000);
        response = r.path("ws").path("v1").path("cluster").path("scheduler/app-activities").queryParams(params).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
        json = response.getEntity(JSONObject.class);
        verifyNumberOfAllocations(json, 2);
        // Finish application 2
        CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
        ContainerId containerId = ContainerId.newContainerId(am2.getApplicationAttemptId(), 1);
        cs.completedContainer(cs.getRMContainer(containerId), ContainerStatus.newInstance(containerId, ContainerState.COMPLETE, "", 0), RMContainerEventType.FINISHED);
        // Do a node heartbeat again
        r = resource();
        params = new MultivaluedMapImpl();
        params.add("appId", app1.getApplicationId().toString());
        response = r.path("ws").path("v1").path("cluster").path("scheduler/app-activities").queryParams(params).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
        json = response.getEntity(JSONObject.class);
        nm2.nodeHeartbeat(true);
        Thread.sleep(1000);
        response = r.path("ws").path("v1").path("cluster").path("scheduler/app-activities").queryParams(params).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
        json = response.getEntity(JSONObject.class);
        verifyNumberOfAllocations(json, 3);
    } finally {
        rm.stop();
    }
}

Example 29 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class TestRMWebServicesSchedulerActivities method testReserveNewContainer.

@Test
public void testReserveNewContainer() throws Exception {
    //Start RM so that it accepts app submissions
    rm.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 4 * 1024, rm.getResourceTrackerService());
    MockNM nm2 = new MockNM("127.0.0.2:1234", 4 * 1024, rm.getResourceTrackerService());
    nm1.registerNode();
    nm2.registerNode();
    try {
        RMApp app1 = rm.submitApp(10, "app1", "user1", null, "b1");
        MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
        RMApp app2 = rm.submitApp(10, "app2", "user1", null, "b2");
        MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2);
        am1.allocate(Arrays.asList(ResourceRequest.newInstance(Priority.UNDEFINED, "*", Resources.createResource(4096), 10)), null);
        // Reserve new container
        WebResource r = resource();
        MultivaluedMapImpl params = new MultivaluedMapImpl();
        params.add("nodeId", "127.0.0.2");
        ClientResponse response = r.path("ws").path("v1").path("cluster").path("scheduler/activities").queryParams(params).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
        JSONObject json = response.getEntity(JSONObject.class);
        nm2.nodeHeartbeat(true);
        Thread.sleep(1000);
        response = r.path("ws").path("v1").path("cluster").path("scheduler/activities").queryParams(params).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
        json = response.getEntity(JSONObject.class);
        verifyNumberOfAllocations(json, 1);
        verifyQueueOrder(json.getJSONObject("allocations"), "root-a-b-b3-b1");
        JSONObject allocations = json.getJSONObject("allocations");
        verifyStateOfAllocations(allocations, "finalAllocationState", "RESERVED");
        // Do a node heartbeat again without releasing container from app2
        r = resource();
        params = new MultivaluedMapImpl();
        params.add("nodeId", "127.0.0.2");
        response = r.path("ws").path("v1").path("cluster").path("scheduler/activities").queryParams(params).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
        json = response.getEntity(JSONObject.class);
        nm2.nodeHeartbeat(true);
        Thread.sleep(1000);
        response = r.path("ws").path("v1").path("cluster").path("scheduler/activities").queryParams(params).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
        json = response.getEntity(JSONObject.class);
        verifyNumberOfAllocations(json, 1);
        verifyQueueOrder(json.getJSONObject("allocations"), "b1");
        allocations = json.getJSONObject("allocations");
        verifyStateOfAllocations(allocations, "finalAllocationState", "SKIPPED");
        // Finish application 2
        CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
        ContainerId containerId = ContainerId.newContainerId(am2.getApplicationAttemptId(), 1);
        cs.completedContainer(cs.getRMContainer(containerId), ContainerStatus.newInstance(containerId, ContainerState.COMPLETE, "", 0), RMContainerEventType.FINISHED);
        // Do a node heartbeat again
        r = resource();
        params = new MultivaluedMapImpl();
        params.add("nodeId", "127.0.0.2");
        response = r.path("ws").path("v1").path("cluster").path("scheduler/activities").queryParams(params).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
        json = response.getEntity(JSONObject.class);
        nm2.nodeHeartbeat(true);
        Thread.sleep(1000);
        response = r.path("ws").path("v1").path("cluster").path("scheduler/activities").queryParams(params).accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
        json = response.getEntity(JSONObject.class);
        verifyNumberOfAllocations(json, 1);
        verifyQueueOrder(json.getJSONObject("allocations"), "b1");
        allocations = json.getJSONObject("allocations");
        verifyStateOfAllocations(allocations, "finalAllocationState", "ALLOCATED_FROM_RESERVED");
    } finally {
        rm.stop();
    }
}

Example 30 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class TestOpportunisticContainerAllocatorAMService method testContainerPromoteAfterContainerComplete.

@Test(timeout = 600000)
public void testContainerPromoteAfterContainerComplete() throws Exception {
    HashMap<NodeId, MockNM> nodes = new HashMap<>();
    MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
    nodes.put(nm1.getNodeId(), nm1);
    MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
    nodes.put(nm2.getNodeId(), nm2);
    nm1.registerNode();
    nm2.registerNode();
    OpportunisticContainerAllocatorAMService amservice = (OpportunisticContainerAllocatorAMService) rm.getApplicationMasterService();
    RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
    ApplicationAttemptId attemptId = app1.getCurrentAppAttempt().getAppAttemptId();
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2);
    ResourceScheduler scheduler = rm.getResourceScheduler();
    RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
    RMNode rmNode2 = rm.getRMContext().getRMNodes().get(nm2.getNodeId());
    nm1.nodeHeartbeat(true);
    nm2.nodeHeartbeat(true);
    ((RMNodeImpl) rmNode1).setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
    ((RMNodeImpl) rmNode2).setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
    OpportunisticContainerContext ctxt = ((CapacityScheduler) scheduler).getApplicationAttempt(attemptId).getOpportunisticContainerContext();
    // Send add and update node events to AM Service.
    amservice.handle(new NodeAddedSchedulerEvent(rmNode1));
    amservice.handle(new NodeAddedSchedulerEvent(rmNode2));
    amservice.handle(new NodeUpdateSchedulerEvent(rmNode1));
    amservice.handle(new NodeUpdateSchedulerEvent(rmNode2));
    // All nodes 1 to 2 will be applicable for scheduling.
    nm1.nodeHeartbeat(true);
    nm2.nodeHeartbeat(true);
    Thread.sleep(1000);
    QueueMetrics metrics = ((CapacityScheduler) scheduler).getRootQueue().getMetrics();
    // Verify Metrics
    verifyMetrics(metrics, 7168, 7, 1024, 1, 1);
    AllocateResponse allocateResponse = am1.allocate(Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), "*", Resources.createResource(1 * GB), 2, true, null, ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true))), null);
    List<Container> allocatedContainers = allocateResponse.getAllocatedContainers();
    Assert.assertEquals(2, allocatedContainers.size());
    Container container = allocatedContainers.get(0);
    MockNM allocNode = nodes.get(container.getNodeId());
    // Start Container in NM
    allocNode.nodeHeartbeat(Arrays.asList(ContainerStatus.newInstance(container.getId(), ExecutionType.OPPORTUNISTIC, ContainerState.RUNNING, "", 0)), true);
    Thread.sleep(200);
    // Verify that container is actually running wrt the RM..
    RMContainer rmContainer = ((CapacityScheduler) scheduler).getApplicationAttempt(container.getId().getApplicationAttemptId()).getRMContainer(container.getId());
    Assert.assertEquals(RMContainerState.RUNNING, rmContainer.getState());
    // Container Completed in the NM
    allocNode.nodeHeartbeat(Arrays.asList(ContainerStatus.newInstance(container.getId(), ExecutionType.OPPORTUNISTIC, ContainerState.COMPLETE, "", 0)), true);
    Thread.sleep(200);
    // Verify that container has been removed..
    rmContainer = ((CapacityScheduler) scheduler).getApplicationAttempt(container.getId().getApplicationAttemptId()).getRMContainer(container.getId());
    Assert.assertNull(rmContainer);
    // Verify Metrics After OPP allocation (Nothing should change)
    verifyMetrics(metrics, 7168, 7, 1024, 1, 1);
    // Send Promotion req... this should result in update error
    // Since the container doesn't exist anymore..
    allocateResponse = am1.sendContainerUpdateRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, container.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE, null, ExecutionType.GUARANTEED)));
    Assert.assertEquals(1, allocateResponse.getCompletedContainersStatuses().size());
    Assert.assertEquals(container.getId(), allocateResponse.getCompletedContainersStatuses().get(0).getContainerId());
    Assert.assertEquals(0, allocateResponse.getUpdatedContainers().size());
    Assert.assertEquals(1, allocateResponse.getUpdateErrors().size());
    Assert.assertEquals("INVALID_CONTAINER_ID", allocateResponse.getUpdateErrors().get(0).getReason());
    Assert.assertEquals(container.getId(), allocateResponse.getUpdateErrors().get(0).getUpdateContainerRequest().getContainerId());
    // Verify Metrics After OPP allocation (Nothing should change again)
    verifyMetrics(metrics, 7168, 7, 1024, 1, 1);
}