Examples with YarnScheduler org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler used on opensource projects

Example 11 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class RMWebServices method getActivities.

@GET
@Path("/scheduler/activities")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public ActivitiesInfo getActivities(@Context HttpServletRequest hsr, @QueryParam("nodeId") String nodeId) {
    YarnScheduler scheduler = rm.getRMContext().getScheduler();
    if (scheduler instanceof AbstractYarnScheduler) {
        String errMessage = "";
        AbstractYarnScheduler abstractYarnScheduler = (AbstractYarnScheduler) scheduler;
        ActivitiesManager activitiesManager = abstractYarnScheduler.getActivitiesManager();
        if (null == activitiesManager) {
            errMessage = "Not Capacity Scheduler";
            return new ActivitiesInfo(errMessage, nodeId);
        }
        List<FiCaSchedulerNode> nodeList = abstractYarnScheduler.getNodeTracker().getAllNodes();
        boolean illegalInput = false;
        if (nodeList.size() == 0) {
            illegalInput = true;
            errMessage = "No node manager running in the cluster";
        } else {
            if (nodeId != null) {
                String hostName = nodeId;
                String portName = "";
                if (nodeId.contains(":")) {
                    int index = nodeId.indexOf(":");
                    hostName = nodeId.substring(0, index);
                    portName = nodeId.substring(index + 1);
                }
                boolean correctNodeId = false;
                for (FiCaSchedulerNode node : nodeList) {
                    if ((portName.equals("") && node.getRMNode().getHostName().equals(hostName)) || (!portName.equals("") && node.getRMNode().getHostName().equals(hostName) && String.valueOf(node.getRMNode().getCommandPort()).equals(portName))) {
                        correctNodeId = true;
                        nodeId = node.getNodeID().toString();
                        break;
                    }
                }
                if (!correctNodeId) {
                    illegalInput = true;
                    errMessage = "Cannot find node manager with given node id";
                }
            }
        }
        if (!illegalInput) {
            activitiesManager.recordNextNodeUpdateActivities(nodeId);
            return activitiesManager.getActivitiesInfo(nodeId);
        }
        // Return a activities info with error message
        return new ActivitiesInfo(errMessage, nodeId);
    }
    return null;
}

Example 12 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestClientRMService method testConcurrentAppSubmit.

@Test(timeout = 4000)
public void testConcurrentAppSubmit() throws IOException, InterruptedException, BrokenBarrierException, YarnException {
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    RMStateStore stateStore = mock(RMStateStore.class);
    when(rmContext.getStateStore()).thenReturn(stateStore);
    RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null, mock(ApplicationACLsManager.class), new Configuration());
    final ApplicationId appId1 = getApplicationId(100);
    final ApplicationId appId2 = getApplicationId(101);
    final SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest(appId1, null, null);
    final SubmitApplicationRequest submitRequest2 = mockSubmitAppRequest(appId2, null, null);
    final CyclicBarrier startBarrier = new CyclicBarrier(2);
    final CyclicBarrier endBarrier = new CyclicBarrier(2);
    EventHandler<Event> eventHandler = new EventHandler<Event>() {

        @Override
        public void handle(Event rawEvent) {
            if (rawEvent instanceof RMAppEvent) {
                RMAppEvent event = (RMAppEvent) rawEvent;
                if (event.getApplicationId().equals(appId1)) {
                    try {
                        startBarrier.await();
                        endBarrier.await();
                    } catch (BrokenBarrierException e) {
                        LOG.warn("Broken Barrier", e);
                    } catch (InterruptedException e) {
                        LOG.warn("Interrupted while awaiting barriers", e);
                    }
                }
            }
        }
    };
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(eventHandler);
    doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext).getRMTimelineCollectorManager();
    final ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, appManager, null, null, null);
    rmService.init(new Configuration());
    // submit an app and wait for it to block while in app submission
    Thread t = new Thread() {

        @Override
        public void run() {
            try {
                rmService.submitApplication(submitRequest1);
            } catch (YarnException | IOException e) {
            }
        }
    };
    t.start();
    // submit another app, so go through while the first app is blocked
    startBarrier.await();
    rmService.submitApplication(submitRequest2);
    endBarrier.await();
    t.join();
}

Example 13 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestClientRMService method testGetApplicationResourceUsageReportDummy.

@Test
public void testGetApplicationResourceUsageReportDummy() throws YarnException, IOException {
    ApplicationAttemptId attemptId = getApplicationAttemptId(1);
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(new EventHandler<Event>() {

        public void handle(Event event) {
        }
    });
    ApplicationSubmissionContext asContext = mock(ApplicationSubmissionContext.class);
    YarnConfiguration config = new YarnConfiguration();
    RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId, rmContext, yarnScheduler, null, asContext, config, null, null);
    ApplicationResourceUsageReport report = rmAppAttemptImpl.getApplicationResourceUsageReport();
    assertEquals(report, RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
}

Example 14 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestNodeLabelContainerAllocation method checkTaskContainersHost.

private void checkTaskContainersHost(ApplicationAttemptId attemptId, ContainerId containerId, ResourceManager rm, String host) {
    YarnScheduler scheduler = rm.getRMContext().getScheduler();
    SchedulerAppReport appReport = scheduler.getSchedulerAppInfo(attemptId);
    Assert.assertTrue(appReport.getLiveContainers().size() > 0);
    for (RMContainer c : appReport.getLiveContainers()) {
        if (c.getContainerId().equals(containerId)) {
            Assert.assertEquals(host, c.getAllocatedNode().getHost());
        }
    }
}

Example 15 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestWorkPreservingRMRestart method testAMContainerStatusWithRMRestart.

@Test(timeout = 30000)
public void testAMContainerStatusWithRMRestart() throws Exception {
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    rm1 = new MockRM(conf, memStore);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
    nm1.registerNode();
    RMApp app1_1 = rm1.submitApp(1024);
    MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1);
    RMAppAttempt attempt0 = app1_1.getCurrentAppAttempt();
    YarnScheduler scheduler = rm1.getResourceScheduler();
    Assert.assertTrue(scheduler.getRMContainer(attempt0.getMasterContainer().getId()).isAMContainer());
    // Re-start RM
    rm2 = new MockRM(conf, memStore);
    rm2.start();
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());
    List<NMContainerStatus> am1_1Containers = createNMContainerStatusForApp(am1_1);
    nm1.registerNode(am1_1Containers, null);
    // Wait for RM to settle down on recovering containers;
    waitForNumContainersToRecover(2, rm2, am1_1.getApplicationAttemptId());
    scheduler = rm2.getResourceScheduler();
    Assert.assertTrue(scheduler.getRMContainer(attempt0.getMasterContainer().getId()).isAMContainer());
}