Examples with YarnScheduler org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler used on opensource projects

Example 1 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class RMWebServices method getAppActivities.

@GET
@Path("/scheduler/app-activities")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public AppActivitiesInfo getAppActivities(@Context HttpServletRequest hsr, @QueryParam("appId") String appId, @QueryParam("maxTime") String time) {
    YarnScheduler scheduler = rm.getRMContext().getScheduler();
    if (scheduler instanceof AbstractYarnScheduler) {
        AbstractYarnScheduler abstractYarnScheduler = (AbstractYarnScheduler) scheduler;
        ActivitiesManager activitiesManager = abstractYarnScheduler.getActivitiesManager();
        if (null == activitiesManager) {
            String errMessage = "Not Capacity Scheduler";
            return new AppActivitiesInfo(errMessage, appId);
        }
        if (appId == null) {
            String errMessage = "Must provide an application Id";
            return new AppActivitiesInfo(errMessage, null);
        }
        double maxTime = 3.0;
        if (time != null) {
            if (time.contains(".")) {
                maxTime = Double.parseDouble(time);
            } else {
                maxTime = Double.parseDouble(time + ".0");
            }
        }
        ApplicationId applicationId;
        try {
            applicationId = ApplicationId.fromString(appId);
            activitiesManager.turnOnAppActivitiesRecording(applicationId, maxTime);
            AppActivitiesInfo appActivitiesInfo = activitiesManager.getAppActivitiesInfo(applicationId);
            return appActivitiesInfo;
        } catch (Exception e) {
            String errMessage = "Cannot find application with given appId";
            return new AppActivitiesInfo(errMessage, appId);
        }
    }
    return null;
}

Example 2 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestClientRMService method testGetQueueInfo.

@Test
public void testGetQueueInfo() throws Exception {
    YarnScheduler yarnScheduler = mock(YarnScheduler.class);
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
    QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
    when(mockQueueACLsManager.checkAccess(any(UserGroupInformation.class), any(QueueACL.class), any(RMApp.class), any(String.class), any())).thenReturn(true);
    when(mockAclsManager.checkAccess(any(UserGroupInformation.class), any(ApplicationAccessType.class), anyString(), any(ApplicationId.class))).thenReturn(true);
    ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, null, mockAclsManager, mockQueueACLsManager, null);
    GetQueueInfoRequest request = recordFactory.newRecordInstance(GetQueueInfoRequest.class);
    request.setQueueName("testqueue");
    request.setIncludeApplications(true);
    GetQueueInfoResponse queueInfo = rmService.getQueueInfo(request);
    List<ApplicationReport> applications = queueInfo.getQueueInfo().getApplications();
    Assert.assertEquals(2, applications.size());
    request.setQueueName("nonexistentqueue");
    request.setIncludeApplications(true);
    // should not throw exception on nonexistent queue
    queueInfo = rmService.getQueueInfo(request);
    // Case where user does not have application access
    ApplicationACLsManager mockAclsManager1 = mock(ApplicationACLsManager.class);
    QueueACLsManager mockQueueACLsManager1 = mock(QueueACLsManager.class);
    when(mockQueueACLsManager1.checkAccess(any(UserGroupInformation.class), any(QueueACL.class), any(RMApp.class), any(String.class), any())).thenReturn(false);
    when(mockAclsManager1.checkAccess(any(UserGroupInformation.class), any(ApplicationAccessType.class), anyString(), any(ApplicationId.class))).thenReturn(false);
    ClientRMService rmService1 = new ClientRMService(rmContext, yarnScheduler, null, mockAclsManager1, mockQueueACLsManager1, null);
    request.setQueueName("testqueue");
    request.setIncludeApplications(true);
    GetQueueInfoResponse queueInfo1 = rmService1.getQueueInfo(request);
    List<ApplicationReport> applications1 = queueInfo1.getQueueInfo().getApplications();
    Assert.assertEquals(0, applications1.size());
}

Example 3 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestClientRMService method mockYarnScheduler.

private static YarnScheduler mockYarnScheduler() throws YarnException {
    YarnScheduler yarnScheduler = mock(YarnScheduler.class);
    when(yarnScheduler.getMinimumResourceCapability()).thenReturn(Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
    when(yarnScheduler.getMaximumResourceCapability()).thenReturn(Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
    when(yarnScheduler.getAppsInQueue(QUEUE_1)).thenReturn(Arrays.asList(getApplicationAttemptId(101), getApplicationAttemptId(102)));
    when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn(Arrays.asList(getApplicationAttemptId(103)));
    ApplicationAttemptId attemptId = getApplicationAttemptId(1);
    when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null);
    ResourceCalculator rs = mock(ResourceCalculator.class);
    when(yarnScheduler.getResourceCalculator()).thenReturn(rs);
    when(yarnScheduler.checkAndGetApplicationPriority(any(Priority.class), any(UserGroupInformation.class), anyString(), any(ApplicationId.class))).thenReturn(Priority.newInstance(0));
    return yarnScheduler;
}

Example 4 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestClientRMService method testGetApplications.

@Test
public void testGetApplications() throws IOException, YarnException {
    /**
     * 1. Submit 3 applications alternately in two queues
     * 2. Test each of the filters
     */
    // Basic setup
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    RMStateStore stateStore = mock(RMStateStore.class);
    when(rmContext.getStateStore()).thenReturn(stateStore);
    doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext).getRMTimelineCollectorManager();
    RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null, mock(ApplicationACLsManager.class), new Configuration());
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(new EventHandler<Event>() {

        public void handle(Event event) {
        }
    });
    ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
    QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
    when(mockQueueACLsManager.checkAccess(any(UserGroupInformation.class), any(QueueACL.class), any(RMApp.class), any(String.class), any())).thenReturn(true);
    ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, appManager, mockAclsManager, mockQueueACLsManager, null);
    rmService.init(new Configuration());
    // Initialize appnames and queues
    String[] queues = { QUEUE_1, QUEUE_2 };
    String[] appNames = { MockApps.newAppName(), MockApps.newAppName(), MockApps.newAppName() };
    ApplicationId[] appIds = { getApplicationId(101), getApplicationId(102), getApplicationId(103) };
    List<String> tags = Arrays.asList("Tag1", "Tag2", "Tag3");
    long[] submitTimeMillis = new long[3];
    // Submit applications
    for (int i = 0; i < appIds.length; i++) {
        ApplicationId appId = appIds[i];
        when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId)).thenReturn(true);
        SubmitApplicationRequest submitRequest = mockSubmitAppRequest(appId, appNames[i], queues[i % queues.length], new HashSet<String>(tags.subList(0, i + 1)));
        rmService.submitApplication(submitRequest);
        submitTimeMillis[i] = System.currentTimeMillis();
    }
    // Test different cases of ClientRMService#getApplications()
    GetApplicationsRequest request = GetApplicationsRequest.newInstance();
    assertEquals("Incorrect total number of apps", 6, rmService.getApplications(request).getApplicationList().size());
    // Check limit
    request.setLimit(1L);
    assertEquals("Failed to limit applications", 1, rmService.getApplications(request).getApplicationList().size());
    // Check start range
    request = GetApplicationsRequest.newInstance();
    request.setStartRange(submitTimeMillis[0], System.currentTimeMillis());
    // 2 applications are submitted after first timeMills
    assertEquals("Incorrect number of matching start range", 2, rmService.getApplications(request).getApplicationList().size());
    // 1 application is submitted after the second timeMills
    request.setStartRange(submitTimeMillis[1], System.currentTimeMillis());
    assertEquals("Incorrect number of matching start range", 1, rmService.getApplications(request).getApplicationList().size());
    // no application is submitted after the third timeMills
    request.setStartRange(submitTimeMillis[2], System.currentTimeMillis());
    assertEquals("Incorrect number of matching start range", 0, rmService.getApplications(request).getApplicationList().size());
    // Check queue
    request = GetApplicationsRequest.newInstance();
    Set<String> queueSet = new HashSet<String>();
    request.setQueues(queueSet);
    queueSet.add(queues[0]);
    assertEquals("Incorrect number of applications in queue", 2, rmService.getApplications(request).getApplicationList().size());
    assertEquals("Incorrect number of applications in queue", 2, rmService.getApplications(request, false).getApplicationList().size());
    queueSet.add(queues[1]);
    assertEquals("Incorrect number of applications in queue", 3, rmService.getApplications(request).getApplicationList().size());
    // Check user
    request = GetApplicationsRequest.newInstance();
    Set<String> userSet = new HashSet<String>();
    request.setUsers(userSet);
    userSet.add("random-user-name");
    assertEquals("Incorrect number of applications for user", 0, rmService.getApplications(request).getApplicationList().size());
    userSet.add(UserGroupInformation.getCurrentUser().getShortUserName());
    assertEquals("Incorrect number of applications for user", 3, rmService.getApplications(request).getApplicationList().size());
    // Check tags
    request = GetApplicationsRequest.newInstance(ApplicationsRequestScope.ALL, null, null, null, null, null, null, null, null);
    Set<String> tagSet = new HashSet<String>();
    request.setApplicationTags(tagSet);
    assertEquals("Incorrect number of matching tags", 6, rmService.getApplications(request).getApplicationList().size());
    tagSet = Sets.newHashSet(tags.get(0));
    request.setApplicationTags(tagSet);
    assertEquals("Incorrect number of matching tags", 3, rmService.getApplications(request).getApplicationList().size());
    tagSet = Sets.newHashSet(tags.get(1));
    request.setApplicationTags(tagSet);
    assertEquals("Incorrect number of matching tags", 2, rmService.getApplications(request).getApplicationList().size());
    tagSet = Sets.newHashSet(tags.get(2));
    request.setApplicationTags(tagSet);
    assertEquals("Incorrect number of matching tags", 1, rmService.getApplications(request).getApplicationList().size());
    // Check scope
    request = GetApplicationsRequest.newInstance(ApplicationsRequestScope.VIEWABLE);
    assertEquals("Incorrect number of applications for the scope", 6, rmService.getApplications(request).getApplicationList().size());
    request = GetApplicationsRequest.newInstance(ApplicationsRequestScope.OWN);
    assertEquals("Incorrect number of applications for the scope", 3, rmService.getApplications(request).getApplicationList().size());
}

Example 5 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestClientRMService method createRMService.

public ClientRMService createRMService() throws IOException, YarnException {
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext, yarnScheduler);
    when(rmContext.getRMApps()).thenReturn(apps);
    when(rmContext.getYarnConfiguration()).thenReturn(new Configuration());
    RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null, mock(ApplicationACLsManager.class), new Configuration());
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(new EventHandler<Event>() {

        public void handle(Event event) {
        }
    });
    ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
    QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
    when(mockQueueACLsManager.checkAccess(any(UserGroupInformation.class), any(QueueACL.class), any(RMApp.class), any(String.class), any())).thenReturn(true);
    return new ClientRMService(rmContext, yarnScheduler, appManager, mockAclsManager, mockQueueACLsManager, null);
}