Examples with Event org.apache.hadoop.yarn.event.Event used on opensource projects

Example 1 with Event

use of org.apache.hadoop.yarn.event.Event in project hadoop by apache.

the class TestClientRMService method testGetApplications.

@Test
public void testGetApplications() throws IOException, YarnException {
    /**
     * 1. Submit 3 applications alternately in two queues
     * 2. Test each of the filters
     */
    // Basic setup
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    RMStateStore stateStore = mock(RMStateStore.class);
    when(rmContext.getStateStore()).thenReturn(stateStore);
    doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext).getRMTimelineCollectorManager();
    RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null, mock(ApplicationACLsManager.class), new Configuration());
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(new EventHandler<Event>() {

        public void handle(Event event) {
        }
    });
    ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
    QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
    when(mockQueueACLsManager.checkAccess(any(UserGroupInformation.class), any(QueueACL.class), any(RMApp.class), any(String.class), any())).thenReturn(true);
    ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, appManager, mockAclsManager, mockQueueACLsManager, null);
    rmService.init(new Configuration());
    // Initialize appnames and queues
    String[] queues = { QUEUE_1, QUEUE_2 };
    String[] appNames = { MockApps.newAppName(), MockApps.newAppName(), MockApps.newAppName() };
    ApplicationId[] appIds = { getApplicationId(101), getApplicationId(102), getApplicationId(103) };
    List<String> tags = Arrays.asList("Tag1", "Tag2", "Tag3");
    long[] submitTimeMillis = new long[3];
    // Submit applications
    for (int i = 0; i < appIds.length; i++) {
        ApplicationId appId = appIds[i];
        when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId)).thenReturn(true);
        SubmitApplicationRequest submitRequest = mockSubmitAppRequest(appId, appNames[i], queues[i % queues.length], new HashSet<String>(tags.subList(0, i + 1)));
        rmService.submitApplication(submitRequest);
        submitTimeMillis[i] = System.currentTimeMillis();
    }
    // Test different cases of ClientRMService#getApplications()
    GetApplicationsRequest request = GetApplicationsRequest.newInstance();
    assertEquals("Incorrect total number of apps", 6, rmService.getApplications(request).getApplicationList().size());
    // Check limit
    request.setLimit(1L);
    assertEquals("Failed to limit applications", 1, rmService.getApplications(request).getApplicationList().size());
    // Check start range
    request = GetApplicationsRequest.newInstance();
    request.setStartRange(submitTimeMillis[0], System.currentTimeMillis());
    // 2 applications are submitted after first timeMills
    assertEquals("Incorrect number of matching start range", 2, rmService.getApplications(request).getApplicationList().size());
    // 1 application is submitted after the second timeMills
    request.setStartRange(submitTimeMillis[1], System.currentTimeMillis());
    assertEquals("Incorrect number of matching start range", 1, rmService.getApplications(request).getApplicationList().size());
    // no application is submitted after the third timeMills
    request.setStartRange(submitTimeMillis[2], System.currentTimeMillis());
    assertEquals("Incorrect number of matching start range", 0, rmService.getApplications(request).getApplicationList().size());
    // Check queue
    request = GetApplicationsRequest.newInstance();
    Set<String> queueSet = new HashSet<String>();
    request.setQueues(queueSet);
    queueSet.add(queues[0]);
    assertEquals("Incorrect number of applications in queue", 2, rmService.getApplications(request).getApplicationList().size());
    assertEquals("Incorrect number of applications in queue", 2, rmService.getApplications(request, false).getApplicationList().size());
    queueSet.add(queues[1]);
    assertEquals("Incorrect number of applications in queue", 3, rmService.getApplications(request).getApplicationList().size());
    // Check user
    request = GetApplicationsRequest.newInstance();
    Set<String> userSet = new HashSet<String>();
    request.setUsers(userSet);
    userSet.add("random-user-name");
    assertEquals("Incorrect number of applications for user", 0, rmService.getApplications(request).getApplicationList().size());
    userSet.add(UserGroupInformation.getCurrentUser().getShortUserName());
    assertEquals("Incorrect number of applications for user", 3, rmService.getApplications(request).getApplicationList().size());
    // Check tags
    request = GetApplicationsRequest.newInstance(ApplicationsRequestScope.ALL, null, null, null, null, null, null, null, null);
    Set<String> tagSet = new HashSet<String>();
    request.setApplicationTags(tagSet);
    assertEquals("Incorrect number of matching tags", 6, rmService.getApplications(request).getApplicationList().size());
    tagSet = Sets.newHashSet(tags.get(0));
    request.setApplicationTags(tagSet);
    assertEquals("Incorrect number of matching tags", 3, rmService.getApplications(request).getApplicationList().size());
    tagSet = Sets.newHashSet(tags.get(1));
    request.setApplicationTags(tagSet);
    assertEquals("Incorrect number of matching tags", 2, rmService.getApplications(request).getApplicationList().size());
    tagSet = Sets.newHashSet(tags.get(2));
    request.setApplicationTags(tagSet);
    assertEquals("Incorrect number of matching tags", 1, rmService.getApplications(request).getApplicationList().size());
    // Check scope
    request = GetApplicationsRequest.newInstance(ApplicationsRequestScope.VIEWABLE);
    assertEquals("Incorrect number of applications for the scope", 6, rmService.getApplications(request).getApplicationList().size());
    request = GetApplicationsRequest.newInstance(ApplicationsRequestScope.OWN);
    assertEquals("Incorrect number of applications for the scope", 3, rmService.getApplications(request).getApplicationList().size());
}

Example 2 with Event

use of org.apache.hadoop.yarn.event.Event in project hadoop by apache.

the class TestClientRMService method mockRMContext.

private void mockRMContext(YarnScheduler yarnScheduler, RMContext rmContext) throws IOException {
    Dispatcher dispatcher = mock(Dispatcher.class);
    when(rmContext.getDispatcher()).thenReturn(dispatcher);
    @SuppressWarnings("unchecked") EventHandler<Event> eventHandler = mock(EventHandler.class);
    when(dispatcher.getEventHandler()).thenReturn(eventHandler);
    QueueInfo queInfo = recordFactory.newRecordInstance(QueueInfo.class);
    queInfo.setQueueName("testqueue");
    when(yarnScheduler.getQueueInfo(eq("testqueue"), anyBoolean(), anyBoolean())).thenReturn(queInfo);
    when(yarnScheduler.getQueueInfo(eq("nonexistentqueue"), anyBoolean(), anyBoolean())).thenThrow(new IOException("queue does not exist"));
    RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
    when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
    SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
    when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
    when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
    ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext, yarnScheduler);
    when(rmContext.getRMApps()).thenReturn(apps);
    when(yarnScheduler.getAppsInQueue(eq("testqueue"))).thenReturn(getSchedulerApps(apps));
    ResourceScheduler rs = mock(ResourceScheduler.class);
    when(rmContext.getScheduler()).thenReturn(rs);
}

Example 3 with Event

use of org.apache.hadoop.yarn.event.Event in project hadoop by apache.

the class TestClientRMService method createRMService.

public ClientRMService createRMService() throws IOException, YarnException {
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext, yarnScheduler);
    when(rmContext.getRMApps()).thenReturn(apps);
    when(rmContext.getYarnConfiguration()).thenReturn(new Configuration());
    RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null, mock(ApplicationACLsManager.class), new Configuration());
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(new EventHandler<Event>() {

        public void handle(Event event) {
        }
    });
    ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
    QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
    when(mockQueueACLsManager.checkAccess(any(UserGroupInformation.class), any(QueueACL.class), any(RMApp.class), any(String.class), any())).thenReturn(true);
    return new ClientRMService(rmContext, yarnScheduler, appManager, mockAclsManager, mockQueueACLsManager, null);
}

Example 4 with Event

use of org.apache.hadoop.yarn.event.Event in project hadoop by apache.

the class TestClientRMService method testAppSubmit.

@Test(timeout = 30000)
@SuppressWarnings("rawtypes")
public void testAppSubmit() throws Exception {
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    RMStateStore stateStore = mock(RMStateStore.class);
    when(rmContext.getStateStore()).thenReturn(stateStore);
    RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null, mock(ApplicationACLsManager.class), new Configuration());
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(new EventHandler<Event>() {

        public void handle(Event event) {
        }
    });
    doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext).getRMTimelineCollectorManager();
    ApplicationId appId1 = getApplicationId(100);
    ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
    when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);
    QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
    when(mockQueueACLsManager.checkAccess(any(UserGroupInformation.class), any(QueueACL.class), any(RMApp.class), any(String.class), any())).thenReturn(true);
    ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, appManager, mockAclsManager, mockQueueACLsManager, null);
    rmService.init(new Configuration());
    // without name and queue
    SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest(appId1, null, null);
    try {
        rmService.submitApplication(submitRequest1);
    } catch (YarnException e) {
        Assert.fail("Exception is not expected.");
    }
    RMApp app1 = rmContext.getRMApps().get(appId1);
    Assert.assertNotNull("app doesn't exist", app1);
    Assert.assertEquals("app name doesn't match", YarnConfiguration.DEFAULT_APPLICATION_NAME, app1.getName());
    Assert.assertEquals("app queue doesn't match", YarnConfiguration.DEFAULT_QUEUE_NAME, app1.getQueue());
    // with name and queue
    String name = MockApps.newAppName();
    String queue = MockApps.newQueue();
    ApplicationId appId2 = getApplicationId(101);
    SubmitApplicationRequest submitRequest2 = mockSubmitAppRequest(appId2, name, queue);
    submitRequest2.getApplicationSubmissionContext().setApplicationType("matchType");
    try {
        rmService.submitApplication(submitRequest2);
    } catch (YarnException e) {
        Assert.fail("Exception is not expected.");
    }
    RMApp app2 = rmContext.getRMApps().get(appId2);
    Assert.assertNotNull("app doesn't exist", app2);
    Assert.assertEquals("app name doesn't match", name, app2.getName());
    Assert.assertEquals("app queue doesn't match", queue, app2.getQueue());
    // duplicate appId
    try {
        rmService.submitApplication(submitRequest2);
    } catch (YarnException e) {
        Assert.fail("Exception is not expected.");
    }
    GetApplicationsRequest getAllAppsRequest = GetApplicationsRequest.newInstance(new HashSet<String>());
    GetApplicationsResponse getAllApplicationsResponse = rmService.getApplications(getAllAppsRequest);
    Assert.assertEquals(5, getAllApplicationsResponse.getApplicationList().size());
    Set<String> appTypes = new HashSet<String>();
    appTypes.add("matchType");
    getAllAppsRequest = GetApplicationsRequest.newInstance(appTypes);
    getAllApplicationsResponse = rmService.getApplications(getAllAppsRequest);
    Assert.assertEquals(1, getAllApplicationsResponse.getApplicationList().size());
    Assert.assertEquals(appId2, getAllApplicationsResponse.getApplicationList().get(0).getApplicationId());
}

Example 5 with Event

use of org.apache.hadoop.yarn.event.Event in project hadoop by apache.

the class TestRMNMRPCResponseId method setUp.

@Before
public void setUp() {
    Configuration conf = new Configuration();
    // Dispatcher that processes events inline
    Dispatcher dispatcher = new InlineDispatcher();
    dispatcher.register(SchedulerEventType.class, new EventHandler<Event>() {

        @Override
        public void handle(Event event) {
            // ignore
            ;
        }
    });
    RMContext context = new RMContextImpl(dispatcher, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), null, null);
    dispatcher.register(RMNodeEventType.class, new ResourceManager.NodeEventDispatcher(context));
    NodesListManager nodesListManager = new NodesListManager(context);
    nodesListManager.init(conf);
    context.getContainerTokenSecretManager().rollMasterKey();
    context.getNMTokenSecretManager().rollMasterKey();
    resourceTrackerService = new ResourceTrackerService(context, nodesListManager, new NMLivelinessMonitor(dispatcher), context.getContainerTokenSecretManager(), context.getNMTokenSecretManager());
    resourceTrackerService.init(conf);
}