Examples with Event org.apache.hadoop.yarn.event.Event used on opensource projects

Example 16 with Event

use of org.apache.hadoop.yarn.event.Event in project hadoop by apache.

the class TestDelegationTokenRenewer method setUp.

@Before
public void setUp() throws Exception {
    counter = new AtomicInteger(0);
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    eventQueue = new LinkedBlockingQueue<Event>();
    dispatcher = new AsyncDispatcher(eventQueue);
    Renewer.reset();
    delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter);
    RMContext mockContext = mock(RMContext.class);
    ClientRMService mockClientRMService = mock(ClientRMService.class);
    when(mockContext.getSystemCredentialsForApps()).thenReturn(new ConcurrentHashMap<ApplicationId, ByteBuffer>());
    when(mockContext.getDelegationTokenRenewer()).thenReturn(delegationTokenRenewer);
    when(mockContext.getDispatcher()).thenReturn(dispatcher);
    when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
    InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234);
    when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
    delegationTokenRenewer.setRMContext(mockContext);
    delegationTokenRenewer.init(conf);
    delegationTokenRenewer.start();
}

Example 17 with Event

use of org.apache.hadoop.yarn.event.Event in project hadoop by apache.

the class TestFifoScheduler method testResourceUpdateDecommissioningNode.

@Test
public void testResourceUpdateDecommissioningNode() throws Exception {
    // Mock the RMNodeResourceUpdate event handler to update SchedulerNode
    // to have 0 available resource
    RMContext spyContext = Mockito.spy(resourceManager.getRMContext());
    Dispatcher mockDispatcher = mock(AsyncDispatcher.class);
    when(mockDispatcher.getEventHandler()).thenReturn(new EventHandler<Event>() {

        @Override
        public void handle(Event event) {
            if (event instanceof RMNodeResourceUpdateEvent) {
                RMNodeResourceUpdateEvent resourceEvent = (RMNodeResourceUpdateEvent) event;
                resourceManager.getResourceScheduler().getSchedulerNode(resourceEvent.getNodeId()).updateTotalResource(resourceEvent.getResourceOption().getResource());
            }
        }
    });
    Mockito.doReturn(mockDispatcher).when(spyContext).getDispatcher();
    ((FifoScheduler) resourceManager.getResourceScheduler()).setRMContext(spyContext);
    ((AsyncDispatcher) mockDispatcher).start();
    // Register node
    String host_0 = "host_0";
    org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4));
    // ResourceRequest priorities
    Priority priority_0 = Priority.newInstance(0);
    // Submit an application
    Application application_0 = new Application("user_0", "a1", resourceManager);
    application_0.submit();
    application_0.addNodeManager(host_0, 1234, nm_0);
    Resource capability_0_0 = Resources.createResource(1 * GB, 1);
    application_0.addResourceRequestSpec(priority_0, capability_0_0);
    Task task_0_0 = new Task(application_0, priority_0, new String[] { host_0 });
    application_0.addTask(task_0_0);
    // Send resource requests to the scheduler
    application_0.schedule();
    RMNode node = resourceManager.getRMContext().getRMNodes().get(nm_0.getNodeId());
    // Send a heartbeat to kick the tires on the Scheduler
    NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node);
    resourceManager.getResourceScheduler().handle(nodeUpdate);
    // Kick off another heartbeat with the node state mocked to decommissioning
    // This should update the schedulernodes to have 0 available resource
    RMNode spyNode = Mockito.spy(resourceManager.getRMContext().getRMNodes().get(nm_0.getNodeId()));
    when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING);
    resourceManager.getResourceScheduler().handle(new NodeUpdateSchedulerEvent(spyNode));
    // Get allocations from the scheduler
    application_0.schedule();
    // Check the used resource is 1 GB 1 core
    // Assert.assertEquals(1 * GB, nm_0.getUsed().getMemory());
    Resource usedResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getAllocatedResource();
    Assert.assertEquals(usedResource.getMemorySize(), 1 * GB);
    Assert.assertEquals(usedResource.getVirtualCores(), 1);
    // Check total resource of scheduler node is also changed to 1 GB 1 core
    Resource totalResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getTotalResource();
    Assert.assertEquals(totalResource.getMemorySize(), 1 * GB);
    Assert.assertEquals(totalResource.getVirtualCores(), 1);
    // Check the available resource is 0/0
    Resource availableResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource();
    Assert.assertEquals(availableResource.getMemorySize(), 0);
    Assert.assertEquals(availableResource.getVirtualCores(), 0);
}

Example 18 with Event

use of org.apache.hadoop.yarn.event.Event in project hadoop by apache.

the class TestClientRMService method testConcurrentAppSubmit.

@Test(timeout = 4000)
public void testConcurrentAppSubmit() throws IOException, InterruptedException, BrokenBarrierException, YarnException {
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    RMStateStore stateStore = mock(RMStateStore.class);
    when(rmContext.getStateStore()).thenReturn(stateStore);
    RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null, mock(ApplicationACLsManager.class), new Configuration());
    final ApplicationId appId1 = getApplicationId(100);
    final ApplicationId appId2 = getApplicationId(101);
    final SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest(appId1, null, null);
    final SubmitApplicationRequest submitRequest2 = mockSubmitAppRequest(appId2, null, null);
    final CyclicBarrier startBarrier = new CyclicBarrier(2);
    final CyclicBarrier endBarrier = new CyclicBarrier(2);
    EventHandler<Event> eventHandler = new EventHandler<Event>() {

        @Override
        public void handle(Event rawEvent) {
            if (rawEvent instanceof RMAppEvent) {
                RMAppEvent event = (RMAppEvent) rawEvent;
                if (event.getApplicationId().equals(appId1)) {
                    try {
                        startBarrier.await();
                        endBarrier.await();
                    } catch (BrokenBarrierException e) {
                        LOG.warn("Broken Barrier", e);
                    } catch (InterruptedException e) {
                        LOG.warn("Interrupted while awaiting barriers", e);
                    }
                }
            }
        }
    };
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(eventHandler);
    doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext).getRMTimelineCollectorManager();
    final ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, appManager, null, null, null);
    rmService.init(new Configuration());
    // submit an app and wait for it to block while in app submission
    Thread t = new Thread() {

        @Override
        public void run() {
            try {
                rmService.submitApplication(submitRequest1);
            } catch (YarnException | IOException e) {
            }
        }
    };
    t.start();
    // submit another app, so go through while the first app is blocked
    startBarrier.await();
    rmService.submitApplication(submitRequest2);
    endBarrier.await();
    t.join();
}

Example 19 with Event

use of org.apache.hadoop.yarn.event.Event in project hadoop by apache.

the class TestClientRMService method testGetApplicationResourceUsageReportDummy.

@Test
public void testGetApplicationResourceUsageReportDummy() throws YarnException, IOException {
    ApplicationAttemptId attemptId = getApplicationAttemptId(1);
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(new EventHandler<Event>() {

        public void handle(Event event) {
        }
    });
    ApplicationSubmissionContext asContext = mock(ApplicationSubmissionContext.class);
    YarnConfiguration config = new YarnConfiguration();
    RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId, rmContext, yarnScheduler, null, asContext, config, null, null);
    ApplicationResourceUsageReport report = rmAppAttemptImpl.getApplicationResourceUsageReport();
    assertEquals(report, RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
}

Example 20 with Event

use of org.apache.hadoop.yarn.event.Event in project hadoop by apache.

the class TestCapacityScheduler method testResourceUpdateDecommissioningNode.

@Test
public void testResourceUpdateDecommissioningNode() throws Exception {
    // Mock the RMNodeResourceUpdate event handler to update SchedulerNode
    // to have 0 available resource
    RMContext spyContext = Mockito.spy(resourceManager.getRMContext());
    Dispatcher mockDispatcher = mock(AsyncDispatcher.class);
    when(mockDispatcher.getEventHandler()).thenReturn(new EventHandler<Event>() {

        @Override
        public void handle(Event event) {
            if (event instanceof RMNodeResourceUpdateEvent) {
                RMNodeResourceUpdateEvent resourceEvent = (RMNodeResourceUpdateEvent) event;
                resourceManager.getResourceScheduler().getSchedulerNode(resourceEvent.getNodeId()).updateTotalResource(resourceEvent.getResourceOption().getResource());
            }
        }
    });
    Mockito.doReturn(mockDispatcher).when(spyContext).getDispatcher();
    ((CapacityScheduler) resourceManager.getResourceScheduler()).setRMContext(spyContext);
    ((AsyncDispatcher) mockDispatcher).start();
    // Register node
    String host_0 = "host_0";
    org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4));
    // ResourceRequest priorities
    Priority priority_0 = Priority.newInstance(0);
    // Submit an application
    Application application_0 = new Application("user_0", "a1", resourceManager);
    application_0.submit();
    application_0.addNodeManager(host_0, 1234, nm_0);
    Resource capability_0_0 = Resources.createResource(1 * GB, 1);
    application_0.addResourceRequestSpec(priority_0, capability_0_0);
    Task task_0_0 = new Task(application_0, priority_0, new String[] { host_0 });
    application_0.addTask(task_0_0);
    // Send resource requests to the scheduler
    application_0.schedule();
    nodeUpdate(nm_0);
    // Kick off another heartbeat with the node state mocked to decommissioning
    // This should update the schedulernodes to have 0 available resource
    RMNode spyNode = Mockito.spy(resourceManager.getRMContext().getRMNodes().get(nm_0.getNodeId()));
    when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING);
    resourceManager.getResourceScheduler().handle(new NodeUpdateSchedulerEvent(spyNode));
    // Get allocations from the scheduler
    application_0.schedule();
    // Check the used resource is 1 GB 1 core
    Assert.assertEquals(1 * GB, nm_0.getUsed().getMemorySize());
    Resource usedResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getAllocatedResource();
    Assert.assertEquals(usedResource.getMemorySize(), 1 * GB);
    Assert.assertEquals(usedResource.getVirtualCores(), 1);
    // Check total resource of scheduler node is also changed to 1 GB 1 core
    Resource totalResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getTotalResource();
    Assert.assertEquals(totalResource.getMemorySize(), 1 * GB);
    Assert.assertEquals(totalResource.getVirtualCores(), 1);
    // Check the available resource is 0/0
    Resource availableResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource();
    Assert.assertEquals(availableResource.getMemorySize(), 0);
    Assert.assertEquals(availableResource.getVirtualCores(), 0);
}