Examples with Dispatcher org.apache.hadoop.yarn.event.Dispatcher used on opensource projects

Example 6 with Dispatcher

use of org.apache.hadoop.yarn.event.Dispatcher in project hadoop by apache.

the class TestNMExpiry method setUp.

@Before
public void setUp() {
    Configuration conf = new Configuration();
    // Dispatcher that processes events inline
    Dispatcher dispatcher = new InlineDispatcher();
    RMContext context = new RMContextImpl(dispatcher, null, null, null, null, null, null, null, null, null);
    dispatcher.register(SchedulerEventType.class, new InlineDispatcher.EmptyEventHandler());
    dispatcher.register(RMNodeEventType.class, new NodeEventDispatcher(context));
    NMLivelinessMonitor nmLivelinessMonitor = new TestNmLivelinessMonitor(dispatcher);
    nmLivelinessMonitor.init(conf);
    nmLivelinessMonitor.start();
    NodesListManager nodesListManager = new NodesListManager(context);
    nodesListManager.init(conf);
    RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf);
    containerTokenSecretManager.start();
    NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf);
    nmTokenSecretManager.start();
    resourceTrackerService = new ResourceTrackerService(context, nodesListManager, nmLivelinessMonitor, containerTokenSecretManager, nmTokenSecretManager);
    resourceTrackerService.init(conf);
    resourceTrackerService.start();
}

Example 7 with Dispatcher

use of org.apache.hadoop.yarn.event.Dispatcher in project hadoop by apache.

the class TestRMAppAttemptImplDiagnostics method createRMAppAttemptImpl.

private RMAppAttemptImpl createRMAppAttemptImpl(final Configuration configuration) {
    final ApplicationAttemptId mockApplicationAttemptId = mock(ApplicationAttemptId.class);
    final ApplicationId mockApplicationId = mock(ApplicationId.class);
    when(mockApplicationAttemptId.getApplicationId()).thenReturn(mockApplicationId);
    final RMContext mockRMContext = mock(RMContext.class);
    final Dispatcher mockDispatcher = mock(Dispatcher.class);
    when(mockRMContext.getDispatcher()).thenReturn(mockDispatcher);
    return new RMAppAttemptImpl(mockApplicationAttemptId, mockRMContext, null, null, null, configuration, null, null);
}

Example 8 with Dispatcher

use of org.apache.hadoop.yarn.event.Dispatcher in project hadoop by apache.

the class TestAMLivelinessMonitor method testResetTimer.

@Test(timeout = 10000)
public void testResetTimer() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    UserGroupInformation.setConfiguration(conf);
    conf.set(YarnConfiguration.RECOVERY_ENABLED, "true");
    conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
    conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true);
    conf.setInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 6000);
    final ControlledClock clock = new ControlledClock();
    clock.setTime(0);
    MemoryRMStateStore memStore = new MemoryRMStateStore() {

        @Override
        public synchronized RMState loadState() throws Exception {
            clock.setTime(8000);
            return super.loadState();
        }
    };
    memStore.init(conf);
    final ApplicationAttemptId attemptId = mock(ApplicationAttemptId.class);
    final Dispatcher dispatcher = mock(Dispatcher.class);
    final boolean[] expired = new boolean[] { false };
    final AMLivelinessMonitor monitor = new AMLivelinessMonitor(dispatcher, clock) {

        @Override
        protected void expire(ApplicationAttemptId id) {
            Assert.assertEquals(id, attemptId);
            expired[0] = true;
        }
    };
    monitor.register(attemptId);
    MockRM rm = new MockRM(conf, memStore) {

        @Override
        protected AMLivelinessMonitor createAMLivelinessMonitor() {
            return monitor;
        }
    };
    rm.start();
    // make sure that monitor has started
    while (monitor.getServiceState() != Service.STATE.STARTED) {
        Thread.sleep(100);
    }
    // expired[0] would be set to true without resetTimer
    Assert.assertFalse(expired[0]);
    rm.stop();
}

Example 9 with Dispatcher

use of org.apache.hadoop.yarn.event.Dispatcher in project hadoop by apache.

the class TestFifoScheduler method testResourceUpdateDecommissioningNode.

@Test
public void testResourceUpdateDecommissioningNode() throws Exception {
    // Mock the RMNodeResourceUpdate event handler to update SchedulerNode
    // to have 0 available resource
    RMContext spyContext = Mockito.spy(resourceManager.getRMContext());
    Dispatcher mockDispatcher = mock(AsyncDispatcher.class);
    when(mockDispatcher.getEventHandler()).thenReturn(new EventHandler<Event>() {

        @Override
        public void handle(Event event) {
            if (event instanceof RMNodeResourceUpdateEvent) {
                RMNodeResourceUpdateEvent resourceEvent = (RMNodeResourceUpdateEvent) event;
                resourceManager.getResourceScheduler().getSchedulerNode(resourceEvent.getNodeId()).updateTotalResource(resourceEvent.getResourceOption().getResource());
            }
        }
    });
    Mockito.doReturn(mockDispatcher).when(spyContext).getDispatcher();
    ((FifoScheduler) resourceManager.getResourceScheduler()).setRMContext(spyContext);
    ((AsyncDispatcher) mockDispatcher).start();
    // Register node
    String host_0 = "host_0";
    org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4));
    // ResourceRequest priorities
    Priority priority_0 = Priority.newInstance(0);
    // Submit an application
    Application application_0 = new Application("user_0", "a1", resourceManager);
    application_0.submit();
    application_0.addNodeManager(host_0, 1234, nm_0);
    Resource capability_0_0 = Resources.createResource(1 * GB, 1);
    application_0.addResourceRequestSpec(priority_0, capability_0_0);
    Task task_0_0 = new Task(application_0, priority_0, new String[] { host_0 });
    application_0.addTask(task_0_0);
    // Send resource requests to the scheduler
    application_0.schedule();
    RMNode node = resourceManager.getRMContext().getRMNodes().get(nm_0.getNodeId());
    // Send a heartbeat to kick the tires on the Scheduler
    NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node);
    resourceManager.getResourceScheduler().handle(nodeUpdate);
    // Kick off another heartbeat with the node state mocked to decommissioning
    // This should update the schedulernodes to have 0 available resource
    RMNode spyNode = Mockito.spy(resourceManager.getRMContext().getRMNodes().get(nm_0.getNodeId()));
    when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING);
    resourceManager.getResourceScheduler().handle(new NodeUpdateSchedulerEvent(spyNode));
    // Get allocations from the scheduler
    application_0.schedule();
    // Check the used resource is 1 GB 1 core
    // Assert.assertEquals(1 * GB, nm_0.getUsed().getMemory());
    Resource usedResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getAllocatedResource();
    Assert.assertEquals(usedResource.getMemorySize(), 1 * GB);
    Assert.assertEquals(usedResource.getVirtualCores(), 1);
    // Check total resource of scheduler node is also changed to 1 GB 1 core
    Resource totalResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getTotalResource();
    Assert.assertEquals(totalResource.getMemorySize(), 1 * GB);
    Assert.assertEquals(totalResource.getVirtualCores(), 1);
    // Check the available resource is 0/0
    Resource availableResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource();
    Assert.assertEquals(availableResource.getMemorySize(), 0);
    Assert.assertEquals(availableResource.getVirtualCores(), 0);
}

Example 10 with Dispatcher

use of org.apache.hadoop.yarn.event.Dispatcher in project hadoop by apache.

the class TestNMWebServicesContainers method addAppContainers.

private HashMap<String, String> addAppContainers(Application app) throws IOException {
    Dispatcher dispatcher = new AsyncDispatcher();
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(app.getAppId(), 1);
    Container container1 = new MockContainer(appAttemptId, dispatcher, conf, app.getUser(), app.getAppId(), 1);
    ((MockContainer) container1).setState(ContainerState.RUNNING);
    Container container2 = new MockContainer(appAttemptId, dispatcher, conf, app.getUser(), app.getAppId(), 2);
    ((MockContainer) container2).setState(ContainerState.RUNNING);
    nmContext.getContainers().put(container1.getContainerId(), container1);
    nmContext.getContainers().put(container2.getContainerId(), container2);
    File appDir = new File(testLogDir + "/" + app.getAppId().toString());
    appDir.mkdir();
    File container1Dir = new File(appDir + "/" + container1.getContainerId().toString());
    container1Dir.mkdir();
    // Create log files for containers.
    new File(container1Dir + "/" + "syslog").createNewFile();
    new File(container1Dir + "/" + "stdout").createNewFile();
    File container2Dir = new File(appDir + "/" + container2.getContainerId().toString());
    container2Dir.mkdir();
    new File(container2Dir + "/" + "syslog").createNewFile();
    new File(container2Dir + "/" + "stdout").createNewFile();
    app.getContainers().put(container1.getContainerId(), container1);
    app.getContainers().put(container2.getContainerId(), container2);
    HashMap<String, String> hash = new HashMap<String, String>();
    hash.put(container1.getContainerId().toString(), container1.getContainerId().toString());
    hash.put(container2.getContainerId().toString(), container2.getContainerId().toString());
    return hash;
}