Examples with Dispatcher org.apache.hadoop.yarn.event.Dispatcher used on opensource projects

Example 26 with Dispatcher

use of org.apache.hadoop.yarn.event.Dispatcher in project incubator-myriad by apache.

the class MyriadOperationsTest method generateRMContext.

private RMContext generateRMContext(AbstractYarnScheduler<FiCaSchedulerApp, FiCaSchedulerNode> scheduler) throws Exception {
    Configuration conf = new Configuration();
    MockRMContext context = null;
    Dispatcher dispatcher = new MockDispatcher();
    RMApplicationHistoryWriter rmApplicationHistoryWriter = new RMApplicationHistoryWriter();
    AMLivelinessMonitor amLivelinessMonitor = new AMLivelinessMonitor(dispatcher);
    AMLivelinessMonitor amFinishingMonitor = new AMLivelinessMonitor(dispatcher);
    RMDelegationTokenSecretManager delegationTokenSecretManager = new RMDelegationTokenSecretManager(1, 1, 1, 1, context);
    context = new MockRMContext();
    context.setStateStore(TestObjectFactory.getStateStore(conf, "tmp/myriad-operations-test"));
    context.setAmLivelinessMonitor(amLivelinessMonitor);
    context.setAmFinishingMonitor(amFinishingMonitor);
    context.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
    context.setRMDelegationTokenSecretManager(delegationTokenSecretManager);
    return context;
}

Example 27 with Dispatcher

use of org.apache.hadoop.yarn.event.Dispatcher in project hadoop by apache.

the class ACLsTestBase method setup.

@Before
public void setup() throws InterruptedException, IOException {
    conf = createConfiguration();
    rpc = YarnRPC.create(conf);
    rmAddress = conf.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT);
    AccessControlList adminACL = new AccessControlList("");
    conf.set(YarnConfiguration.YARN_ADMIN_ACL, adminACL.getAclString());
    conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
    resourceManager = new MockRM(conf) {

        protected ClientRMService createClientRMService() {
            return new ClientRMService(getRMContext(), this.scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, getRMContext().getRMDelegationTokenSecretManager());
        }

        @Override
        protected Dispatcher createDispatcher() {
            return new DrainDispatcher();
        }

        @Override
        protected void doSecureLogin() throws IOException {
        }
    };
    new Thread() {

        public void run() {
            resourceManager.start();
        }

        ;
    }.start();
    int waitCount = 0;
    while (resourceManager.getServiceState() == STATE.INITED && waitCount++ < 60) {
        LOG.info("Waiting for RM to start...");
        Thread.sleep(1500);
    }
    if (resourceManager.getServiceState() != STATE.STARTED) {
        // RM could have failed.
        throw new IOException("ResourceManager failed to start. Final state is " + resourceManager.getServiceState());
    }
}

Example 28 with Dispatcher

use of org.apache.hadoop.yarn.event.Dispatcher in project hadoop by apache.

the class TestCapacityScheduler method testResourceUpdateDecommissioningNode.

@Test
public void testResourceUpdateDecommissioningNode() throws Exception {
    // Mock the RMNodeResourceUpdate event handler to update SchedulerNode
    // to have 0 available resource
    RMContext spyContext = Mockito.spy(resourceManager.getRMContext());
    Dispatcher mockDispatcher = mock(AsyncDispatcher.class);
    when(mockDispatcher.getEventHandler()).thenReturn(new EventHandler<Event>() {

        @Override
        public void handle(Event event) {
            if (event instanceof RMNodeResourceUpdateEvent) {
                RMNodeResourceUpdateEvent resourceEvent = (RMNodeResourceUpdateEvent) event;
                resourceManager.getResourceScheduler().getSchedulerNode(resourceEvent.getNodeId()).updateTotalResource(resourceEvent.getResourceOption().getResource());
            }
        }
    });
    Mockito.doReturn(mockDispatcher).when(spyContext).getDispatcher();
    ((CapacityScheduler) resourceManager.getResourceScheduler()).setRMContext(spyContext);
    ((AsyncDispatcher) mockDispatcher).start();
    // Register node
    String host_0 = "host_0";
    org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4));
    // ResourceRequest priorities
    Priority priority_0 = Priority.newInstance(0);
    // Submit an application
    Application application_0 = new Application("user_0", "a1", resourceManager);
    application_0.submit();
    application_0.addNodeManager(host_0, 1234, nm_0);
    Resource capability_0_0 = Resources.createResource(1 * GB, 1);
    application_0.addResourceRequestSpec(priority_0, capability_0_0);
    Task task_0_0 = new Task(application_0, priority_0, new String[] { host_0 });
    application_0.addTask(task_0_0);
    // Send resource requests to the scheduler
    application_0.schedule();
    nodeUpdate(nm_0);
    // Kick off another heartbeat with the node state mocked to decommissioning
    // This should update the schedulernodes to have 0 available resource
    RMNode spyNode = Mockito.spy(resourceManager.getRMContext().getRMNodes().get(nm_0.getNodeId()));
    when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING);
    resourceManager.getResourceScheduler().handle(new NodeUpdateSchedulerEvent(spyNode));
    // Get allocations from the scheduler
    application_0.schedule();
    // Check the used resource is 1 GB 1 core
    Assert.assertEquals(1 * GB, nm_0.getUsed().getMemorySize());
    Resource usedResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getAllocatedResource();
    Assert.assertEquals(usedResource.getMemorySize(), 1 * GB);
    Assert.assertEquals(usedResource.getVirtualCores(), 1);
    // Check total resource of scheduler node is also changed to 1 GB 1 core
    Resource totalResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getTotalResource();
    Assert.assertEquals(totalResource.getMemorySize(), 1 * GB);
    Assert.assertEquals(totalResource.getVirtualCores(), 1);
    // Check the available resource is 0/0
    Resource availableResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource();
    Assert.assertEquals(availableResource.getMemorySize(), 0);
    Assert.assertEquals(availableResource.getVirtualCores(), 0);
}

Example 29 with Dispatcher

use of org.apache.hadoop.yarn.event.Dispatcher in project hadoop by apache.

the class TestFairScheduler method testResourceUpdateDecommissioningNode.

@Test
public void testResourceUpdateDecommissioningNode() throws Exception {
    // Mock the RMNodeResourceUpdate event handler to update SchedulerNode
    // to have 0 available resource
    RMContext spyContext = Mockito.spy(resourceManager.getRMContext());
    Dispatcher mockDispatcher = mock(AsyncDispatcher.class);
    when(mockDispatcher.getEventHandler()).thenReturn(new EventHandler() {

        @Override
        public void handle(Event event) {
            if (event instanceof RMNodeResourceUpdateEvent) {
                RMNodeResourceUpdateEvent resourceEvent = (RMNodeResourceUpdateEvent) event;
                resourceManager.getResourceScheduler().getSchedulerNode(resourceEvent.getNodeId()).updateTotalResource(resourceEvent.getResourceOption().getResource());
            }
        }
    });
    Mockito.doReturn(mockDispatcher).when(spyContext).getDispatcher();
    ((FairScheduler) resourceManager.getResourceScheduler()).setRMContext(spyContext);
    ((AsyncDispatcher) mockDispatcher).start();
    // Register node
    String host_0 = "host_0";
    org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_0 = registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, Resources.createResource(8 * GB, 4));
    RMNode node = resourceManager.getRMContext().getRMNodes().get(nm_0.getNodeId());
    // Send a heartbeat to kick the tires on the Scheduler
    NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node);
    resourceManager.getResourceScheduler().handle(nodeUpdate);
    // Kick off another heartbeat with the node state mocked to decommissioning
    // This should update the schedulernodes to have 0 available resource
    RMNode spyNode = Mockito.spy(resourceManager.getRMContext().getRMNodes().get(nm_0.getNodeId()));
    when(spyNode.getState()).thenReturn(NodeState.DECOMMISSIONING);
    resourceManager.getResourceScheduler().handle(new NodeUpdateSchedulerEvent(spyNode));
    // Check the used resource is 0 GB 0 core
    // Assert.assertEquals(1 * GB, nm_0.getUsed().getMemory());
    Resource usedResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getAllocatedResource();
    Assert.assertEquals(usedResource.getMemorySize(), 0);
    Assert.assertEquals(usedResource.getVirtualCores(), 0);
    // Check total resource of scheduler node is also changed to 0 GB 0 core
    Resource totalResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getTotalResource();
    Assert.assertEquals(totalResource.getMemorySize(), 0 * GB);
    Assert.assertEquals(totalResource.getVirtualCores(), 0);
    // Check the available resource is 0/0
    Resource availableResource = resourceManager.getResourceScheduler().getSchedulerNode(nm_0.getNodeId()).getUnallocatedResource();
    Assert.assertEquals(availableResource.getMemorySize(), 0);
    Assert.assertEquals(availableResource.getVirtualCores(), 0);
}

Example 30 with Dispatcher

use of org.apache.hadoop.yarn.event.Dispatcher in project hadoop by apache.

the class TestRM method testKillFailingApp.

// Test Kill an app while the app is failing
@Test(timeout = 30000)
public void testKillFailingApp() throws Exception {
    // this dispatcher ignores RMAppAttemptEventType.KILL event
    final Dispatcher dispatcher = new DrainDispatcher() {

        @Override
        public EventHandler<Event> getEventHandler() {
            class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {

                @Override
                public boolean matches(Object argument) {
                    if (argument instanceof RMAppAttemptEvent) {
                        if (((RMAppAttemptEvent) argument).getType().equals(RMAppAttemptEventType.KILL)) {
                            return true;
                        }
                    }
                    return false;
                }
            }
            EventHandler handler = spy(super.getEventHandler());
            doNothing().when(handler).handle(argThat(new EventArgMatcher()));
            return handler;
        }
    };
    MockRM rm1 = new MockRM(conf) {

        @Override
        protected Dispatcher createDispatcher() {
            return dispatcher;
        }
    };
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
    nm1.registerNode();
    RMApp app1 = rm1.submitApp(200);
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    rm1.killApp(app1.getApplicationId());
    // fail the app by sending container_finished event.
    nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
    rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FAILED);
    // app is killed, not launching a new attempt
    rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED);
}