Examples with YarnConfiguration org.apache.hadoop.yarn.conf.YarnConfiguration used on opensource projects

Example 91 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestNodesListManager method testCachedResolverWithEvent.

@Test
public void testCachedResolverWithEvent() throws Exception {
    Logger rootLogger = LogManager.getRootLogger();
    rootLogger.setLevel(Level.DEBUG);
    YarnConfiguration conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.RM_NODE_IP_CACHE_EXPIRY_INTERVAL_SECS, 30);
    MockRM rm = new MockRM(conf);
    rm.init(conf);
    NodesListManager nodesListManager = rm.getNodesListManager();
    nodesListManager.init(conf);
    nodesListManager.start();
    NodesListManager.CachedResolver resolver = (NodesListManager.CachedResolver) nodesListManager.getResolver();
    resolver.addToCache("testCachedResolverHost1", "1.1.1.1");
    resolver.addToCache("testCachedResolverHost2", "1.1.1.2");
    Assert.assertEquals("1.1.1.1", resolver.resolve("testCachedResolverHost1"));
    Assert.assertEquals("1.1.1.2", resolver.resolve("testCachedResolverHost2"));
    RMNode rmnode1 = MockNodes.newNodeInfo(1, Resource.newInstance(28000, 8), 1, "testCachedResolverHost1", 1234);
    RMNode rmnode2 = MockNodes.newNodeInfo(1, Resource.newInstance(28000, 8), 1, "testCachedResolverHost2", 1234);
    nodesListManager.handle(new NodesListManagerEvent(NodesListManagerEventType.NODE_USABLE, rmnode1));
    Assert.assertNotEquals("1.1.1.1", resolver.resolve("testCachedResolverHost1"));
    Assert.assertEquals("1.1.1.2", resolver.resolve("testCachedResolverHost2"));
    nodesListManager.handle(new NodesListManagerEvent(NodesListManagerEventType.NODE_USABLE, rmnode2));
    Assert.assertNotEquals("1.1.1.1", resolver.resolve("testCachedResolverHost1"));
    Assert.assertNotEquals("1.1.1.2", resolver.resolve("testCachedResolverHost2"));
}

Example 92 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestNodesListManager method testDefaultResolver.

@Test
public void testDefaultResolver() throws Exception {
    Logger rootLogger = LogManager.getRootLogger();
    rootLogger.setLevel(Level.DEBUG);
    YarnConfiguration conf = new YarnConfiguration();
    MockRM rm = new MockRM(conf);
    rm.init(conf);
    NodesListManager nodesListManager = rm.getNodesListManager();
    NodesListManager.Resolver resolver = nodesListManager.getResolver();
    Assert.assertTrue("default resolver should be DirectResolver", resolver instanceof NodesListManager.DirectResolver);
}

Example 93 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestNMReconnect method testCompareRMNodeAfterReconnect.

@Test
public void testCompareRMNodeAfterReconnect() throws Exception {
    AbstractYarnScheduler scheduler = getScheduler();
    Configuration yarnConf = new YarnConfiguration();
    ConfigurationProvider configurationProvider = ConfigurationProviderFactory.getConfigurationProvider(yarnConf);
    configurationProvider.init(yarnConf);
    context.setConfigurationProvider(configurationProvider);
    RMNodeLabelsManager nlm = new RMNodeLabelsManager();
    nlm.init(yarnConf);
    nlm.start();
    context.setNodeLabelManager(nlm);
    scheduler.setRMContext(context);
    scheduler.init(yarnConf);
    scheduler.start();
    dispatcher.register(SchedulerEventType.class, scheduler);
    String hostname1 = "localhost1";
    Resource capability = BuilderUtils.newResource(4096, 4);
    RegisterNodeManagerRequest request1 = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
    NodeId nodeId1 = NodeId.newInstance(hostname1, 0);
    request1.setNodeId(nodeId1);
    request1.setHttpPort(0);
    request1.setResource(capability);
    resourceTrackerService.registerNodeManager(request1);
    Assert.assertNotNull(context.getRMNodes().get(nodeId1));
    // verify Scheduler and RMContext use same RMNode reference.
    Assert.assertTrue(scheduler.getSchedulerNode(nodeId1).getRMNode() == context.getRMNodes().get(nodeId1));
    Assert.assertEquals(context.getRMNodes().get(nodeId1).getTotalCapability(), capability);
    Resource capability1 = BuilderUtils.newResource(2048, 2);
    request1.setResource(capability1);
    resourceTrackerService.registerNodeManager(request1);
    Assert.assertNotNull(context.getRMNodes().get(nodeId1));
    // verify Scheduler and RMContext use same RMNode reference
    // after reconnect.
    Assert.assertTrue(scheduler.getSchedulerNode(nodeId1).getRMNode() == context.getRMNodes().get(nodeId1));
    // verify RMNode's capability is changed.
    Assert.assertEquals(context.getRMNodes().get(nodeId1).getTotalCapability(), capability1);
    nlm.stop();
    scheduler.stop();
}

Example 94 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestRMContainerImpl method testReleaseWhileRunning.

@Test
public void testReleaseWhileRunning() {
    DrainDispatcher drainDispatcher = new DrainDispatcher();
    EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
    EventHandler generic = mock(EventHandler.class);
    drainDispatcher.register(RMAppAttemptEventType.class, appAttemptEventHandler);
    drainDispatcher.register(RMNodeEventType.class, generic);
    drainDispatcher.init(new YarnConfiguration());
    drainDispatcher.start();
    NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
    ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
    ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);
    Resource resource = BuilderUtils.newResource(512, 1);
    Priority priority = BuilderUtils.newPriority(5);
    Container container = BuilderUtils.newContainer(containerId, nodeId, "host:3465", resource, priority, null);
    ConcurrentMap<ApplicationId, RMApp> rmApps = spy(new ConcurrentHashMap<ApplicationId, RMApp>());
    RMApp rmApp = mock(RMApp.class);
    when(rmApp.getRMAppAttempt((ApplicationAttemptId) Matchers.any())).thenReturn(null);
    Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId) Matchers.any());
    RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
    SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
    when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
    when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
    when(rmContext.getRMApps()).thenReturn(rmApps);
    when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
    YarnConfiguration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO, true);
    when(rmContext.getYarnConfiguration()).thenReturn(conf);
    RMContainer rmContainer = new RMContainerImpl(container, SchedulerRequestKey.extractFrom(container), appAttemptId, nodeId, "user", rmContext);
    assertEquals(RMContainerState.NEW, rmContainer.getState());
    assertEquals(resource, rmContainer.getAllocatedResource());
    assertEquals(nodeId, rmContainer.getAllocatedNode());
    assertEquals(priority, rmContainer.getAllocatedSchedulerKey().getPriority());
    verify(writer).containerStarted(any(RMContainer.class));
    verify(publisher).containerCreated(any(RMContainer.class), anyLong());
    rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START));
    drainDispatcher.await();
    assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());
    rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.ACQUIRED));
    drainDispatcher.await();
    assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());
    rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED));
    drainDispatcher.await();
    assertEquals(RMContainerState.RUNNING, rmContainer.getState());
    assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL());
    // In RUNNING state. Verify RELEASED and associated actions.
    reset(appAttemptEventHandler);
    ContainerStatus containerStatus = SchedulerUtils.createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER);
    rmContainer.handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType.RELEASED));
    drainDispatcher.await();
    assertEquals(RMContainerState.RELEASED, rmContainer.getState());
    assertEquals(SchedulerUtils.RELEASED_CONTAINER, rmContainer.getDiagnosticsInfo());
    assertEquals(ContainerExitStatus.ABORTED, rmContainer.getContainerExitStatus());
    assertEquals(ContainerState.COMPLETE, rmContainer.getContainerState());
    verify(writer).containerFinished(any(RMContainer.class));
    verify(publisher).containerFinished(any(RMContainer.class), anyLong());
    ArgumentCaptor<RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor.forClass(RMAppAttemptContainerFinishedEvent.class);
    verify(appAttemptEventHandler).handle(captor.capture());
    RMAppAttemptContainerFinishedEvent cfEvent = captor.getValue();
    assertEquals(appAttemptId, cfEvent.getApplicationAttemptId());
    assertEquals(containerStatus, cfEvent.getContainerStatus());
    assertEquals(RMAppAttemptEventType.CONTAINER_FINISHED, cfEvent.getType());
    // In RELEASED state. A FINIHSED event may come in.
    rmContainer.handle(new RMContainerFinishedEvent(containerId, SchedulerUtils.createAbnormalContainerStatus(containerId, "FinishedContainer"), RMContainerEventType.FINISHED));
    assertEquals(RMContainerState.RELEASED, rmContainer.getState());
}

Example 95 with YarnConfiguration

use of org.apache.hadoop.yarn.conf.YarnConfiguration in project hadoop by apache.

the class TestAMLivelinessMonitor method testResetTimer.

@Test(timeout = 10000)
public void testResetTimer() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    UserGroupInformation.setConfiguration(conf);
    conf.set(YarnConfiguration.RECOVERY_ENABLED, "true");
    conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
    conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true);
    conf.setInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 6000);
    final ControlledClock clock = new ControlledClock();
    clock.setTime(0);
    MemoryRMStateStore memStore = new MemoryRMStateStore() {

        @Override
        public synchronized RMState loadState() throws Exception {
            clock.setTime(8000);
            return super.loadState();
        }
    };
    memStore.init(conf);
    final ApplicationAttemptId attemptId = mock(ApplicationAttemptId.class);
    final Dispatcher dispatcher = mock(Dispatcher.class);
    final boolean[] expired = new boolean[] { false };
    final AMLivelinessMonitor monitor = new AMLivelinessMonitor(dispatcher, clock) {

        @Override
        protected void expire(ApplicationAttemptId id) {
            Assert.assertEquals(id, attemptId);
            expired[0] = true;
        }
    };
    monitor.register(attemptId);
    MockRM rm = new MockRM(conf, memStore) {

        @Override
        protected AMLivelinessMonitor createAMLivelinessMonitor() {
            return monitor;
        }
    };
    rm.start();
    // make sure that monitor has started
    while (monitor.getServiceState() != Service.STATE.STARTED) {
        Thread.sleep(100);
    }
    // expired[0] would be set to true without resetTimer
    Assert.assertFalse(expired[0]);
    rm.stop();
}