Search in sources :

Example 26 with AppContext

use of org.apache.hadoop.mapreduce.v2.app.AppContext in project hadoop by apache.

the class TestLocalContainerAllocator method testAllocatedContainerResourceIsNotNull.

@Test
public void testAllocatedContainerResourceIsNotNull() {
    ArgumentCaptor<TaskAttemptContainerAssignedEvent> containerAssignedCaptor = ArgumentCaptor.forClass(TaskAttemptContainerAssignedEvent.class);
    @SuppressWarnings("unchecked") EventHandler<Event> eventHandler = mock(EventHandler.class);
    AppContext context = mock(AppContext.class);
    when(context.getEventHandler()).thenReturn(eventHandler);
    ContainerId containerId = ContainerId.fromString("container_1427562107907_0002_01_000001");
    LocalContainerAllocator containerAllocator = new LocalContainerAllocator(mock(ClientService.class), context, "localhost", -1, -1, containerId);
    ContainerAllocatorEvent containerAllocatorEvent = createContainerRequestEvent();
    containerAllocator.handle(containerAllocatorEvent);
    verify(eventHandler, times(1)).handle(containerAssignedCaptor.capture());
    Container container = containerAssignedCaptor.getValue().getContainer();
    Resource containerResource = container.getResource();
    Assert.assertNotNull(containerResource);
    Assert.assertEquals(containerResource.getMemorySize(), 0);
    Assert.assertEquals(containerResource.getVirtualCores(), 0);
}
Also used : UpdatedContainer(org.apache.hadoop.yarn.api.records.UpdatedContainer) Container(org.apache.hadoop.yarn.api.records.Container) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) ClientService(org.apache.hadoop.mapreduce.v2.app.client.ClientService) AppContext(org.apache.hadoop.mapreduce.v2.app.AppContext) ContainerAllocatorEvent(org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent) Resource(org.apache.hadoop.yarn.api.records.Resource) Event(org.apache.hadoop.yarn.event.Event) TaskAttemptContainerAssignedEvent(org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent) ContainerAllocatorEvent(org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent) TaskAttemptContainerAssignedEvent(org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent) Test(org.junit.Test)

Example 27 with AppContext

use of org.apache.hadoop.mapreduce.v2.app.AppContext in project hadoop by apache.

the class TestRMCommunicator method testRMContainerAllocatorYarnRuntimeExceptionIsHandled.

@Test(timeout = 2000)
public void testRMContainerAllocatorYarnRuntimeExceptionIsHandled() throws Exception {
    ClientService mockClientService = mock(ClientService.class);
    AppContext mockContext = mock(AppContext.class);
    MockRMCommunicator mockRMCommunicator = new MockRMCommunicator(mockClientService, mockContext);
    final RMCommunicator communicator = spy(mockRMCommunicator);
    Clock mockClock = mock(Clock.class);
    when(mockContext.getClock()).thenReturn(mockClock);
    doThrow(new YarnRuntimeException("Test")).doNothing().when(communicator).heartbeat();
    when(mockClock.getTime()).thenReturn(1L).thenAnswer(new Answer<Integer>() {

        @Override
        public Integer answer(InvocationOnMock invocation) throws Throwable {
            communicator.stop();
            return 2;
        }
    }).thenThrow(new AssertionError("GetClock called second time, when it should not have since the thread " + "should have quit"));
    AllocatorRunnable testRunnable = communicator.new AllocatorRunnable();
    testRunnable.run();
    verify(mockClock, times(2)).getTime();
}
Also used : YarnRuntimeException(org.apache.hadoop.yarn.exceptions.YarnRuntimeException) Answer(org.mockito.stubbing.Answer) ClientService(org.apache.hadoop.mapreduce.v2.app.client.ClientService) InvocationOnMock(org.mockito.invocation.InvocationOnMock) AllocatorRunnable(org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.AllocatorRunnable) AppContext(org.apache.hadoop.mapreduce.v2.app.AppContext) Clock(org.apache.hadoop.yarn.util.Clock) Test(org.junit.Test)

Example 28 with AppContext

use of org.apache.hadoop.mapreduce.v2.app.AppContext in project hadoop by apache.

the class TestRMCommunicator method testRMContainerAllocatorExceptionIsHandled.

@Test(timeout = 2000)
public void testRMContainerAllocatorExceptionIsHandled() throws Exception {
    ClientService mockClientService = mock(ClientService.class);
    AppContext mockContext = mock(AppContext.class);
    MockRMCommunicator mockRMCommunicator = new MockRMCommunicator(mockClientService, mockContext);
    RMCommunicator communicator = spy(mockRMCommunicator);
    Clock mockClock = mock(Clock.class);
    when(mockContext.getClock()).thenReturn(mockClock);
    doThrow(new RMContainerAllocationException("Test")).doNothing().when(communicator).heartbeat();
    when(mockClock.getTime()).thenReturn(1L).thenThrow(new AssertionError("GetClock called second time, when it should not have since the " + "thread should have quit"));
    AllocatorRunnable testRunnable = communicator.new AllocatorRunnable();
    testRunnable.run();
}
Also used : ClientService(org.apache.hadoop.mapreduce.v2.app.client.ClientService) AllocatorRunnable(org.apache.hadoop.mapreduce.v2.app.rm.RMCommunicator.AllocatorRunnable) AppContext(org.apache.hadoop.mapreduce.v2.app.AppContext) Clock(org.apache.hadoop.yarn.util.Clock) Test(org.junit.Test)

Example 29 with AppContext

use of org.apache.hadoop.mapreduce.v2.app.AppContext in project hadoop by apache.

the class TestContainerLauncher method testPoolSize.

@Test(timeout = 10000)
public void testPoolSize() throws InterruptedException {
    ApplicationId appId = ApplicationId.newInstance(12345, 67);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 3);
    JobId jobId = MRBuilderUtils.newJobId(appId, 8);
    TaskId taskId = MRBuilderUtils.newTaskId(jobId, 9, TaskType.MAP);
    AppContext context = mock(AppContext.class);
    CustomContainerLauncher containerLauncher = new CustomContainerLauncher(context);
    containerLauncher.init(new Configuration());
    containerLauncher.start();
    ThreadPoolExecutor threadPool = containerLauncher.getThreadPool();
    // No events yet
    Assert.assertEquals(containerLauncher.initialPoolSize, MRJobConfig.DEFAULT_MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE);
    Assert.assertEquals(0, threadPool.getPoolSize());
    Assert.assertEquals(containerLauncher.initialPoolSize, threadPool.getCorePoolSize());
    Assert.assertNull(containerLauncher.foundErrors);
    containerLauncher.expectedCorePoolSize = containerLauncher.initialPoolSize;
    for (int i = 0; i < 10; i++) {
        ContainerId containerId = ContainerId.newContainerId(appAttemptId, i);
        TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId, i);
        containerLauncher.handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host" + i + ":1234", null, ContainerLauncher.EventType.CONTAINER_REMOTE_LAUNCH));
    }
    waitForEvents(containerLauncher, 10);
    Assert.assertEquals(10, threadPool.getPoolSize());
    Assert.assertNull(containerLauncher.foundErrors);
    // Same set of hosts, so no change
    containerLauncher.finishEventHandling = true;
    int timeOut = 0;
    while (containerLauncher.numEventsProcessed.get() < 10 && timeOut++ < 200) {
        LOG.info("Waiting for number of events processed to become " + 10 + ". It is now " + containerLauncher.numEventsProcessed.get() + ". Timeout is " + timeOut);
        Thread.sleep(1000);
    }
    Assert.assertEquals(10, containerLauncher.numEventsProcessed.get());
    containerLauncher.finishEventHandling = false;
    for (int i = 0; i < 10; i++) {
        ContainerId containerId = ContainerId.newContainerId(appAttemptId, i + 10);
        TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId, i + 10);
        containerLauncher.handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host" + i + ":1234", null, ContainerLauncher.EventType.CONTAINER_REMOTE_LAUNCH));
    }
    waitForEvents(containerLauncher, 20);
    Assert.assertEquals(10, threadPool.getPoolSize());
    Assert.assertNull(containerLauncher.foundErrors);
    // Different hosts, there should be an increase in core-thread-pool size to
    // 21(11hosts+10buffer)
    // Core pool size should be 21 but the live pool size should be only 11.
    containerLauncher.expectedCorePoolSize = 11 + containerLauncher.initialPoolSize;
    containerLauncher.finishEventHandling = false;
    ContainerId containerId = ContainerId.newContainerId(appAttemptId, 21);
    TaskAttemptId taskAttemptId = MRBuilderUtils.newTaskAttemptId(taskId, 21);
    containerLauncher.handle(new ContainerLauncherEvent(taskAttemptId, containerId, "host11:1234", null, ContainerLauncher.EventType.CONTAINER_REMOTE_LAUNCH));
    waitForEvents(containerLauncher, 21);
    Assert.assertEquals(11, threadPool.getPoolSize());
    Assert.assertNull(containerLauncher.foundErrors);
    containerLauncher.stop();
    // change configuration MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE
    // and verify initialPoolSize value.
    Configuration conf = new Configuration();
    conf.setInt(MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE, 20);
    containerLauncher = new CustomContainerLauncher(context);
    containerLauncher.init(conf);
    Assert.assertEquals(containerLauncher.initialPoolSize, 20);
}
Also used : TaskId(org.apache.hadoop.mapreduce.v2.api.records.TaskId) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) TaskAttemptId(org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId) AppContext(org.apache.hadoop.mapreduce.v2.app.AppContext) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) JobId(org.apache.hadoop.mapreduce.v2.api.records.JobId) Test(org.junit.Test)

Example 30 with AppContext

use of org.apache.hadoop.mapreduce.v2.app.AppContext in project hadoop by apache.

the class TestRMContainerAllocator method testUnregistrationOnlyIfRegistered.

@Test
public void testUnregistrationOnlyIfRegistered() throws Exception {
    Configuration conf = new Configuration();
    final MyResourceManager rm = new MyResourceManager(conf);
    rm.start();
    DrainDispatcher rmDispatcher = (DrainDispatcher) rm.getRMContext().getDispatcher();
    // Submit the application
    RMApp rmApp = rm.submitApp(1024);
    rmDispatcher.await();
    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 11264);
    amNodeManager.nodeHeartbeat(true);
    rmDispatcher.await();
    final ApplicationAttemptId appAttemptId = rmApp.getCurrentAppAttempt().getAppAttemptId();
    rm.sendAMLaunched(appAttemptId);
    rmDispatcher.await();
    MRApp mrApp = new MRApp(appAttemptId, ContainerId.newContainerId(appAttemptId, 0), 10, 0, false, this.getClass().getName(), true, 1) {

        @Override
        protected Dispatcher createDispatcher() {
            return new DrainDispatcher();
        }

        protected ContainerAllocator createContainerAllocator(ClientService clientService, AppContext context) {
            return new MyContainerAllocator(rm, appAttemptId, context);
        }

        ;
    };
    mrApp.submit(conf);
    DrainDispatcher amDispatcher = (DrainDispatcher) mrApp.getDispatcher();
    MyContainerAllocator allocator = (MyContainerAllocator) mrApp.getContainerAllocator();
    amDispatcher.await();
    Assert.assertTrue(allocator.isApplicationMasterRegistered());
    mrApp.stop();
    Assert.assertTrue(allocator.isUnregistered());
}
Also used : DrainDispatcher(org.apache.hadoop.yarn.event.DrainDispatcher) RMApp(org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ClientService(org.apache.hadoop.mapreduce.v2.app.client.ClientService) MockNM(org.apache.hadoop.yarn.server.resourcemanager.MockNM) RunningAppContext(org.apache.hadoop.mapreduce.v2.app.MRAppMaster.RunningAppContext) AppContext(org.apache.hadoop.mapreduce.v2.app.AppContext) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) MRApp(org.apache.hadoop.mapreduce.v2.app.MRApp) Test(org.junit.Test)

Aggregations

Test (org.junit.Test)74 AppContext (org.apache.hadoop.mapreduce.v2.app.AppContext)73 JobId (org.apache.hadoop.mapreduce.v2.api.records.JobId)47 TaskId (org.apache.hadoop.mapreduce.v2.api.records.TaskId)32 Job (org.apache.hadoop.mapreduce.v2.app.job.Job)32 Configuration (org.apache.hadoop.conf.Configuration)31 TaskAttemptId (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId)26 ApplicationAttemptId (org.apache.hadoop.yarn.api.records.ApplicationAttemptId)22 ContainerId (org.apache.hadoop.yarn.api.records.ContainerId)21 Path (org.apache.hadoop.fs.Path)18 MockAppContext (org.apache.hadoop.mapreduce.v2.app.MockAppContext)18 ApplicationId (org.apache.hadoop.yarn.api.records.ApplicationId)18 Task (org.apache.hadoop.mapreduce.v2.app.job.Task)17 Container (org.apache.hadoop.yarn.api.records.Container)14 JobConf (org.apache.hadoop.mapred.JobConf)13 TaskSplitMetaInfo (org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo)13 InetSocketAddress (java.net.InetSocketAddress)12 ClusterInfo (org.apache.hadoop.mapreduce.v2.app.ClusterInfo)12 TaskAttemptListener (org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener)12 TaskAttemptEvent (org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent)12