Search in sources :

Example 41 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestResourceLocalizationService method testFailedPublicResource.

@Test(timeout = 20000)
// mocked generics
@SuppressWarnings("unchecked")
public void testFailedPublicResource() throws Exception {
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    DrainDispatcher dispatcher = new DrainDispatcher();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    DeletionService delService = mock(DeletionService.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    dispatcher.init(conf);
    dispatcher.start();
    try {
        ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext);
        ResourceLocalizationService spyService = spy(rawService);
        doReturn(mockServer).when(spyService).createServer();
        doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
        spyService.init(conf);
        spyService.start();
        final String user = "user0";
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn(user);
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        // init container.
        final Container c = getMockContainer(appId, 42, user);
        // init resources
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        // cause chmod to fail after a delay
        final CyclicBarrier barrier = new CyclicBarrier(2);
        doAnswer(new Answer<Void>() {

            public Void answer(InvocationOnMock invocation) throws IOException {
                try {
                    barrier.await();
                } catch (InterruptedException e) {
                } catch (BrokenBarrierException e) {
                }
                throw new IOException("forced failure");
            }
        }).when(spylfs).setPermission(isA(Path.class), isA(FsPermission.class));
        // Queue up two localization requests for the same public resource
        final LocalResource pubResource = getPublicMockedResource(r);
        final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq));
        Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>();
        pubRsrcs.add(pubReq);
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        dispatcher.await();
        // allow the chmod to fail now that both requests have been queued
        barrier.await();
        verify(containerBus, timeout(5000).times(2)).handle(isA(ContainerResourceFailedEvent.class));
    } finally {
        dispatcher.stop();
    }
}
Also used : DrainDispatcher(org.apache.hadoop.yarn.event.DrainDispatcher) ContainerExecutor(org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor) DefaultContainerExecutor(org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor) BrokenBarrierException(java.util.concurrent.BrokenBarrierException) ContainerLocalizationRequestEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) LocalResourceVisibility(org.apache.hadoop.yarn.api.records.LocalResourceVisibility) Container(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container) Random(java.util.Random) ContainerResourceFailedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent) ApplicationLocalizationEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent) FsPermission(org.apache.hadoop.fs.permission.FsPermission) HashSet(java.util.HashSet) Path(org.apache.hadoop.fs.Path) ContainerEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent) ApplicationEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent) DeletionService(org.apache.hadoop.yarn.server.nodemanager.DeletionService) IOException(java.io.IOException) LocalDirsHandlerService(org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService) CyclicBarrier(java.util.concurrent.CyclicBarrier) LocalResource(org.apache.hadoop.yarn.api.records.LocalResource) InvocationOnMock(org.mockito.invocation.InvocationOnMock) Collection(java.util.Collection) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) Application(org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application) Test(org.junit.Test)

Example 42 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestResourceLocalizationService method testRecovery.

@Test
// mocked generics
@SuppressWarnings("unchecked")
public void testRecovery() throws Exception {
    final String user1 = "user1";
    final String user2 = "user2";
    final ApplicationId appId1 = ApplicationId.newInstance(1, 1);
    final ApplicationId appId2 = ApplicationId.newInstance(1, 2);
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
    NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService();
    stateStore.init(conf);
    stateStore.start();
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    //Ignore actual localization
    EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerBus);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    ResourceLocalizationService spyService = createSpyService(dispatcher, dirsHandler, stateStore);
    try {
        spyService.init(conf);
        spyService.start();
        final Application app1 = mock(Application.class);
        when(app1.getUser()).thenReturn(user1);
        when(app1.getAppId()).thenReturn(appId1);
        final Application app2 = mock(Application.class);
        when(app2.getUser()).thenReturn(user2);
        when(app2.getAppId()).thenReturn(appId2);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app1));
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app2));
        dispatcher.await();
        //Get a handle on the trackers after they're setup with INIT_APP_RESOURCES
        LocalResourcesTracker appTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user1, appId1);
        LocalResourcesTracker privTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user1, null);
        LocalResourcesTracker appTracker2 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user2, appId2);
        LocalResourcesTracker pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, null, null);
        // init containers
        final Container c1 = getMockContainer(appId1, 1, user1);
        final Container c2 = getMockContainer(appId2, 2, user2);
        // init resources
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        // Send localization requests of each type.
        final LocalResource privResource1 = getPrivateMockedResource(r);
        final LocalResourceRequest privReq1 = new LocalResourceRequest(privResource1);
        final LocalResource privResource2 = getPrivateMockedResource(r);
        final LocalResourceRequest privReq2 = new LocalResourceRequest(privResource2);
        final LocalResource pubResource1 = getPublicMockedResource(r);
        final LocalResourceRequest pubReq1 = new LocalResourceRequest(pubResource1);
        final LocalResource pubResource2 = getPublicMockedResource(r);
        final LocalResourceRequest pubReq2 = new LocalResourceRequest(pubResource2);
        final LocalResource appResource1 = getAppMockedResource(r);
        final LocalResourceRequest appReq1 = new LocalResourceRequest(appResource1);
        final LocalResource appResource2 = getAppMockedResource(r);
        final LocalResourceRequest appReq2 = new LocalResourceRequest(appResource2);
        final LocalResource appResource3 = getAppMockedResource(r);
        final LocalResourceRequest appReq3 = new LocalResourceRequest(appResource3);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req1 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req1.put(LocalResourceVisibility.PRIVATE, Arrays.asList(new LocalResourceRequest[] { privReq1, privReq2 }));
        req1.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq1));
        req1.put(LocalResourceVisibility.APPLICATION, Collections.singletonList(appReq1));
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req2 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req2.put(LocalResourceVisibility.APPLICATION, Arrays.asList(new LocalResourceRequest[] { appReq2, appReq3 }));
        req2.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq2));
        // Send Request event
        spyService.handle(new ContainerLocalizationRequestEvent(c1, req1));
        spyService.handle(new ContainerLocalizationRequestEvent(c2, req2));
        dispatcher.await();
        // Simulate start of localization for all resources
        privTracker1.getPathForLocalization(privReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE + user1), null);
        privTracker1.getPathForLocalization(privReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE + user1), null);
        LocalizedResource privLr1 = privTracker1.getLocalizedResource(privReq1);
        LocalizedResource privLr2 = privTracker1.getLocalizedResource(privReq2);
        appTracker1.getPathForLocalization(appReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId1), null);
        LocalizedResource appLr1 = appTracker1.getLocalizedResource(appReq1);
        appTracker2.getPathForLocalization(appReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId2), null);
        LocalizedResource appLr2 = appTracker2.getLocalizedResource(appReq2);
        appTracker2.getPathForLocalization(appReq3, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId2), null);
        LocalizedResource appLr3 = appTracker2.getLocalizedResource(appReq3);
        pubTracker.getPathForLocalization(pubReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.FILECACHE), null);
        LocalizedResource pubLr1 = pubTracker.getLocalizedResource(pubReq1);
        pubTracker.getPathForLocalization(pubReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.FILECACHE), null);
        LocalizedResource pubLr2 = pubTracker.getLocalizedResource(pubReq2);
        // Simulate completion of localization for most resources with
        // possibly different sizes than in the request
        assertNotNull("Localization not started", privLr1.getLocalPath());
        privTracker1.handle(new ResourceLocalizedEvent(privReq1, privLr1.getLocalPath(), privLr1.getSize() + 5));
        assertNotNull("Localization not started", privLr2.getLocalPath());
        privTracker1.handle(new ResourceLocalizedEvent(privReq2, privLr2.getLocalPath(), privLr2.getSize() + 10));
        assertNotNull("Localization not started", appLr1.getLocalPath());
        appTracker1.handle(new ResourceLocalizedEvent(appReq1, appLr1.getLocalPath(), appLr1.getSize()));
        assertNotNull("Localization not started", appLr3.getLocalPath());
        appTracker2.handle(new ResourceLocalizedEvent(appReq3, appLr3.getLocalPath(), appLr3.getSize() + 7));
        assertNotNull("Localization not started", pubLr1.getLocalPath());
        pubTracker.handle(new ResourceLocalizedEvent(pubReq1, pubLr1.getLocalPath(), pubLr1.getSize() + 1000));
        assertNotNull("Localization not started", pubLr2.getLocalPath());
        pubTracker.handle(new ResourceLocalizedEvent(pubReq2, pubLr2.getLocalPath(), pubLr2.getSize() + 99999));
        dispatcher.await();
        assertEquals(ResourceState.LOCALIZED, privLr1.getState());
        assertEquals(ResourceState.LOCALIZED, privLr2.getState());
        assertEquals(ResourceState.LOCALIZED, appLr1.getState());
        assertEquals(ResourceState.DOWNLOADING, appLr2.getState());
        assertEquals(ResourceState.LOCALIZED, appLr3.getState());
        assertEquals(ResourceState.LOCALIZED, pubLr1.getState());
        assertEquals(ResourceState.LOCALIZED, pubLr2.getState());
        // restart and recover
        spyService = createSpyService(dispatcher, dirsHandler, stateStore);
        spyService.init(conf);
        spyService.recoverLocalizedResources(stateStore.loadLocalizationState());
        dispatcher.await();
        appTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user1, appId1);
        privTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user1, null);
        appTracker2 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user2, appId2);
        pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, null, null);
        LocalizedResource recoveredRsrc = privTracker1.getLocalizedResource(privReq1);
        assertEquals(privReq1, recoveredRsrc.getRequest());
        assertEquals(privLr1.getLocalPath(), recoveredRsrc.getLocalPath());
        assertEquals(privLr1.getSize(), recoveredRsrc.getSize());
        assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState());
        recoveredRsrc = privTracker1.getLocalizedResource(privReq2);
        assertEquals(privReq2, recoveredRsrc.getRequest());
        assertEquals(privLr2.getLocalPath(), recoveredRsrc.getLocalPath());
        assertEquals(privLr2.getSize(), recoveredRsrc.getSize());
        assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState());
        recoveredRsrc = appTracker1.getLocalizedResource(appReq1);
        assertEquals(appReq1, recoveredRsrc.getRequest());
        assertEquals(appLr1.getLocalPath(), recoveredRsrc.getLocalPath());
        assertEquals(appLr1.getSize(), recoveredRsrc.getSize());
        assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState());
        recoveredRsrc = appTracker2.getLocalizedResource(appReq2);
        assertNull("in-progress resource should not be present", recoveredRsrc);
        recoveredRsrc = appTracker2.getLocalizedResource(appReq3);
        assertEquals(appReq3, recoveredRsrc.getRequest());
        assertEquals(appLr3.getLocalPath(), recoveredRsrc.getLocalPath());
        assertEquals(appLr3.getSize(), recoveredRsrc.getSize());
        assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState());
    } finally {
        dispatcher.stop();
        stateStore.close();
    }
}
Also used : DrainDispatcher(org.apache.hadoop.yarn.event.DrainDispatcher) ContainerLocalizationRequestEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent) ResourceLocalizedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) LocalResourceVisibility(org.apache.hadoop.yarn.api.records.LocalResourceVisibility) Container(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container) Random(java.util.Random) ApplicationLocalizationEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent) NMMemoryStateStoreService(org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService) Path(org.apache.hadoop.fs.Path) ContainerEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent) ApplicationEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent) LocalDirsHandlerService(org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService) LocalResource(org.apache.hadoop.yarn.api.records.LocalResource) LocalizerEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent) Collection(java.util.Collection) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) Application(org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application) Test(org.junit.Test)

Example 43 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestJavaSandboxLinuxContainerRuntime method createRuntimeContext.

public ContainerRuntimeContext.Builder createRuntimeContext() {
    Container container = mock(Container.class);
    ContainerLaunchContext ctx = mock(ContainerLaunchContext.class);
    when(container.getLaunchContext()).thenReturn(ctx);
    when(ctx.getEnvironment()).thenReturn(env);
    ContainerRuntimeContext.Builder builder = new ContainerRuntimeContext.Builder(container);
    List<String> localDirs = new ArrayList<>();
    builder.setExecutionAttribute(LOCALIZED_RESOURCES, resources).setExecutionAttribute(RUN_AS_USER, NORMAL_USER).setExecutionAttribute(CONTAINER_ID_STR, CONTAINER_ID).setExecutionAttribute(APPID, APPLICATION_ID).setExecutionAttribute(CONTAINER_WORK_DIR, new Path(containerDir.toString())).setExecutionAttribute(LOCAL_DIRS, localDirs).setExecutionAttribute(LOG_DIRS, localDirs).setExecutionAttribute(FILECACHE_DIRS, localDirs).setExecutionAttribute(USER_LOCAL_DIRS, localDirs).setExecutionAttribute(CONTAINER_LOCAL_DIRS, localDirs).setExecutionAttribute(CONTAINER_RUN_CMDS, localDirs);
    return builder;
}
Also used : Path(org.apache.hadoop.fs.Path) Container(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container) ArrayList(java.util.ArrayList) ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) ContainerRuntimeContext(org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext)

Example 44 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestCGroupsCpuResourceHandlerImpl method testPreStartRestrictedContainers.

@Test
public void testPreStartRestrictedContainers() throws Exception {
    String id = "container_01_01";
    String path = "test-path/" + id;
    int defaultVCores = 8;
    Configuration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE, true);
    int cpuPerc = 75;
    conf.setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, cpuPerc);
    cGroupsCpuResourceHandler.bootstrap(plugin, conf);
    verify(mockCGroupsHandler, times(1)).updateCGroupParam(CGroupsHandler.CGroupController.CPU, "", CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf("333333"));
    verify(mockCGroupsHandler, times(1)).updateCGroupParam(CGroupsHandler.CGroupController.CPU, "", CGroupsHandler.CGROUP_CPU_QUOTA_US, String.valueOf(CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US));
    float yarnCores = (cpuPerc * numProcessors) / 100;
    int[] containerVCores = { 2, 4 };
    for (int cVcores : containerVCores) {
        ContainerId mockContainerId = mock(ContainerId.class);
        when(mockContainerId.toString()).thenReturn(id);
        Container mockContainer = mock(Container.class);
        when(mockContainer.getContainerId()).thenReturn(mockContainerId);
        when(mockCGroupsHandler.getPathForCGroupTasks(CGroupsHandler.CGroupController.CPU, id)).thenReturn(path);
        when(mockContainer.getResource()).thenReturn(Resource.newInstance(1024, cVcores));
        when(mockCGroupsHandler.getPathForCGroupTasks(CGroupsHandler.CGroupController.CPU, id)).thenReturn(path);
        float share = (cVcores * yarnCores) / defaultVCores;
        int quotaUS;
        int periodUS;
        if (share > 1.0f) {
            quotaUS = CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US;
            periodUS = (int) ((float) CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US / share);
        } else {
            quotaUS = (int) (CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US * share);
            periodUS = CGroupsCpuResourceHandlerImpl.MAX_QUOTA_US;
        }
        cGroupsCpuResourceHandler.preStart(mockContainer);
        verify(mockCGroupsHandler, times(1)).updateCGroupParam(CGroupsHandler.CGroupController.CPU, id, CGroupsHandler.CGROUP_CPU_SHARES, String.valueOf(CGroupsCpuResourceHandlerImpl.CPU_DEFAULT_WEIGHT * cVcores));
        // set quota and period
        verify(mockCGroupsHandler, times(1)).updateCGroupParam(CGroupsHandler.CGroupController.CPU, id, CGroupsHandler.CGROUP_CPU_PERIOD_US, String.valueOf(periodUS));
        verify(mockCGroupsHandler, times(1)).updateCGroupParam(CGroupsHandler.CGroupController.CPU, id, CGroupsHandler.CGROUP_CPU_QUOTA_US, String.valueOf(quotaUS));
    }
}
Also used : Container(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) Test(org.junit.Test)

Example 45 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestContainerManager method testChangeContainerResource.

@Test
public void testChangeContainerResource() throws Exception {
    containerManager.start();
    File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
    PrintWriter fileWriter = new PrintWriter(scriptFile);
    // Construct the Container-id
    ContainerId cId = createContainerId(0);
    if (Shell.WINDOWS) {
        fileWriter.println("@ping -n 100 127.0.0.1 >nul");
    } else {
        fileWriter.write("\numask 0");
        fileWriter.write("\nexec sleep 100");
    }
    fileWriter.close();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    URL resource_alpha = URL.fromPath(localFS.makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
    rsrc_alpha.setResource(resource_alpha);
    rsrc_alpha.setSize(-1);
    rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
    rsrc_alpha.setType(LocalResourceType.FILE);
    rsrc_alpha.setTimestamp(scriptFile.lastModified());
    String destinationFile = "dest_file";
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(destinationFile, rsrc_alpha);
    containerLaunchContext.setLocalResources(localResources);
    List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
    containerLaunchContext.setCommands(commands);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    // Make sure the container reaches RUNNING state
    BaseContainerManagerTest.waitForNMContainerState(containerManager, cId, org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.RUNNING);
    // Construct container resource increase request,
    List<Token> increaseTokens = new ArrayList<>();
    // Add increase request.
    Resource targetResource = Resource.newInstance(4096, 2);
    Token containerToken = createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, targetResource, context.getContainerTokenSecretManager(), null);
    increaseTokens.add(containerToken);
    IncreaseContainersResourceRequest increaseRequest = IncreaseContainersResourceRequest.newInstance(increaseTokens);
    IncreaseContainersResourceResponse increaseResponse = containerManager.increaseContainersResource(increaseRequest);
    Assert.assertEquals(1, increaseResponse.getSuccessfullyIncreasedContainers().size());
    Assert.assertTrue(increaseResponse.getFailedRequests().isEmpty());
    // Check status
    List<ContainerId> containerIds = new ArrayList<>();
    containerIds.add(cId);
    GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.newInstance(containerIds);
    ContainerStatus containerStatus = containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
    // Check status immediately as resource increase is blocking
    assertEquals(targetResource, containerStatus.getCapability());
    // Simulate a decrease request
    List<org.apache.hadoop.yarn.api.records.Container> containersToDecrease = new ArrayList<>();
    targetResource = Resource.newInstance(2048, 2);
    org.apache.hadoop.yarn.api.records.Container decreasedContainer = org.apache.hadoop.yarn.api.records.Container.newInstance(cId, null, null, targetResource, null, null);
    containersToDecrease.add(decreasedContainer);
    containerManager.handle(new CMgrDecreaseContainersResourceEvent(containersToDecrease));
    // Check status with retry
    containerStatus = containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
    int retry = 0;
    while (!targetResource.equals(containerStatus.getCapability()) && (retry++ < 5)) {
        Thread.sleep(200);
        containerStatus = containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
    }
    assertEquals(targetResource, containerStatus.getCapability());
}
Also used : HashMap(java.util.HashMap) GetContainerStatusesRequest(org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest) ArrayList(java.util.ArrayList) Token(org.apache.hadoop.yarn.api.records.Token) URL(org.apache.hadoop.yarn.api.records.URL) ContainerStatus(org.apache.hadoop.yarn.api.records.ContainerStatus) Container(org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container) ContainerId(org.apache.hadoop.yarn.api.records.ContainerId) IncreaseContainersResourceResponse(org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse) PrintWriter(java.io.PrintWriter) Path(org.apache.hadoop.fs.Path) StartContainersRequest(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest) CMgrDecreaseContainersResourceEvent(org.apache.hadoop.yarn.server.nodemanager.CMgrDecreaseContainersResourceEvent) Resource(org.apache.hadoop.yarn.api.records.Resource) LocalResource(org.apache.hadoop.yarn.api.records.LocalResource) IncreaseContainersResourceRequest(org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest) ContainerLaunchContext(org.apache.hadoop.yarn.api.records.ContainerLaunchContext) LocalResource(org.apache.hadoop.yarn.api.records.LocalResource) StartContainerRequest(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest) File(java.io.File) Test(org.junit.Test)

Aggregations

Container (org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container)109 ContainerId (org.apache.hadoop.yarn.api.records.ContainerId)55 Test (org.junit.Test)43 ApplicationId (org.apache.hadoop.yarn.api.records.ApplicationId)33 Path (org.apache.hadoop.fs.Path)31 ArrayList (java.util.ArrayList)29 Application (org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application)29 HashMap (java.util.HashMap)27 YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)27 Configuration (org.apache.hadoop.conf.Configuration)24 IOException (java.io.IOException)20 ContainerLaunchContext (org.apache.hadoop.yarn.api.records.ContainerLaunchContext)18 ContainerEvent (org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent)17 LocalDirsHandlerService (org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService)16 Collection (java.util.Collection)14 ApplicationAttemptId (org.apache.hadoop.yarn.api.records.ApplicationAttemptId)14 LocalResourceVisibility (org.apache.hadoop.yarn.api.records.LocalResourceVisibility)14 YarnException (org.apache.hadoop.yarn.exceptions.YarnException)14 LocalResource (org.apache.hadoop.yarn.api.records.LocalResource)13 ApplicationEvent (org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent)13