Examples with ResourceLocalizationSpec org.apache.hadoop.yarn.server.nodemanager.api.ResourceLocalizationSpec used on opensource projects

Example 1 with ResourceLocalizationSpec

use of org.apache.hadoop.yarn.server.nodemanager.api.ResourceLocalizationSpec in project hadoop by apache.

the class LocalizerHeartbeatResponsePBImpl method initResources.

private void initResources() {
    if (this.resourceSpecs != null) {
        return;
    }
    LocalizerHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
    List<ResourceLocalizationSpecProto> list = p.getResourcesList();
    this.resourceSpecs = new ArrayList<ResourceLocalizationSpec>();
    for (ResourceLocalizationSpecProto c : list) {
        this.resourceSpecs.add(convertFromProtoFormat(c));
    }
}

Example 2 with ResourceLocalizationSpec

use of org.apache.hadoop.yarn.server.nodemanager.api.ResourceLocalizationSpec in project hadoop by apache.

the class ContainerLocalizer method localizeFiles.

protected void localizeFiles(LocalizationProtocol nodemanager, CompletionService<Path> cs, UserGroupInformation ugi) throws IOException, YarnException {
    while (true) {
        try {
            LocalizerStatus status = createStatus();
            LocalizerHeartbeatResponse response = nodemanager.heartbeat(status);
            switch(response.getLocalizerAction()) {
                case LIVE:
                    List<ResourceLocalizationSpec> newRsrcs = response.getResourceSpecs();
                    for (ResourceLocalizationSpec newRsrc : newRsrcs) {
                        if (!pendingResources.containsKey(newRsrc.getResource())) {
                            pendingResources.put(newRsrc.getResource(), cs.submit(download(new Path(newRsrc.getDestinationDirectory().getFile()), newRsrc.getResource(), ugi)));
                        }
                    }
                    break;
                case DIE:
                    // killall running localizations
                    for (Future<Path> pending : pendingResources.values()) {
                        pending.cancel(true);
                    }
                    status = createStatus();
                    // ignore response while dying.
                    try {
                        nodemanager.heartbeat(status);
                    } catch (YarnException e) {
                        // Cannot do anything about this during death stage, let's just log
                        // it.
                        e.printStackTrace(System.out);
                        LOG.error("Heartbeat failed while dying: ", e);
                    }
                    return;
            }
            cs.poll(1000, TimeUnit.MILLISECONDS);
        } catch (InterruptedException e) {
            return;
        } catch (YarnException e) {
            // TODO cleanup
            throw e;
        }
    }
}

Example 3 with ResourceLocalizationSpec

use of org.apache.hadoop.yarn.server.nodemanager.api.ResourceLocalizationSpec in project hadoop by apache.

the class TestResourceLocalizationService method testLocalResourcePath.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void testLocalResourcePath() throws Exception {
    // test the local path where application and user cache files will be
    // localized.
    DrainDispatcher dispatcher1 = null;
    try {
        dispatcher1 = new DrainDispatcher();
        String user = "testuser";
        ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
        // creating one local directory
        List<Path> localDirs = new ArrayList<Path>();
        String[] sDirs = new String[1];
        for (int i = 0; i < 1; ++i) {
            localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
            sDirs[i] = localDirs.get(i).toString();
        }
        conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
        LocalDirsHandlerService localDirHandler = new LocalDirsHandlerService();
        localDirHandler.init(conf);
        // Registering event handlers
        EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
        dispatcher1.register(ApplicationEventType.class, applicationBus);
        EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
        dispatcher1.register(ContainerEventType.class, containerBus);
        ContainerExecutor exec = mock(ContainerExecutor.class);
        DeletionService delService = mock(DeletionService.class);
        LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
        // initializing directory handler.
        dirsHandler.init(conf);
        dispatcher1.init(conf);
        dispatcher1.start();
        ResourceLocalizationService rls = new ResourceLocalizationService(dispatcher1, exec, delService, localDirHandler, nmContext);
        dispatcher1.register(LocalizationEventType.class, rls);
        rls.init(conf);
        rls.handle(createApplicationLocalizationEvent(user, appId));
        // We need to pre-populate the LocalizerRunner as the
        // Resource Localization Service code internally starts them which
        // definitely we don't want.
        // creating new container and populating corresponding localizer runner
        // Container - 1
        Container container1 = createMockContainer(user, 1);
        String localizerId1 = container1.getContainerId().toString();
        rls.getPrivateLocalizers().put(localizerId1, rls.new LocalizerRunner(new LocalizerContext(user, container1.getContainerId(), null), localizerId1));
        // Creating two requests for container
        // 1) Private resource
        // 2) Application resource
        LocalResourceRequest reqPriv = new LocalResourceRequest(new Path("file:///tmp1"), 123L, LocalResourceType.FILE, LocalResourceVisibility.PRIVATE, "");
        List<LocalResourceRequest> privList = new ArrayList<LocalResourceRequest>();
        privList.add(reqPriv);
        LocalResourceRequest reqApp = new LocalResourceRequest(new Path("file:///tmp2"), 123L, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, "");
        List<LocalResourceRequest> appList = new ArrayList<LocalResourceRequest>();
        appList.add(reqApp);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        rsrcs.put(LocalResourceVisibility.APPLICATION, appList);
        rsrcs.put(LocalResourceVisibility.PRIVATE, privList);
        dispatcher1.getEventHandler().handle(new ContainerLocalizationRequestEvent(container1, rsrcs));
        // Now waiting for resource download to start. Here actual will not start
        // Only the resources will be populated into pending list.
        Assert.assertTrue(waitForPrivateDownloadToStart(rls, localizerId1, 2, 5000));
        // Validating user and application cache paths
        String userCachePath = StringUtils.join(Path.SEPARATOR, Arrays.asList(localDirs.get(0).toUri().getRawPath(), ContainerLocalizer.USERCACHE, user, ContainerLocalizer.FILECACHE));
        String userAppCachePath = StringUtils.join(Path.SEPARATOR, Arrays.asList(localDirs.get(0).toUri().getRawPath(), ContainerLocalizer.USERCACHE, user, ContainerLocalizer.APPCACHE, appId.toString(), ContainerLocalizer.FILECACHE));
        // Now the Application and private resources may come in any order
        // for download.
        // For User cahce :
        // returned destinationPath = user cache path + random number
        // For App cache :
        // returned destinationPath = user app cache path + random number
        int returnedResources = 0;
        boolean appRsrc = false, privRsrc = false;
        while (returnedResources < 2) {
            LocalizerHeartbeatResponse response = rls.heartbeat(createLocalizerStatus(localizerId1));
            for (ResourceLocalizationSpec resourceSpec : response.getResourceSpecs()) {
                returnedResources++;
                Path destinationDirectory = new Path(resourceSpec.getDestinationDirectory().getFile());
                if (resourceSpec.getResource().getVisibility() == LocalResourceVisibility.APPLICATION) {
                    appRsrc = true;
                    Assert.assertEquals(userAppCachePath, destinationDirectory.getParent().toUri().toString());
                } else if (resourceSpec.getResource().getVisibility() == LocalResourceVisibility.PRIVATE) {
                    privRsrc = true;
                    Assert.assertEquals(userCachePath, destinationDirectory.getParent().toUri().toString());
                } else {
                    throw new Exception("Unexpected resource received.");
                }
            }
        }
        // We should receive both the resources (Application and Private)
        Assert.assertTrue(appRsrc && privRsrc);
    } finally {
        if (dispatcher1 != null) {
            dispatcher1.stop();
        }
    }
}

Example 4 with ResourceLocalizationSpec

use of org.apache.hadoop.yarn.server.nodemanager.api.ResourceLocalizationSpec in project hadoop by apache.

the class TestContainerLocalizer method testMain.

@Test
public void testMain() throws Exception {
    ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper();
    ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest();
    Random random = wrapper.random;
    List<Path> localDirs = wrapper.localDirs;
    Path tokenPath = wrapper.tokenPath;
    LocalizationProtocol nmProxy = wrapper.nmProxy;
    AbstractFileSystem spylfs = wrapper.spylfs;
    mockOutDownloads(localizer);
    // verify created cache
    List<Path> privCacheList = new ArrayList<Path>();
    List<Path> appCacheList = new ArrayList<Path>();
    for (Path p : localDirs) {
        Path base = new Path(new Path(p, ContainerLocalizer.USERCACHE), appUser);
        Path privcache = new Path(base, ContainerLocalizer.FILECACHE);
        privCacheList.add(privcache);
        Path appDir = new Path(base, new Path(ContainerLocalizer.APPCACHE, appId));
        Path appcache = new Path(appDir, ContainerLocalizer.FILECACHE);
        appCacheList.add(appcache);
    }
    // mock heartbeat responses from NM
    ResourceLocalizationSpec rsrcA = getMockRsrc(random, LocalResourceVisibility.PRIVATE, privCacheList.get(0));
    ResourceLocalizationSpec rsrcB = getMockRsrc(random, LocalResourceVisibility.PRIVATE, privCacheList.get(0));
    ResourceLocalizationSpec rsrcC = getMockRsrc(random, LocalResourceVisibility.APPLICATION, appCacheList.get(0));
    ResourceLocalizationSpec rsrcD = getMockRsrc(random, LocalResourceVisibility.PRIVATE, privCacheList.get(0));
    when(nmProxy.heartbeat(isA(LocalizerStatus.class))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcA))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcB))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcC))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcD))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.<ResourceLocalizationSpec>emptyList())).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.DIE, null));
    LocalResource tRsrcA = rsrcA.getResource();
    LocalResource tRsrcB = rsrcB.getResource();
    LocalResource tRsrcC = rsrcC.getResource();
    LocalResource tRsrcD = rsrcD.getResource();
    doReturn(new FakeDownload(rsrcA.getResource().getResource().getFile(), true)).when(localizer).download(isA(Path.class), eq(tRsrcA), isA(UserGroupInformation.class));
    doReturn(new FakeDownload(rsrcB.getResource().getResource().getFile(), true)).when(localizer).download(isA(Path.class), eq(tRsrcB), isA(UserGroupInformation.class));
    doReturn(new FakeDownload(rsrcC.getResource().getResource().getFile(), true)).when(localizer).download(isA(Path.class), eq(tRsrcC), isA(UserGroupInformation.class));
    doReturn(new FakeDownload(rsrcD.getResource().getResource().getFile(), true)).when(localizer).download(isA(Path.class), eq(tRsrcD), isA(UserGroupInformation.class));
    // run localization
    localizer.runLocalization(nmAddr);
    for (Path p : localDirs) {
        Path base = new Path(new Path(p, ContainerLocalizer.USERCACHE), appUser);
        Path privcache = new Path(base, ContainerLocalizer.FILECACHE);
        // $x/usercache/$user/filecache
        verify(spylfs).mkdir(eq(privcache), eq(CACHE_DIR_PERM), eq(false));
        Path appDir = new Path(base, new Path(ContainerLocalizer.APPCACHE, appId));
        // $x/usercache/$user/appcache/$appId/filecache
        Path appcache = new Path(appDir, ContainerLocalizer.FILECACHE);
        verify(spylfs).mkdir(eq(appcache), eq(CACHE_DIR_PERM), eq(false));
    }
    // verify tokens read at expected location
    verify(spylfs).open(tokenPath);
    // verify downloaded resources reported to NM
    verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcA.getResource())));
    verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcB.getResource())));
    verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcC.getResource())));
    verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcD.getResource())));
    // verify all HB use localizerID provided
    verify(nmProxy, never()).heartbeat(argThat(new ArgumentMatcher<LocalizerStatus>() {

        @Override
        public boolean matches(Object o) {
            LocalizerStatus status = (LocalizerStatus) o;
            return !containerId.equals(status.getLocalizerId());
        }
    }));
}

Example 5 with ResourceLocalizationSpec

use of org.apache.hadoop.yarn.server.nodemanager.api.ResourceLocalizationSpec in project hadoop by apache.

the class TestContainerLocalizer method getMockRsrc.

static ResourceLocalizationSpec getMockRsrc(Random r, LocalResourceVisibility vis, Path p) {
    ResourceLocalizationSpec resourceLocalizationSpec = mock(ResourceLocalizationSpec.class);
    LocalResource rsrc = mock(LocalResource.class);
    String name = Long.toHexString(r.nextLong());
    URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
    when(uri.getScheme()).thenReturn("file");
    when(uri.getHost()).thenReturn(null);
    when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);
    when(rsrc.getResource()).thenReturn(uri);
    when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
    when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
    when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
    when(rsrc.getVisibility()).thenReturn(vis);
    when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
    when(resourceLocalizationSpec.getDestinationDirectory()).thenReturn(URL.fromPath(p));
    return resourceLocalizationSpec;
}