Examples with LocalizerStatus org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus used on opensource projects

Example 1 with LocalizerStatus

use of org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus in project hadoop by apache.

the class ContainerLocalizer method localizeFiles.

protected void localizeFiles(LocalizationProtocol nodemanager, CompletionService<Path> cs, UserGroupInformation ugi) throws IOException, YarnException {
    while (true) {
        try {
            LocalizerStatus status = createStatus();
            LocalizerHeartbeatResponse response = nodemanager.heartbeat(status);
            switch(response.getLocalizerAction()) {
                case LIVE:
                    List<ResourceLocalizationSpec> newRsrcs = response.getResourceSpecs();
                    for (ResourceLocalizationSpec newRsrc : newRsrcs) {
                        if (!pendingResources.containsKey(newRsrc.getResource())) {
                            pendingResources.put(newRsrc.getResource(), cs.submit(download(new Path(newRsrc.getDestinationDirectory().getFile()), newRsrc.getResource(), ugi)));
                        }
                    }
                    break;
                case DIE:
                    // killall running localizations
                    for (Future<Path> pending : pendingResources.values()) {
                        pending.cancel(true);
                    }
                    status = createStatus();
                    // ignore response while dying.
                    try {
                        nodemanager.heartbeat(status);
                    } catch (YarnException e) {
                        // Cannot do anything about this during death stage, let's just log
                        // it.
                        e.printStackTrace(System.out);
                        LOG.error("Heartbeat failed while dying: ", e);
                    }
                    return;
            }
            cs.poll(1000, TimeUnit.MILLISECONDS);
        } catch (InterruptedException e) {
            return;
        } catch (YarnException e) {
            // TODO cleanup
            throw e;
        }
    }
}

Example 2 with LocalizerStatus

use of org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus in project hadoop by apache.

the class ContainerLocalizer method createStatus.

/**
   * Create the payload for the HeartBeat. Mainly the list of
   * {@link LocalResourceStatus}es
   * 
   * @return a {@link LocalizerStatus} that can be sent via heartbeat.
   * @throws InterruptedException
   */
private LocalizerStatus createStatus() throws InterruptedException {
    final List<LocalResourceStatus> currentResources = new ArrayList<LocalResourceStatus>();
    // TODO: Synchronization??
    for (Iterator<LocalResource> i = pendingResources.keySet().iterator(); i.hasNext(); ) {
        LocalResource rsrc = i.next();
        LocalResourceStatus stat = recordFactory.newRecordInstance(LocalResourceStatus.class);
        stat.setResource(rsrc);
        Future<Path> fPath = pendingResources.get(rsrc);
        if (fPath.isDone()) {
            try {
                Path localPath = fPath.get();
                stat.setLocalPath(URL.fromPath(localPath));
                stat.setLocalSize(FileUtil.getDU(new File(localPath.getParent().toUri())));
                stat.setStatus(ResourceStatusType.FETCH_SUCCESS);
            } catch (ExecutionException e) {
                stat.setStatus(ResourceStatusType.FETCH_FAILURE);
                stat.setException(SerializedException.newInstance(e.getCause()));
            } catch (CancellationException e) {
                stat.setStatus(ResourceStatusType.FETCH_FAILURE);
                stat.setException(SerializedException.newInstance(e));
            }
            // TODO shouldn't remove until ACK
            i.remove();
        } else {
            stat.setStatus(ResourceStatusType.FETCH_PENDING);
        }
        currentResources.add(stat);
    }
    LocalizerStatus status = recordFactory.newRecordInstance(LocalizerStatus.class);
    status.setLocalizerId(localizerId);
    status.addAllResources(currentResources);
    return status;
}

Example 3 with LocalizerStatus

use of org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus in project hadoop by apache.

the class TestResourceLocalizationService method testDownloadingResourcesOnContainerKill.

@Test(timeout = 20000)
@SuppressWarnings("unchecked")
public void testDownloadingResourcesOnContainerKill() throws Exception {
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[1];
    localDirs.add(lfs.makeQualified(new Path(basedir, 0 + "")));
    sDirs[0] = localDirs.get(0).toString();
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    DummyExecutor exec = new DummyExecutor();
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    DeletionService delServiceReal = new DeletionService(exec);
    DeletionService delService = spy(delServiceReal);
    delService.init(new Configuration());
    delService.start();
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
    FsPermission defaultPermission = FsPermission.getDirDefault().applyUMask(lfs.getUMask());
    FsPermission nmPermission = ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask());
    final Path userDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.USERCACHE);
    final Path fileDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.FILECACHE);
    final Path sysDir = new Path(sDirs[0].substring("file:".length()), ResourceLocalizationService.NM_PRIVATE_DIR);
    final FileStatus fs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, defaultPermission, "", "", new Path(sDirs[0]));
    final FileStatus nmFs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, nmPermission, "", "", sysDir);
    doAnswer(new Answer<FileStatus>() {

        @Override
        public FileStatus answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            if (args.length > 0) {
                if (args[0].equals(userDir) || args[0].equals(fileDir)) {
                    return fs;
                }
            }
            return nmFs;
        }
    }).when(spylfs).getFileStatus(isA(Path.class));
    try {
        spyService.init(conf);
        spyService.start();
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        String user = "user0";
        when(app.getUser()).thenReturn(user);
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        ArgumentMatcher<ApplicationEvent> matchesAppInit = new ArgumentMatcher<ApplicationEvent>() {

            @Override
            public boolean matches(Object o) {
                ApplicationEvent evt = (ApplicationEvent) o;
                return evt.getType() == ApplicationEventType.APPLICATION_INITED && appId == evt.getApplicationID();
            }
        };
        dispatcher.await();
        verify(applicationBus).handle(argThat(matchesAppInit));
        // Initialize localizer.
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        final Container c1 = getMockContainer(appId, 42, "user0");
        final Container c2 = getMockContainer(appId, 43, "user0");
        FSDataOutputStream out = new FSDataOutputStream(new DataOutputBuffer(), null);
        doReturn(out).when(spylfs).createInternal(isA(Path.class), isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean());
        final LocalResource resource1 = getPrivateMockedResource(r);
        LocalResource resource2 = null;
        do {
            resource2 = getPrivateMockedResource(r);
        } while (resource2 == null || resource2.equals(resource1));
        LocalResource resource3 = null;
        do {
            resource3 = getPrivateMockedResource(r);
        } while (resource3 == null || resource3.equals(resource1) || resource3.equals(resource2));
        // Send localization requests for container c1 and c2.
        final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
        final LocalResourceRequest req2 = new LocalResourceRequest(resource2);
        final LocalResourceRequest req3 = new LocalResourceRequest(resource3);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>();
        privateResourceList.add(req1);
        privateResourceList.add(req2);
        privateResourceList.add(req3);
        rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);
        spyService.handle(new ContainerLocalizationRequestEvent(c1, rsrcs));
        final LocalResourceRequest req1_1 = new LocalResourceRequest(resource2);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs1 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        List<LocalResourceRequest> privateResourceList1 = new ArrayList<LocalResourceRequest>();
        privateResourceList1.add(req1_1);
        rsrcs1.put(LocalResourceVisibility.PRIVATE, privateResourceList1);
        spyService.handle(new ContainerLocalizationRequestEvent(c2, rsrcs1));
        dispatcher.await();
        // Wait for localizers of both container c1 and c2 to begin.
        exec.waitForLocalizers(2);
        LocalizerRunner locC1 = spyService.getLocalizerRunner(c1.getContainerId().toString());
        final String containerIdStr = c1.getContainerId().toString();
        // Heartbeats from container localizer
        LocalResourceStatus rsrc1success = mock(LocalResourceStatus.class);
        LocalResourceStatus rsrc2pending = mock(LocalResourceStatus.class);
        LocalizerStatus stat = mock(LocalizerStatus.class);
        when(stat.getLocalizerId()).thenReturn(containerIdStr);
        when(rsrc1success.getResource()).thenReturn(resource1);
        when(rsrc2pending.getResource()).thenReturn(resource2);
        when(rsrc1success.getLocalSize()).thenReturn(4344L);
        URL locPath = getPath("/some/path");
        when(rsrc1success.getLocalPath()).thenReturn(locPath);
        when(rsrc1success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
        when(rsrc2pending.getStatus()).thenReturn(ResourceStatusType.FETCH_PENDING);
        when(stat.getResources()).thenReturn(Collections.<LocalResourceStatus>emptyList()).thenReturn(Collections.singletonList(rsrc1success)).thenReturn(Collections.singletonList(rsrc2pending)).thenReturn(Collections.singletonList(rsrc2pending)).thenReturn(Collections.<LocalResourceStatus>emptyList());
        // First heartbeat which schedules first resource.
        LocalizerHeartbeatResponse response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
        // Second heartbeat which reports first resource as success.
        // Second resource is scheduled.
        response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
        final String locPath1 = response.getResourceSpecs().get(0).getDestinationDirectory().getFile();
        // Third heartbeat which reports second resource as pending.
        // Third resource is scheduled.
        response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
        final String locPath2 = response.getResourceSpecs().get(0).getDestinationDirectory().getFile();
        // Container c1 is killed which leads to cleanup
        spyService.handle(new ContainerLocalizationCleanupEvent(c1, rsrcs));
        // This heartbeat will indicate to container localizer to die as localizer
        // runner has stopped.
        response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.DIE, response.getLocalizerAction());
        exec.setStopLocalization();
        dispatcher.await();
        // verify container notification
        ArgumentMatcher<ContainerEvent> successContainerLoc = new ArgumentMatcher<ContainerEvent>() {

            @Override
            public boolean matches(Object o) {
                ContainerEvent evt = (ContainerEvent) o;
                return evt.getType() == ContainerEventType.RESOURCE_LOCALIZED && c1.getContainerId() == evt.getContainerID();
            }
        };
        // Only one resource gets localized for container c1.
        verify(containerBus).handle(argThat(successContainerLoc));
        Set<Path> paths = Sets.newHashSet(new Path(locPath1), new Path(locPath1 + "_tmp"), new Path(locPath2), new Path(locPath2 + "_tmp"));
        // Wait for localizer runner thread for container c1 to finish.
        while (locC1.getState() != Thread.State.TERMINATED) {
            Thread.sleep(50);
        }
        // Verify if downloading resources were submitted for deletion.
        verify(delService).delete(eq(user), (Path) eq(null), argThat(new DownloadingPathsMatcher(paths)));
        LocalResourcesTracker tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, "user0", appId);
        // Container c1 was killed but this resource was localized before kill
        // hence its not removed despite ref cnt being 0.
        LocalizedResource rsrc1 = tracker.getLocalizedResource(req1);
        assertNotNull(rsrc1);
        assertEquals(rsrc1.getState(), ResourceState.LOCALIZED);
        assertEquals(rsrc1.getRefCount(), 0);
        // Container c1 was killed but this resource is referenced by container c2
        // as well hence its ref cnt is 1.
        LocalizedResource rsrc2 = tracker.getLocalizedResource(req2);
        assertNotNull(rsrc2);
        assertEquals(rsrc2.getState(), ResourceState.DOWNLOADING);
        assertEquals(rsrc2.getRefCount(), 1);
        // As container c1 was killed and this resource was not referenced by any
        // other container, hence its removed.
        LocalizedResource rsrc3 = tracker.getLocalizedResource(req3);
        assertNull(rsrc3);
    } finally {
        spyService.stop();
        dispatcher.stop();
        delService.stop();
    }
}

Example 4 with LocalizerStatus

use of org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus in project hadoop by apache.

the class TestResourceLocalizationService method createLocalizerStatus.

private LocalizerStatus createLocalizerStatus(String localizerId1) {
    LocalizerStatus status = new LocalizerStatusPBImpl();
    status.setLocalizerId(localizerId1);
    return status;
}

Example 5 with LocalizerStatus

use of org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus in project hadoop by apache.

the class TestContainerLocalizer method testMain.

@Test
public void testMain() throws Exception {
    ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper();
    ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest();
    Random random = wrapper.random;
    List<Path> localDirs = wrapper.localDirs;
    Path tokenPath = wrapper.tokenPath;
    LocalizationProtocol nmProxy = wrapper.nmProxy;
    AbstractFileSystem spylfs = wrapper.spylfs;
    mockOutDownloads(localizer);
    // verify created cache
    List<Path> privCacheList = new ArrayList<Path>();
    List<Path> appCacheList = new ArrayList<Path>();
    for (Path p : localDirs) {
        Path base = new Path(new Path(p, ContainerLocalizer.USERCACHE), appUser);
        Path privcache = new Path(base, ContainerLocalizer.FILECACHE);
        privCacheList.add(privcache);
        Path appDir = new Path(base, new Path(ContainerLocalizer.APPCACHE, appId));
        Path appcache = new Path(appDir, ContainerLocalizer.FILECACHE);
        appCacheList.add(appcache);
    }
    // mock heartbeat responses from NM
    ResourceLocalizationSpec rsrcA = getMockRsrc(random, LocalResourceVisibility.PRIVATE, privCacheList.get(0));
    ResourceLocalizationSpec rsrcB = getMockRsrc(random, LocalResourceVisibility.PRIVATE, privCacheList.get(0));
    ResourceLocalizationSpec rsrcC = getMockRsrc(random, LocalResourceVisibility.APPLICATION, appCacheList.get(0));
    ResourceLocalizationSpec rsrcD = getMockRsrc(random, LocalResourceVisibility.PRIVATE, privCacheList.get(0));
    when(nmProxy.heartbeat(isA(LocalizerStatus.class))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcA))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcB))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcC))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcD))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.<ResourceLocalizationSpec>emptyList())).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.DIE, null));
    LocalResource tRsrcA = rsrcA.getResource();
    LocalResource tRsrcB = rsrcB.getResource();
    LocalResource tRsrcC = rsrcC.getResource();
    LocalResource tRsrcD = rsrcD.getResource();
    doReturn(new FakeDownload(rsrcA.getResource().getResource().getFile(), true)).when(localizer).download(isA(Path.class), eq(tRsrcA), isA(UserGroupInformation.class));
    doReturn(new FakeDownload(rsrcB.getResource().getResource().getFile(), true)).when(localizer).download(isA(Path.class), eq(tRsrcB), isA(UserGroupInformation.class));
    doReturn(new FakeDownload(rsrcC.getResource().getResource().getFile(), true)).when(localizer).download(isA(Path.class), eq(tRsrcC), isA(UserGroupInformation.class));
    doReturn(new FakeDownload(rsrcD.getResource().getResource().getFile(), true)).when(localizer).download(isA(Path.class), eq(tRsrcD), isA(UserGroupInformation.class));
    // run localization
    localizer.runLocalization(nmAddr);
    for (Path p : localDirs) {
        Path base = new Path(new Path(p, ContainerLocalizer.USERCACHE), appUser);
        Path privcache = new Path(base, ContainerLocalizer.FILECACHE);
        // $x/usercache/$user/filecache
        verify(spylfs).mkdir(eq(privcache), eq(CACHE_DIR_PERM), eq(false));
        Path appDir = new Path(base, new Path(ContainerLocalizer.APPCACHE, appId));
        // $x/usercache/$user/appcache/$appId/filecache
        Path appcache = new Path(appDir, ContainerLocalizer.FILECACHE);
        verify(spylfs).mkdir(eq(appcache), eq(CACHE_DIR_PERM), eq(false));
    }
    // verify tokens read at expected location
    verify(spylfs).open(tokenPath);
    // verify downloaded resources reported to NM
    verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcA.getResource())));
    verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcB.getResource())));
    verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcC.getResource())));
    verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcD.getResource())));
    // verify all HB use localizerID provided
    verify(nmProxy, never()).heartbeat(argThat(new ArgumentMatcher<LocalizerStatus>() {

        @Override
        public boolean matches(Object o) {
            LocalizerStatus status = (LocalizerStatus) o;
            return !containerId.equals(status.getLocalizerId());
        }
    }));
}