Examples with LocalizationProtocol org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol used on opensource projects

Example 1 with LocalizationProtocol

use of org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol in project hadoop by apache.

the class TestContainerLocalizer method testMain.

@Test
public void testMain() throws Exception {
    ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper();
    ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest();
    Random random = wrapper.random;
    List<Path> localDirs = wrapper.localDirs;
    Path tokenPath = wrapper.tokenPath;
    LocalizationProtocol nmProxy = wrapper.nmProxy;
    AbstractFileSystem spylfs = wrapper.spylfs;
    mockOutDownloads(localizer);
    // verify created cache
    List<Path> privCacheList = new ArrayList<Path>();
    List<Path> appCacheList = new ArrayList<Path>();
    for (Path p : localDirs) {
        Path base = new Path(new Path(p, ContainerLocalizer.USERCACHE), appUser);
        Path privcache = new Path(base, ContainerLocalizer.FILECACHE);
        privCacheList.add(privcache);
        Path appDir = new Path(base, new Path(ContainerLocalizer.APPCACHE, appId));
        Path appcache = new Path(appDir, ContainerLocalizer.FILECACHE);
        appCacheList.add(appcache);
    }
    // mock heartbeat responses from NM
    ResourceLocalizationSpec rsrcA = getMockRsrc(random, LocalResourceVisibility.PRIVATE, privCacheList.get(0));
    ResourceLocalizationSpec rsrcB = getMockRsrc(random, LocalResourceVisibility.PRIVATE, privCacheList.get(0));
    ResourceLocalizationSpec rsrcC = getMockRsrc(random, LocalResourceVisibility.APPLICATION, appCacheList.get(0));
    ResourceLocalizationSpec rsrcD = getMockRsrc(random, LocalResourceVisibility.PRIVATE, privCacheList.get(0));
    when(nmProxy.heartbeat(isA(LocalizerStatus.class))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcA))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcB))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcC))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcD))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.<ResourceLocalizationSpec>emptyList())).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.DIE, null));
    LocalResource tRsrcA = rsrcA.getResource();
    LocalResource tRsrcB = rsrcB.getResource();
    LocalResource tRsrcC = rsrcC.getResource();
    LocalResource tRsrcD = rsrcD.getResource();
    doReturn(new FakeDownload(rsrcA.getResource().getResource().getFile(), true)).when(localizer).download(isA(Path.class), eq(tRsrcA), isA(UserGroupInformation.class));
    doReturn(new FakeDownload(rsrcB.getResource().getResource().getFile(), true)).when(localizer).download(isA(Path.class), eq(tRsrcB), isA(UserGroupInformation.class));
    doReturn(new FakeDownload(rsrcC.getResource().getResource().getFile(), true)).when(localizer).download(isA(Path.class), eq(tRsrcC), isA(UserGroupInformation.class));
    doReturn(new FakeDownload(rsrcD.getResource().getResource().getFile(), true)).when(localizer).download(isA(Path.class), eq(tRsrcD), isA(UserGroupInformation.class));
    // run localization
    localizer.runLocalization(nmAddr);
    for (Path p : localDirs) {
        Path base = new Path(new Path(p, ContainerLocalizer.USERCACHE), appUser);
        Path privcache = new Path(base, ContainerLocalizer.FILECACHE);
        // $x/usercache/$user/filecache
        verify(spylfs).mkdir(eq(privcache), eq(CACHE_DIR_PERM), eq(false));
        Path appDir = new Path(base, new Path(ContainerLocalizer.APPCACHE, appId));
        // $x/usercache/$user/appcache/$appId/filecache
        Path appcache = new Path(appDir, ContainerLocalizer.FILECACHE);
        verify(spylfs).mkdir(eq(appcache), eq(CACHE_DIR_PERM), eq(false));
    }
    // verify tokens read at expected location
    verify(spylfs).open(tokenPath);
    // verify downloaded resources reported to NM
    verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcA.getResource())));
    verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcB.getResource())));
    verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcC.getResource())));
    verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcD.getResource())));
    // verify all HB use localizerID provided
    verify(nmProxy, never()).heartbeat(argThat(new ArgumentMatcher<LocalizerStatus>() {

        @Override
        public boolean matches(Object o) {
            LocalizerStatus status = (LocalizerStatus) o;
            return !containerId.equals(status.getLocalizerId());
        }
    }));
}

Example 2 with LocalizationProtocol

use of org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol in project hadoop by apache.

the class TestRPCFactories method testPbServerFactory.

private void testPbServerFactory() {
    InetSocketAddress addr = new InetSocketAddress(0);
    Configuration conf = new Configuration();
    LocalizationProtocol instance = new LocalizationProtocolTestImpl();
    Server server = null;
    try {
        server = RpcServerFactoryPBImpl.get().getServer(LocalizationProtocol.class, instance, addr, conf, null, 1);
        server.start();
    } catch (YarnRuntimeException e) {
        e.printStackTrace();
        Assert.fail("Failed to create server");
    } finally {
        if (server != null) {
            server.stop();
        }
    }
}

Example 3 with LocalizationProtocol

use of org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol in project hadoop by apache.

the class ContainerLocalizer method runLocalization.

@SuppressWarnings("deprecation")
public void runLocalization(final InetSocketAddress nmAddr) throws IOException, InterruptedException {
    // load credentials
    initDirs(conf, user, appId, lfs, localDirs);
    final Credentials creds = new Credentials();
    DataInputStream credFile = null;
    try {
        // assume credentials in cwd
        // TODO: Fix
        Path tokenPath = new Path(String.format(TOKEN_FILE_NAME_FMT, localizerId));
        credFile = lfs.open(tokenPath);
        creds.readTokenStorageStream(credFile);
        // Explicitly deleting token file.
        lfs.delete(tokenPath, false);
    } finally {
        if (credFile != null) {
            credFile.close();
        }
    }
    // create localizer context
    UserGroupInformation remoteUser = UserGroupInformation.createRemoteUser(user);
    remoteUser.addToken(creds.getToken(LocalizerTokenIdentifier.KIND));
    final LocalizationProtocol nodeManager = remoteUser.doAs(new PrivilegedAction<LocalizationProtocol>() {

        @Override
        public LocalizationProtocol run() {
            return getProxy(nmAddr);
        }
    });
    // create user context
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
    for (Token<? extends TokenIdentifier> token : creds.getAllTokens()) {
        ugi.addToken(token);
    }
    ExecutorService exec = null;
    try {
        exec = createDownloadThreadPool();
        CompletionService<Path> ecs = createCompletionService(exec);
        localizeFiles(nodeManager, ecs, ugi);
    } catch (Throwable e) {
        throw new IOException(e);
    } finally {
        try {
            if (exec != null) {
                exec.shutdown();
                destroyShellProcesses(getAllShells());
                exec.awaitTermination(10, TimeUnit.SECONDS);
            }
            LocalDirAllocator.removeContext(appCacheDirContextName);
        } finally {
            closeFileSystems(ugi);
        }
    }
}

Example 4 with LocalizationProtocol

use of org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol in project hadoop by apache.

the class TestRPCFactories method testPbClientFactory.

private void testPbClientFactory() {
    InetSocketAddress addr = new InetSocketAddress(0);
    System.err.println(addr.getHostName() + addr.getPort());
    Configuration conf = new Configuration();
    LocalizationProtocol instance = new LocalizationProtocolTestImpl();
    Server server = null;
    try {
        server = RpcServerFactoryPBImpl.get().getServer(LocalizationProtocol.class, instance, addr, conf, null, 1);
        server.start();
        System.err.println(server.getListenerAddress());
        System.err.println(NetUtils.getConnectAddress(server));
        try {
            LocalizationProtocol client = (LocalizationProtocol) RpcClientFactoryPBImpl.get().getClient(LocalizationProtocol.class, 1, NetUtils.getConnectAddress(server), conf);
            Assert.assertNotNull(client);
        } catch (YarnRuntimeException e) {
            e.printStackTrace();
            Assert.fail("Failed to create client");
        }
    } catch (YarnRuntimeException e) {
        e.printStackTrace();
        Assert.fail("Failed to create server");
    } finally {
        server.stop();
    }
}

Example 5 with LocalizationProtocol

use of org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol in project hadoop by apache.

the class TestDefaultContainerExecutor method testStartLocalizer.

@Test(timeout = 30000)
public void testStartLocalizer() throws IOException, InterruptedException, YarnException {
    final Path firstDir = new Path(BASE_TMP_PATH, "localDir1");
    List<String> localDirs = new ArrayList<String>();
    final Path secondDir = new Path(BASE_TMP_PATH, "localDir2");
    List<String> logDirs = new ArrayList<String>();
    final Path logDir = new Path(BASE_TMP_PATH, "logDir");
    final Path tokenDir = new Path(BASE_TMP_PATH, "tokenDir");
    FsPermission perms = new FsPermission((short) 0770);
    Configuration conf = new Configuration();
    final FileContext mockLfs = spy(FileContext.getLocalFSFileContext(conf));
    final FileContext.Util mockUtil = spy(mockLfs.util());
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            return mockUtil;
        }
    }).when(mockLfs).util();
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            Path dest = (Path) invocationOnMock.getArguments()[1];
            if (dest.toString().contains(firstDir.toString())) {
                // to simulate no space on the first drive
                throw new IOException("No space on this drive " + dest.toString());
            } else {
                // copy token to the second local dir
                DataOutputStream tokenOut = null;
                try {
                    Credentials credentials = new Credentials();
                    tokenOut = mockLfs.create(dest, EnumSet.of(CREATE, OVERWRITE));
                    credentials.writeTokenStorageToStream(tokenOut);
                } finally {
                    if (tokenOut != null) {
                        tokenOut.close();
                    }
                }
            }
            return null;
        }
    }).when(mockUtil).copy(any(Path.class), any(Path.class), anyBoolean(), anyBoolean());
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            Path p = (Path) invocationOnMock.getArguments()[0];
            // first local directory
            if (p.toString().contains(firstDir.toString())) {
                return new FsStatus(2000, 2000, 0);
            } else {
                return new FsStatus(1000, 0, 1000);
            }
        }
    }).when(mockLfs).getFsStatus(any(Path.class));
    DefaultContainerExecutor mockExec = spy(new DefaultContainerExecutor(mockLfs) {

        @Override
        public ContainerLocalizer createContainerLocalizer(String user, String appId, String locId, List<String> localDirs, FileContext localizerFc) throws IOException {
            // Spy on the localizer and make it return valid heart-beat
            // responses even though there is no real NodeManager.
            ContainerLocalizer localizer = super.createContainerLocalizer(user, appId, locId, localDirs, localizerFc);
            ContainerLocalizer spyLocalizer = spy(localizer);
            LocalizationProtocol nmProxy = mock(LocalizationProtocol.class);
            try {
                when(nmProxy.heartbeat(isA(LocalizerStatus.class))).thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.DIE, new ArrayList<ResourceLocalizationSpec>()));
            } catch (YarnException e) {
                throw new IOException(e);
            }
            when(spyLocalizer.getProxy(any(InetSocketAddress.class))).thenReturn(nmProxy);
            return spyLocalizer;
        }
    });
    mockExec.setConf(conf);
    localDirs.add(mockLfs.makeQualified(firstDir).toString());
    localDirs.add(mockLfs.makeQualified(secondDir).toString());
    logDirs.add(mockLfs.makeQualified(logDir).toString());
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, localDirs.toArray(new String[localDirs.size()]));
    conf.set(YarnConfiguration.NM_LOG_DIRS, logDir.toString());
    mockLfs.mkdir(tokenDir, perms, true);
    Path nmPrivateCTokensPath = new Path(tokenDir, "test.tokens");
    String appSubmitter = "nobody";
    String appId = "APP_ID";
    String locId = "LOC_ID";
    LocalDirsHandlerService dirsHandler = mock(LocalDirsHandlerService.class);
    when(dirsHandler.getLocalDirs()).thenReturn(localDirs);
    when(dirsHandler.getLogDirs()).thenReturn(logDirs);
    try {
        mockExec.startLocalizer(new LocalizerStartContext.Builder().setNmPrivateContainerTokens(nmPrivateCTokensPath).setNmAddr(null).setUser(appSubmitter).setAppId(appId).setLocId(locId).setDirsHandler(dirsHandler).build());
    } catch (IOException e) {
        Assert.fail("StartLocalizer failed to copy token file: " + StringUtils.stringifyException(e));
    } finally {
        mockExec.deleteAsUser(new DeletionAsUserContext.Builder().setUser(appSubmitter).setSubDir(firstDir).build());
        mockExec.deleteAsUser(new DeletionAsUserContext.Builder().setUser(appSubmitter).setSubDir(secondDir).build());
        mockExec.deleteAsUser(new DeletionAsUserContext.Builder().setUser(appSubmitter).setSubDir(logDir).build());
        deleteTmpFiles();
    }
    // Verify that the calls happen the expected number of times
    verify(mockUtil, times(1)).copy(any(Path.class), any(Path.class), anyBoolean(), anyBoolean());
    verify(mockLfs, times(2)).getFsStatus(any(Path.class));
}