Examples with YarnRPC org.apache.hadoop.yarn.ipc.YarnRPC used on opensource projects

Example 21 with YarnRPC

use of org.apache.hadoop.yarn.ipc.YarnRPC in project hadoop by apache.

the class TestNodeManagerShutdown method startContainer.

public static void startContainer(NodeManager nm, ContainerId cId, FileContext localFS, File scriptFileDir, File processStartFile, final int port) throws IOException, YarnException {
    File scriptFile = createUnhaltingScriptFile(cId, scriptFileDir, processStartFile);
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    NodeId nodeId = BuilderUtils.newNodeId(InetAddress.getByName("localhost").getCanonicalHostName(), port);
    URL localResourceUri = URL.fromPath(localFS.makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource localResource = recordFactory.newRecordInstance(LocalResource.class);
    localResource.setResource(localResourceUri);
    localResource.setSize(-1);
    localResource.setVisibility(LocalResourceVisibility.APPLICATION);
    localResource.setType(LocalResourceType.FILE);
    localResource.setTimestamp(scriptFile.lastModified());
    String destinationFile = "dest_file";
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    localResources.put(destinationFile, localResource);
    containerLaunchContext.setLocalResources(localResources);
    List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
    containerLaunchContext.setCommands(commands);
    final InetSocketAddress containerManagerBindAddress = NetUtils.createSocketAddrForHost("127.0.0.1", port);
    UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(cId.toString());
    org.apache.hadoop.security.token.Token<NMTokenIdentifier> nmToken = ConverterUtils.convertFromYarn(nm.getNMContext().getNMTokenSecretManager().createNMToken(cId.getApplicationAttemptId(), nodeId, user), containerManagerBindAddress);
    currentUser.addToken(nmToken);
    ContainerManagementProtocol containerManager = currentUser.doAs(new PrivilegedAction<ContainerManagementProtocol>() {

        @Override
        public ContainerManagementProtocol run() {
            Configuration conf = new Configuration();
            YarnRPC rpc = YarnRPC.create(conf);
            InetSocketAddress containerManagerBindAddress = NetUtils.createSocketAddrForHost("127.0.0.1", port);
            return (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, containerManagerBindAddress, conf);
        }
    });
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, TestContainerManager.createContainerToken(cId, 0, nodeId, user, nm.getNMContext().getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(cId);
    GetContainerStatusesRequest request = GetContainerStatusesRequest.newInstance(containerIds);
    ContainerStatus containerStatus = containerManager.getContainerStatuses(request).getContainerStatuses().get(0);
    Assert.assertTrue(EnumSet.of(ContainerState.RUNNING, ContainerState.SCHEDULED).contains(containerStatus.getState()));
}

Example 22 with YarnRPC

use of org.apache.hadoop.yarn.ipc.YarnRPC in project hadoop by apache.

the class ClientProtocolService method serviceStart.

@Override
protected void serviceStart() throws Exception {
    Configuration conf = getConfig();
    this.metrics = ClientSCMMetrics.getInstance();
    YarnRPC rpc = YarnRPC.create(conf);
    this.server = rpc.getServer(ClientSCMProtocol.class, this, clientBindAddress, // Secret manager null for now (security not supported)
    conf, // Secret manager null for now (security not supported)
    null, conf.getInt(YarnConfiguration.SCM_CLIENT_SERVER_THREAD_COUNT, YarnConfiguration.DEFAULT_SCM_CLIENT_SERVER_THREAD_COUNT));
    // TODO (YARN-2774): Enable service authorization
    this.server.start();
    clientBindAddress = conf.updateConnectAddr(YarnConfiguration.SCM_CLIENT_SERVER_ADDRESS, server.getListenerAddress());
    super.serviceStart();
}

Example 23 with YarnRPC

use of org.apache.hadoop.yarn.ipc.YarnRPC in project hadoop by apache.

the class TestClientSCMProtocolService method startUp.

@Before
public void startUp() {
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.SCM_STORE_CLASS, InMemorySCMStore.class.getName());
    conf.set(YarnConfiguration.SHARED_CACHE_ROOT, testDir.getPath());
    AppChecker appChecker = spy(new DummyAppChecker());
    store = new InMemorySCMStore(appChecker);
    store.init(conf);
    store.start();
    service = new ClientProtocolService(store);
    service.init(conf);
    service.start();
    YarnRPC rpc = YarnRPC.create(new Configuration());
    InetSocketAddress scmAddress = conf.getSocketAddr(YarnConfiguration.SCM_CLIENT_SERVER_ADDRESS, YarnConfiguration.DEFAULT_SCM_CLIENT_SERVER_ADDRESS, YarnConfiguration.DEFAULT_SCM_CLIENT_SERVER_PORT);
    clientSCMProxy = (ClientSCMProtocol) rpc.getProxy(ClientSCMProtocol.class, scmAddress, conf);
}

Example 24 with YarnRPC

use of org.apache.hadoop.yarn.ipc.YarnRPC in project hadoop by apache.

the class TestContainerManagerSecurity method testContainerTokenWithEpoch.

/**
   * This tests whether a containerId is serialized/deserialized with epoch.
   *
   * @throws IOException
   * @throws InterruptedException
   * @throws YarnException
   */
private void testContainerTokenWithEpoch(Configuration conf) throws IOException, InterruptedException, YarnException {
    LOG.info("Running test for serializing/deserializing containerIds");
    NMTokenSecretManagerInRM nmTokenSecretManagerInRM = yarnCluster.getResourceManager().getRMContext().getNMTokenSecretManager();
    ApplicationId appId = ApplicationId.newInstance(1, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 0);
    ContainerId cId = ContainerId.newContainerId(appAttemptId, (5L << 40) | 3L);
    NodeManager nm = yarnCluster.getNodeManager(0);
    NMTokenSecretManagerInNM nmTokenSecretManagerInNM = nm.getNMContext().getNMTokenSecretManager();
    String user = "test";
    waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);
    NodeId nodeId = nm.getNMContext().getNodeId();
    // Both id should be equal.
    Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(), nmTokenSecretManagerInRM.getCurrentKey().getKeyId());
    // Creating a normal Container Token
    RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.getResourceManager().getRMContext().getContainerTokenSecretManager();
    Resource r = Resource.newInstance(1230, 2);
    Token containerToken = containerTokenSecretManager.createContainerToken(cId, 0, nodeId, user, r, Priority.newInstance(0), 0);
    ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier();
    byte[] tokenIdentifierContent = containerToken.getIdentifier().array();
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(tokenIdentifierContent, tokenIdentifierContent.length);
    containerTokenIdentifier.readFields(dib);
    Assert.assertEquals(cId, containerTokenIdentifier.getContainerID());
    Assert.assertEquals(cId.toString(), containerTokenIdentifier.getContainerID().toString());
    Token nmToken = nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);
    YarnRPC rpc = YarnRPC.create(conf);
    testStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken, false);
    List<ContainerId> containerIds = new LinkedList<ContainerId>();
    containerIds.add(cId);
    ContainerManagementProtocol proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    GetContainerStatusesResponse res = proxy.getContainerStatuses(GetContainerStatusesRequest.newInstance(containerIds));
    Assert.assertNotNull(res.getContainerStatuses().get(0));
    Assert.assertEquals(cId, res.getContainerStatuses().get(0).getContainerId());
    Assert.assertEquals(cId.toString(), res.getContainerStatuses().get(0).getContainerId().toString());
}

Example 25 with YarnRPC

use of org.apache.hadoop.yarn.ipc.YarnRPC in project hadoop by apache.

the class SCMAdminProtocolService method serviceStart.

@Override
protected void serviceStart() throws Exception {
    Configuration conf = getConfig();
    YarnRPC rpc = YarnRPC.create(conf);
    this.server = rpc.getServer(SCMAdminProtocol.class, this, clientBindAddress, // Secret manager null for now (security not supported)
    conf, // Secret manager null for now (security not supported)
    null, conf.getInt(YarnConfiguration.SCM_ADMIN_CLIENT_THREAD_COUNT, YarnConfiguration.DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT));
    // TODO: Enable service authorization (see YARN-2774)
    this.server.start();
    clientBindAddress = conf.updateConnectAddr(YarnConfiguration.SCM_ADMIN_ADDRESS, server.getListenerAddress());
    super.serviceStart();
}