Examples with YarnRPC org.apache.hadoop.yarn.ipc.YarnRPC used on opensource projects

Example 41 with YarnRPC

use of org.apache.hadoop.yarn.ipc.YarnRPC in project hadoop by apache.

the class TestClientRMService method testGetNodeLabels.

@Test
public void testGetNodeLabels() throws Exception {
    MockRM rm = new MockRM() {

        protected ClientRMService createClientRMService() {
            return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, this.getRMContext().getRMDelegationTokenSecretManager());
        }

        ;
    };
    rm.start();
    NodeLabel labelX = NodeLabel.newInstance("x", false);
    NodeLabel labelY = NodeLabel.newInstance("y");
    RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager();
    labelsMgr.addToCluserNodeLabels(ImmutableSet.of(labelX, labelY));
    NodeId node1 = NodeId.newInstance("host1", 1234);
    NodeId node2 = NodeId.newInstance("host2", 1234);
    Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>();
    map.put(node1, ImmutableSet.of("x"));
    map.put(node2, ImmutableSet.of("y"));
    labelsMgr.replaceLabelsOnNode(map);
    // Create a client.
    Configuration conf = new Configuration();
    YarnRPC rpc = YarnRPC.create(conf);
    InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
    LOG.info("Connecting to ResourceManager at " + rmAddress);
    ApplicationClientProtocol client = (ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class, rmAddress, conf);
    // Get node labels collection
    GetClusterNodeLabelsResponse response = client.getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance());
    Assert.assertTrue(response.getNodeLabelList().containsAll(Arrays.asList(labelX, labelY)));
    // Get node labels mapping
    GetNodesToLabelsResponse response1 = client.getNodeToLabels(GetNodesToLabelsRequest.newInstance());
    Map<NodeId, Set<String>> nodeToLabels = response1.getNodeToLabels();
    Assert.assertTrue(nodeToLabels.keySet().containsAll(Arrays.asList(node1, node2)));
    Assert.assertTrue(nodeToLabels.get(node1).containsAll(Arrays.asList(labelX.getName())));
    Assert.assertTrue(nodeToLabels.get(node2).containsAll(Arrays.asList(labelY.getName())));
    // Below label "x" is not present in the response as exclusivity is true
    Assert.assertFalse(nodeToLabels.get(node1).containsAll(Arrays.asList(NodeLabel.newInstance("x"))));
    rpc.stopProxy(client, conf);
    rm.stop();
}

Example 42 with YarnRPC

use of org.apache.hadoop.yarn.ipc.YarnRPC in project hadoop by apache.

the class TestClientRMService method testGetLabelsToNodes.

@Test
public void testGetLabelsToNodes() throws Exception {
    MockRM rm = new MockRM() {

        protected ClientRMService createClientRMService() {
            return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, this.getRMContext().getRMDelegationTokenSecretManager());
        }

        ;
    };
    rm.start();
    NodeLabel labelX = NodeLabel.newInstance("x", false);
    NodeLabel labelY = NodeLabel.newInstance("y", false);
    NodeLabel labelZ = NodeLabel.newInstance("z", false);
    RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager();
    labelsMgr.addToCluserNodeLabels(ImmutableSet.of(labelX, labelY, labelZ));
    NodeId node1A = NodeId.newInstance("host1", 1234);
    NodeId node1B = NodeId.newInstance("host1", 5678);
    NodeId node2A = NodeId.newInstance("host2", 1234);
    NodeId node3A = NodeId.newInstance("host3", 1234);
    NodeId node3B = NodeId.newInstance("host3", 5678);
    Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>();
    map.put(node1A, ImmutableSet.of("x"));
    map.put(node1B, ImmutableSet.of("z"));
    map.put(node2A, ImmutableSet.of("y"));
    map.put(node3A, ImmutableSet.of("y"));
    map.put(node3B, ImmutableSet.of("z"));
    labelsMgr.replaceLabelsOnNode(map);
    // Create a client.
    Configuration conf = new Configuration();
    YarnRPC rpc = YarnRPC.create(conf);
    InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
    LOG.info("Connecting to ResourceManager at " + rmAddress);
    ApplicationClientProtocol client = (ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class, rmAddress, conf);
    // Get node labels collection
    GetClusterNodeLabelsResponse response = client.getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance());
    Assert.assertTrue(response.getNodeLabelList().containsAll(Arrays.asList(labelX, labelY, labelZ)));
    // Get labels to nodes mapping
    GetLabelsToNodesResponse response1 = client.getLabelsToNodes(GetLabelsToNodesRequest.newInstance());
    Map<String, Set<NodeId>> labelsToNodes = response1.getLabelsToNodes();
    Assert.assertTrue(labelsToNodes.keySet().containsAll(Arrays.asList(labelX.getName(), labelY.getName(), labelZ.getName())));
    Assert.assertTrue(labelsToNodes.get(labelX.getName()).containsAll(Arrays.asList(node1A)));
    Assert.assertTrue(labelsToNodes.get(labelY.getName()).containsAll(Arrays.asList(node2A, node3A)));
    Assert.assertTrue(labelsToNodes.get(labelZ.getName()).containsAll(Arrays.asList(node1B, node3B)));
    // Get labels to nodes mapping for specific labels
    Set<String> setlabels = new HashSet<String>(Arrays.asList(new String[] { "x", "z" }));
    GetLabelsToNodesResponse response2 = client.getLabelsToNodes(GetLabelsToNodesRequest.newInstance(setlabels));
    labelsToNodes = response2.getLabelsToNodes();
    Assert.assertTrue(labelsToNodes.keySet().containsAll(Arrays.asList(labelX.getName(), labelZ.getName())));
    Assert.assertTrue(labelsToNodes.get(labelX.getName()).containsAll(Arrays.asList(node1A)));
    Assert.assertTrue(labelsToNodes.get(labelZ.getName()).containsAll(Arrays.asList(node1B, node3B)));
    Assert.assertEquals(labelsToNodes.get(labelY.getName()), null);
    rpc.stopProxy(client, conf);
    rm.close();
}

Example 43 with YarnRPC

use of org.apache.hadoop.yarn.ipc.YarnRPC in project hadoop by apache.

the class TestClientRMTokens method getClientRMProtocolWithDT.

private ApplicationClientProtocol getClientRMProtocolWithDT(org.apache.hadoop.yarn.api.records.Token token, final InetSocketAddress rmAddress, String user, final Configuration conf) {
    // Maybe consider converting to Hadoop token, serialize de-serialize etc
    // before trying to renew the token.
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
    ugi.addToken(ConverterUtils.convertFromYarn(token, rmAddress));
    final YarnRPC rpc = YarnRPC.create(conf);
    ApplicationClientProtocol clientRMWithDT = ugi.doAs(new PrivilegedAction<ApplicationClientProtocol>() {

        @Override
        public ApplicationClientProtocol run() {
            return (ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class, rmAddress, conf);
        }
    });
    return clientRMWithDT;
}

Example 44 with YarnRPC

use of org.apache.hadoop.yarn.ipc.YarnRPC in project hadoop by apache.

the class TestRPC method test.

private void test(String rpcClass) throws Exception {
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
    YarnRPC rpc = YarnRPC.create(conf);
    String bindAddr = "localhost:0";
    InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
    Server server = rpc.getServer(ContainerManagementProtocol.class, new DummyContainerManager(), addr, conf, null, 1);
    server.start();
    RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class, ProtobufRpcEngine.class);
    ContainerManagementProtocol proxy = (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, NetUtils.getConnectAddress(server), conf);
    ContainerLaunchContext containerLaunchContext = RECORD_FACTORY.newRecordInstance(ContainerLaunchContext.class);
    ApplicationId applicationId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
    ContainerId containerId = ContainerId.newContainerId(applicationAttemptId, 100);
    NodeId nodeId = NodeId.newInstance("localhost", 1234);
    Resource resource = Resource.newInstance(1234, 2);
    ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(containerId, "localhost", "user", resource, System.currentTimeMillis() + 10000, 42, 42, Priority.newInstance(0), 0);
    Token containerToken = newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    proxy.startContainers(allRequests);
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(containerId);
    GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.newInstance(containerIds);
    GetContainerStatusesResponse response = proxy.getContainerStatuses(gcsRequest);
    List<ContainerStatus> statuses = response.getContainerStatuses();
    //test remote exception
    boolean exception = false;
    try {
        StopContainersRequest stopRequest = RECORD_FACTORY.newRecordInstance(StopContainersRequest.class);
        stopRequest.setContainerIds(containerIds);
        proxy.stopContainers(stopRequest);
    } catch (YarnException e) {
        exception = true;
        Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG));
        Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE));
        System.out.println("Test Exception is " + e.getMessage());
    } catch (Exception ex) {
        ex.printStackTrace();
    } finally {
        server.stop();
    }
    Assert.assertTrue(exception);
    Assert.assertNotNull(statuses.get(0));
    Assert.assertEquals(ContainerState.RUNNING, statuses.get(0).getState());
}

Example 45 with YarnRPC

use of org.apache.hadoop.yarn.ipc.YarnRPC in project hadoop by apache.

the class SharedCacheUploadService method createSCMClient.

private SCMUploaderProtocol createSCMClient(Configuration conf) {
    YarnRPC rpc = YarnRPC.create(conf);
    InetSocketAddress scmAddress = conf.getSocketAddr(YarnConfiguration.SCM_UPLOADER_SERVER_ADDRESS, YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_ADDRESS, YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_PORT);
    return (SCMUploaderProtocol) rpc.getProxy(SCMUploaderProtocol.class, scmAddress, conf);
}