Examples with ApplicationClientProtocol org.apache.hadoop.yarn.api.ApplicationClientProtocol used on opensource projects

Example 36 with ApplicationClientProtocol

use of org.apache.hadoop.yarn.api.ApplicationClientProtocol in project hadoop by apache.

the class ReservationACLsTestBase method verifyUpdateReservationSuccess.

private void verifyUpdateReservationSuccess(String updater, String originalSubmitter, String queueName) throws Exception {
    ReservationId reservationId = createReservation(originalSubmitter);
    submitReservation(originalSubmitter, queueName, reservationId);
    final ReservationUpdateRequest updateRequest = ReservationUpdateRequest.newInstance(makeSimpleReservationDefinition(), reservationId);
    ApplicationClientProtocol ownerClient = getRMClientForUser(updater);
    ownerClient.updateReservation(updateRequest);
    deleteReservation(updater, reservationId);
}

Example 37 with ApplicationClientProtocol

use of org.apache.hadoop.yarn.api.ApplicationClientProtocol in project hadoop by apache.

the class TestClientRMService method testGetClusterNodes.

@Test
public void testGetClusterNodes() throws Exception {
    MockRM rm = new MockRM() {

        protected ClientRMService createClientRMService() {
            return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, this.getRMContext().getRMDelegationTokenSecretManager());
        }

        ;
    };
    rm.start();
    RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager();
    labelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y"));
    // Add a healthy node with label = x
    MockNM node = rm.registerNode("host1:1234", 1024);
    Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>();
    map.put(node.getNodeId(), ImmutableSet.of("x"));
    labelsMgr.replaceLabelsOnNode(map);
    rm.sendNodeStarted(node);
    node.nodeHeartbeat(true);
    // Add and lose a node with label = y
    MockNM lostNode = rm.registerNode("host2:1235", 1024);
    rm.sendNodeStarted(lostNode);
    lostNode.nodeHeartbeat(true);
    rm.waitForState(lostNode.getNodeId(), NodeState.RUNNING);
    rm.sendNodeLost(lostNode);
    // Create a client.
    Configuration conf = new Configuration();
    YarnRPC rpc = YarnRPC.create(conf);
    InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
    LOG.info("Connecting to ResourceManager at " + rmAddress);
    ApplicationClientProtocol client = (ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class, rmAddress, conf);
    // Make call
    GetClusterNodesRequest request = GetClusterNodesRequest.newInstance(EnumSet.of(NodeState.RUNNING));
    List<NodeReport> nodeReports = client.getClusterNodes(request).getNodeReports();
    Assert.assertEquals(1, nodeReports.size());
    Assert.assertNotSame("Node is expected to be healthy!", NodeState.UNHEALTHY, nodeReports.get(0).getNodeState());
    // Check node's label = x
    Assert.assertTrue(nodeReports.get(0).getNodeLabels().contains("x"));
    // Now make the node unhealthy.
    node.nodeHeartbeat(false);
    rm.waitForState(node.getNodeId(), NodeState.UNHEALTHY);
    // Call again
    nodeReports = client.getClusterNodes(request).getNodeReports();
    Assert.assertEquals("Unhealthy nodes should not show up by default", 0, nodeReports.size());
    // Change label of host1 to y
    map = new HashMap<NodeId, Set<String>>();
    map.put(node.getNodeId(), ImmutableSet.of("y"));
    labelsMgr.replaceLabelsOnNode(map);
    // Now query for UNHEALTHY nodes
    request = GetClusterNodesRequest.newInstance(EnumSet.of(NodeState.UNHEALTHY));
    nodeReports = client.getClusterNodes(request).getNodeReports();
    Assert.assertEquals(1, nodeReports.size());
    Assert.assertEquals("Node is expected to be unhealthy!", NodeState.UNHEALTHY, nodeReports.get(0).getNodeState());
    Assert.assertTrue(nodeReports.get(0).getNodeLabels().contains("y"));
    // Remove labels of host1
    map = new HashMap<NodeId, Set<String>>();
    map.put(node.getNodeId(), ImmutableSet.of("y"));
    labelsMgr.removeLabelsFromNode(map);
    // Query all states should return all nodes
    rm.registerNode("host3:1236", 1024);
    request = GetClusterNodesRequest.newInstance(EnumSet.allOf(NodeState.class));
    nodeReports = client.getClusterNodes(request).getNodeReports();
    Assert.assertEquals(3, nodeReports.size());
    // All host1-3's label should be empty (instead of null)
    for (NodeReport report : nodeReports) {
        Assert.assertTrue(report.getNodeLabels() != null && report.getNodeLabels().isEmpty());
    }
    rpc.stopProxy(client, conf);
    rm.close();
}

Example 38 with ApplicationClientProtocol

use of org.apache.hadoop.yarn.api.ApplicationClientProtocol in project hadoop by apache.

the class TestClientRMService method testGetNodeLabels.

@Test
public void testGetNodeLabels() throws Exception {
    MockRM rm = new MockRM() {

        protected ClientRMService createClientRMService() {
            return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, this.getRMContext().getRMDelegationTokenSecretManager());
        }

        ;
    };
    rm.start();
    NodeLabel labelX = NodeLabel.newInstance("x", false);
    NodeLabel labelY = NodeLabel.newInstance("y");
    RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager();
    labelsMgr.addToCluserNodeLabels(ImmutableSet.of(labelX, labelY));
    NodeId node1 = NodeId.newInstance("host1", 1234);
    NodeId node2 = NodeId.newInstance("host2", 1234);
    Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>();
    map.put(node1, ImmutableSet.of("x"));
    map.put(node2, ImmutableSet.of("y"));
    labelsMgr.replaceLabelsOnNode(map);
    // Create a client.
    Configuration conf = new Configuration();
    YarnRPC rpc = YarnRPC.create(conf);
    InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
    LOG.info("Connecting to ResourceManager at " + rmAddress);
    ApplicationClientProtocol client = (ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class, rmAddress, conf);
    // Get node labels collection
    GetClusterNodeLabelsResponse response = client.getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance());
    Assert.assertTrue(response.getNodeLabelList().containsAll(Arrays.asList(labelX, labelY)));
    // Get node labels mapping
    GetNodesToLabelsResponse response1 = client.getNodeToLabels(GetNodesToLabelsRequest.newInstance());
    Map<NodeId, Set<String>> nodeToLabels = response1.getNodeToLabels();
    Assert.assertTrue(nodeToLabels.keySet().containsAll(Arrays.asList(node1, node2)));
    Assert.assertTrue(nodeToLabels.get(node1).containsAll(Arrays.asList(labelX.getName())));
    Assert.assertTrue(nodeToLabels.get(node2).containsAll(Arrays.asList(labelY.getName())));
    // Below label "x" is not present in the response as exclusivity is true
    Assert.assertFalse(nodeToLabels.get(node1).containsAll(Arrays.asList(NodeLabel.newInstance("x"))));
    rpc.stopProxy(client, conf);
    rm.stop();
}

Example 39 with ApplicationClientProtocol

use of org.apache.hadoop.yarn.api.ApplicationClientProtocol in project hadoop by apache.

the class TestClientRMService method testGetLabelsToNodes.

@Test
public void testGetLabelsToNodes() throws Exception {
    MockRM rm = new MockRM() {

        protected ClientRMService createClientRMService() {
            return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, this.getRMContext().getRMDelegationTokenSecretManager());
        }

        ;
    };
    rm.start();
    NodeLabel labelX = NodeLabel.newInstance("x", false);
    NodeLabel labelY = NodeLabel.newInstance("y", false);
    NodeLabel labelZ = NodeLabel.newInstance("z", false);
    RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager();
    labelsMgr.addToCluserNodeLabels(ImmutableSet.of(labelX, labelY, labelZ));
    NodeId node1A = NodeId.newInstance("host1", 1234);
    NodeId node1B = NodeId.newInstance("host1", 5678);
    NodeId node2A = NodeId.newInstance("host2", 1234);
    NodeId node3A = NodeId.newInstance("host3", 1234);
    NodeId node3B = NodeId.newInstance("host3", 5678);
    Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>();
    map.put(node1A, ImmutableSet.of("x"));
    map.put(node1B, ImmutableSet.of("z"));
    map.put(node2A, ImmutableSet.of("y"));
    map.put(node3A, ImmutableSet.of("y"));
    map.put(node3B, ImmutableSet.of("z"));
    labelsMgr.replaceLabelsOnNode(map);
    // Create a client.
    Configuration conf = new Configuration();
    YarnRPC rpc = YarnRPC.create(conf);
    InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
    LOG.info("Connecting to ResourceManager at " + rmAddress);
    ApplicationClientProtocol client = (ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class, rmAddress, conf);
    // Get node labels collection
    GetClusterNodeLabelsResponse response = client.getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance());
    Assert.assertTrue(response.getNodeLabelList().containsAll(Arrays.asList(labelX, labelY, labelZ)));
    // Get labels to nodes mapping
    GetLabelsToNodesResponse response1 = client.getLabelsToNodes(GetLabelsToNodesRequest.newInstance());
    Map<String, Set<NodeId>> labelsToNodes = response1.getLabelsToNodes();
    Assert.assertTrue(labelsToNodes.keySet().containsAll(Arrays.asList(labelX.getName(), labelY.getName(), labelZ.getName())));
    Assert.assertTrue(labelsToNodes.get(labelX.getName()).containsAll(Arrays.asList(node1A)));
    Assert.assertTrue(labelsToNodes.get(labelY.getName()).containsAll(Arrays.asList(node2A, node3A)));
    Assert.assertTrue(labelsToNodes.get(labelZ.getName()).containsAll(Arrays.asList(node1B, node3B)));
    // Get labels to nodes mapping for specific labels
    Set<String> setlabels = new HashSet<String>(Arrays.asList(new String[] { "x", "z" }));
    GetLabelsToNodesResponse response2 = client.getLabelsToNodes(GetLabelsToNodesRequest.newInstance(setlabels));
    labelsToNodes = response2.getLabelsToNodes();
    Assert.assertTrue(labelsToNodes.keySet().containsAll(Arrays.asList(labelX.getName(), labelZ.getName())));
    Assert.assertTrue(labelsToNodes.get(labelX.getName()).containsAll(Arrays.asList(node1A)));
    Assert.assertTrue(labelsToNodes.get(labelZ.getName()).containsAll(Arrays.asList(node1B, node3B)));
    Assert.assertEquals(labelsToNodes.get(labelY.getName()), null);
    rpc.stopProxy(client, conf);
    rm.close();
}

Example 40 with ApplicationClientProtocol

use of org.apache.hadoop.yarn.api.ApplicationClientProtocol in project hadoop by apache.

the class TestClientRMTokens method testDelegationToken.

@Test
public void testDelegationToken() throws IOException, InterruptedException {
    final YarnConfiguration conf = new YarnConfiguration();
    conf.set(YarnConfiguration.RM_PRINCIPAL, "testuser/localhost@apache.org");
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    ResourceScheduler scheduler = createMockScheduler(conf);
    long initialInterval = 10000l;
    long maxLifetime = 20000l;
    long renewInterval = 10000l;
    RMDelegationTokenSecretManager rmDtSecretManager = createRMDelegationTokenSecretManager(initialInterval, maxLifetime, renewInterval);
    rmDtSecretManager.startThreads();
    LOG.info("Creating DelegationTokenSecretManager with initialInterval: " + initialInterval + ", maxLifetime: " + maxLifetime + ", renewInterval: " + renewInterval);
    final ClientRMService clientRMService = new ClientRMServiceForTest(conf, scheduler, rmDtSecretManager);
    clientRMService.init(conf);
    clientRMService.start();
    ApplicationClientProtocol clientRMWithDT = null;
    try {
        // Create a user for the renewr and fake the authentication-method
        UserGroupInformation loggedInUser = UserGroupInformation.createRemoteUser("testrenewer@APACHE.ORG");
        Assert.assertEquals("testrenewer", loggedInUser.getShortUserName());
        // Default realm is APACHE.ORG
        loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
        org.apache.hadoop.yarn.api.records.Token token = getDelegationToken(loggedInUser, clientRMService, loggedInUser.getShortUserName());
        long tokenFetchTime = System.currentTimeMillis();
        LOG.info("Got delegation token at: " + tokenFetchTime);
        // Now try talking to RMService using the delegation token
        clientRMWithDT = getClientRMProtocolWithDT(token, clientRMService.getBindAddress(), "loginuser1", conf);
        GetNewApplicationRequest request = Records.newRecord(GetNewApplicationRequest.class);
        try {
            clientRMWithDT.getNewApplication(request);
        } catch (IOException e) {
            fail("Unexpected exception" + e);
        } catch (YarnException e) {
            fail("Unexpected exception" + e);
        }
        // Renew after 50% of token age.
        while (System.currentTimeMillis() < tokenFetchTime + initialInterval / 2) {
            Thread.sleep(500l);
        }
        long nextExpTime = renewDelegationToken(loggedInUser, clientRMService, token);
        long renewalTime = System.currentTimeMillis();
        LOG.info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + nextExpTime);
        // Wait for first expiry, but before renewed expiry.
        while (System.currentTimeMillis() > tokenFetchTime + initialInterval && System.currentTimeMillis() < nextExpTime) {
            Thread.sleep(500l);
        }
        Thread.sleep(50l);
        // Valid token because of renewal.
        try {
            clientRMWithDT.getNewApplication(request);
        } catch (IOException e) {
            fail("Unexpected exception" + e);
        } catch (YarnException e) {
            fail("Unexpected exception" + e);
        }
        // Wait for expiry.
        while (System.currentTimeMillis() < renewalTime + renewInterval) {
            Thread.sleep(500l);
        }
        Thread.sleep(50l);
        LOG.info("At time: " + System.currentTimeMillis() + ", token should be invalid");
        // Token should have expired.      
        try {
            clientRMWithDT.getNewApplication(request);
            fail("Should not have succeeded with an expired token");
        } catch (Exception e) {
            assertEquals(InvalidToken.class.getName(), e.getClass().getName());
            assertTrue(e.getMessage().contains("is expired"));
        }
        // Stop the existing proxy, start another.
        if (clientRMWithDT != null) {
            RPC.stopProxy(clientRMWithDT);
            clientRMWithDT = null;
        }
        token = getDelegationToken(loggedInUser, clientRMService, loggedInUser.getShortUserName());
        tokenFetchTime = System.currentTimeMillis();
        LOG.info("Got delegation token at: " + tokenFetchTime);
        // Now try talking to RMService using the delegation token
        clientRMWithDT = getClientRMProtocolWithDT(token, clientRMService.getBindAddress(), "loginuser2", conf);
        request = Records.newRecord(GetNewApplicationRequest.class);
        try {
            clientRMWithDT.getNewApplication(request);
        } catch (IOException e) {
            fail("Unexpected exception" + e);
        } catch (YarnException e) {
            fail("Unexpected exception" + e);
        }
        cancelDelegationToken(loggedInUser, clientRMService, token);
        if (clientRMWithDT != null) {
            RPC.stopProxy(clientRMWithDT);
            clientRMWithDT = null;
        }
        // Creating a new connection.
        clientRMWithDT = getClientRMProtocolWithDT(token, clientRMService.getBindAddress(), "loginuser2", conf);
        LOG.info("Cancelled delegation token at: " + System.currentTimeMillis());
        // Verify cancellation worked.
        try {
            clientRMWithDT.getNewApplication(request);
            fail("Should not have succeeded with a cancelled delegation token");
        } catch (IOException e) {
        } catch (YarnException e) {
        }
        // Stop the existing proxy, start another.
        if (clientRMWithDT != null) {
            RPC.stopProxy(clientRMWithDT);
            clientRMWithDT = null;
        }
        token = getDelegationToken(loggedInUser, clientRMService, loggedInUser.getShortUserName());
        byte[] tokenIdentifierContent = token.getIdentifier().array();
        RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier();
        DataInputBuffer dib = new DataInputBuffer();
        dib.reset(tokenIdentifierContent, tokenIdentifierContent.length);
        tokenIdentifier.readFields(dib);
        // Construct new version RMDelegationTokenIdentifier with additional field
        RMDelegationTokenIdentifierForTest newVersionTokenIdentifier = new RMDelegationTokenIdentifierForTest(tokenIdentifier, "message");
        Token<RMDelegationTokenIdentifier> newRMDTtoken = new Token<RMDelegationTokenIdentifier>(newVersionTokenIdentifier, rmDtSecretManager);
        org.apache.hadoop.yarn.api.records.Token newToken = BuilderUtils.newDelegationToken(newRMDTtoken.getIdentifier(), newRMDTtoken.getKind().toString(), newRMDTtoken.getPassword(), newRMDTtoken.getService().toString());
        // Now try talking to RMService using the new version delegation token
        clientRMWithDT = getClientRMProtocolWithDT(newToken, clientRMService.getBindAddress(), "loginuser3", conf);
        request = Records.newRecord(GetNewApplicationRequest.class);
        try {
            clientRMWithDT.getNewApplication(request);
        } catch (IOException e) {
            fail("Unexpected exception" + e);
        } catch (YarnException e) {
            fail("Unexpected exception" + e);
        }
    } finally {
        rmDtSecretManager.stopThreads();
        // TODO PRECOMMIT Close proxies.
        if (clientRMWithDT != null) {
            RPC.stopProxy(clientRMWithDT);
        }
    }
}