Examples with NodeHeartbeatResponse org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse used on opensource projects

Example 16 with NodeHeartbeatResponse

use of org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse in project hadoop by apache.

the class TestRMNodeTransitions method getMockRMNodeStatusEventWithoutRunningApps.

private RMNodeStatusEvent getMockRMNodeStatusEventWithoutRunningApps() {
    NodeHeartbeatResponse response = mock(NodeHeartbeatResponse.class);
    NodeHealthStatus healthStatus = mock(NodeHealthStatus.class);
    Boolean yes = new Boolean(true);
    doReturn(yes).when(healthStatus).getIsNodeHealthy();
    RMNodeStatusEvent event = mock(RMNodeStatusEvent.class);
    doReturn(healthStatus).when(event).getNodeHealthStatus();
    doReturn(response).when(event).getLatestResponse();
    doReturn(RMNodeEventType.STATUS_UPDATE).when(event).getType();
    doReturn(null).when(event).getKeepAliveAppIds();
    return event;
}

Example 17 with NodeHeartbeatResponse

use of org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse in project hadoop by apache.

the class TestRMNodeTransitions method testUpdateHeartbeatResponseForCleanup.

@Test(timeout = 20000)
public void testUpdateHeartbeatResponseForCleanup() {
    RMNodeImpl node = getRunningNode();
    NodeId nodeId = node.getNodeID();
    // Expire a container
    ContainerId completedContainerId = BuilderUtils.newContainerId(BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
    node.handle(new RMNodeCleanContainerEvent(nodeId, completedContainerId));
    Assert.assertEquals(1, node.getContainersToCleanUp().size());
    // Finish an application
    ApplicationId finishedAppId = BuilderUtils.newApplicationId(0, 1);
    node.handle(new RMNodeCleanAppEvent(nodeId, finishedAppId));
    Assert.assertEquals(1, node.getAppsToCleanup().size());
    // Verify status update does not clear containers/apps to cleanup
    // but updating heartbeat response for cleanup does
    RMNodeStatusEvent statusEvent = getMockRMNodeStatusEvent(null);
    node.handle(statusEvent);
    Assert.assertEquals(1, node.getContainersToCleanUp().size());
    Assert.assertEquals(1, node.getAppsToCleanup().size());
    NodeHeartbeatResponse hbrsp = Records.newRecord(NodeHeartbeatResponse.class);
    node.updateNodeHeartbeatResponseForCleanup(hbrsp);
    Assert.assertEquals(0, node.getContainersToCleanUp().size());
    Assert.assertEquals(0, node.getAppsToCleanup().size());
    Assert.assertEquals(1, hbrsp.getContainersToCleanup().size());
    Assert.assertEquals(completedContainerId, hbrsp.getContainersToCleanup().get(0));
    Assert.assertEquals(1, hbrsp.getApplicationsToCleanup().size());
    Assert.assertEquals(finishedAppId, hbrsp.getApplicationsToCleanup().get(0));
}

Example 18 with NodeHeartbeatResponse

use of org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse in project hadoop by apache.

the class TestRMNMRPCResponseId method testRPCResponseId.

@Test
public void testRPCResponseId() throws IOException, YarnException {
    String node = "localhost";
    Resource capability = BuilderUtils.newResource(1024, 1);
    RegisterNodeManagerRequest request = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
    nodeId = NodeId.newInstance(node, 1234);
    request.setNodeId(nodeId);
    request.setHttpPort(0);
    request.setResource(capability);
    RegisterNodeManagerRequest request1 = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
    request1.setNodeId(nodeId);
    request1.setHttpPort(0);
    request1.setResource(capability);
    resourceTrackerService.registerNodeManager(request1);
    org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = recordFactory.newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class);
    nodeStatus.setNodeId(nodeId);
    NodeHealthStatus nodeHealthStatus = recordFactory.newRecordInstance(NodeHealthStatus.class);
    nodeHealthStatus.setIsNodeHealthy(true);
    nodeStatus.setNodeHealthStatus(nodeHealthStatus);
    NodeHeartbeatRequest nodeHeartBeatRequest = recordFactory.newRecordInstance(NodeHeartbeatRequest.class);
    nodeHeartBeatRequest.setNodeStatus(nodeStatus);
    nodeStatus.setResponseId(0);
    NodeHeartbeatResponse response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest);
    Assert.assertTrue(response.getResponseId() == 1);
    nodeStatus.setResponseId(response.getResponseId());
    response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest);
    Assert.assertTrue(response.getResponseId() == 2);
    /* try calling with less response id */
    response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest);
    Assert.assertTrue(response.getResponseId() == 2);
    nodeStatus.setResponseId(0);
    response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest);
    Assert.assertTrue(NodeAction.RESYNC.equals(response.getNodeAction()));
    Assert.assertEquals("Too far behind rm response id:2 nm response id:0", response.getDiagnosticsMessage());
}

Example 19 with NodeHeartbeatResponse

use of org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse in project hadoop by apache.

the class TestDelegationTokenRenewer method testRMRestartWithExpiredToken.

// 1. token is expired before app completes.
// 2. RM shutdown.
// 3. When RM recovers the app, token renewal will fail as token expired.
//    RM should request a new token and sent it to NM for log-aggregation.
@Test
public void testRMRestartWithExpiredToken() throws Exception {
    Configuration yarnConf = new YarnConfiguration();
    yarnConf.setBoolean(YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED, true);
    yarnConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    yarnConf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
    yarnConf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
    UserGroupInformation.setConfiguration(yarnConf);
    // create Token1:
    Text userText1 = new Text("user1");
    DelegationTokenIdentifier dtId1 = new DelegationTokenIdentifier(userText1, new Text("renewer1"), userText1);
    final Token<DelegationTokenIdentifier> originalToken = new Token<>(dtId1.getBytes(), "password1".getBytes(), dtId1.getKind(), new Text("service1"));
    Credentials credentials = new Credentials();
    credentials.addToken(userText1, originalToken);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(yarnConf);
    MockRM rm1 = new TestSecurityMockRM(yarnConf, memStore);
    rm1.start();
    RMApp app = rm1.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), false, "default", 1, credentials);
    // create token2
    Text userText2 = new Text("user1");
    DelegationTokenIdentifier dtId2 = new DelegationTokenIdentifier(userText1, new Text("renewer2"), userText2);
    final Token<DelegationTokenIdentifier> updatedToken = new Token<DelegationTokenIdentifier>(dtId2.getBytes(), "password2".getBytes(), dtId2.getKind(), new Text("service2"));
    AtomicBoolean firstRenewInvoked = new AtomicBoolean(false);
    AtomicBoolean secondRenewInvoked = new AtomicBoolean(false);
    MockRM rm2 = new TestSecurityMockRM(yarnConf, memStore) {

        @Override
        protected DelegationTokenRenewer createDelegationTokenRenewer() {
            return new DelegationTokenRenewer() {

                @Override
                protected void renewToken(final DelegationTokenToRenew dttr) throws IOException {
                    if (dttr.token.equals(updatedToken)) {
                        secondRenewInvoked.set(true);
                        super.renewToken(dttr);
                    } else if (dttr.token.equals(originalToken)) {
                        firstRenewInvoked.set(true);
                        throw new InvalidToken("Failed to renew");
                    } else {
                        throw new IOException("Unexpected");
                    }
                }

                @Override
                protected Token<?>[] obtainSystemTokensForUser(String user, final Credentials credentials) throws IOException {
                    credentials.addToken(updatedToken.getService(), updatedToken);
                    return new Token<?>[] { updatedToken };
                }
            };
        }
    };
    // simulating restart the rm
    rm2.start();
    // check nm can retrieve the token
    final MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm2.getResourceTrackerService());
    nm1.registerNode();
    NodeHeartbeatResponse response = nm1.nodeHeartbeat(true);
    ByteBuffer tokenBuffer = response.getSystemCredentialsForApps().get(app.getApplicationId());
    Assert.assertNotNull(tokenBuffer);
    Credentials appCredentials = new Credentials();
    DataInputByteBuffer buf = new DataInputByteBuffer();
    tokenBuffer.rewind();
    buf.reset(tokenBuffer);
    appCredentials.readTokenStorageStream(buf);
    Assert.assertTrue(firstRenewInvoked.get() && secondRenewInvoked.get());
    Assert.assertTrue(appCredentials.getAllTokens().contains(updatedToken));
}

Example 20 with NodeHeartbeatResponse

use of org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse in project hadoop by apache.

the class TestDelegationTokenRenewer method testAppSubmissionWithoutDelegationToken.

// YARN will get the token for the app submitted without the delegation token.
@Test
public void testAppSubmissionWithoutDelegationToken() throws Exception {
    conf.setBoolean(YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED, true);
    // create token2
    Text userText2 = new Text("user2");
    DelegationTokenIdentifier dtId2 = new DelegationTokenIdentifier(new Text("user2"), new Text("renewer2"), userText2);
    final Token<DelegationTokenIdentifier> token2 = new Token<DelegationTokenIdentifier>(dtId2.getBytes(), "password2".getBytes(), dtId2.getKind(), new Text("service2"));
    final MockRM rm = new TestSecurityMockRM(conf, null) {

        @Override
        protected DelegationTokenRenewer createDelegationTokenRenewer() {
            return new DelegationTokenRenewer() {

                @Override
                protected Token<?>[] obtainSystemTokensForUser(String user, final Credentials credentials) throws IOException {
                    credentials.addToken(token2.getService(), token2);
                    return new Token<?>[] { token2 };
                }
            };
        }
    };
    rm.start();
    // submit an app without delegationToken
    RMApp app = rm.submitApp(200);
    // wait for the new retrieved hdfs token.
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        public Boolean get() {
            return rm.getRMContext().getDelegationTokenRenewer().getDelegationTokens().contains(token2);
        }
    }, 1000, 20000);
    // check nm can retrieve the token
    final MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
    nm1.registerNode();
    NodeHeartbeatResponse response = nm1.nodeHeartbeat(true);
    ByteBuffer tokenBuffer = response.getSystemCredentialsForApps().get(app.getApplicationId());
    Assert.assertNotNull(tokenBuffer);
    Credentials appCredentials = new Credentials();
    DataInputByteBuffer buf = new DataInputByteBuffer();
    tokenBuffer.rewind();
    buf.reset(tokenBuffer);
    appCredentials.readTokenStorageStream(buf);
    Assert.assertTrue(appCredentials.getAllTokens().contains(token2));
}