Examples with NodeHeartbeatRequest org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest used on opensource projects

Example 1 with NodeHeartbeatRequest

use of org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest in project hadoop by apache.

the class NMSimulator method middleStep.

@Override
public void middleStep() throws Exception {
    // we check the lifetime for each running containers
    ContainerSimulator cs = null;
    synchronized (completedContainerList) {
        while ((cs = containerQueue.poll()) != null) {
            runningContainers.remove(cs.getId());
            completedContainerList.add(cs.getId());
            LOG.debug(MessageFormat.format("Container {0} has completed", cs.getId()));
        }
    }
    // send heart beat
    NodeHeartbeatRequest beatRequest = Records.newRecord(NodeHeartbeatRequest.class);
    beatRequest.setLastKnownNMTokenMasterKey(masterKey);
    NodeStatus ns = Records.newRecord(NodeStatus.class);
    ns.setContainersStatuses(generateContainerStatusList());
    ns.setNodeId(node.getNodeID());
    ns.setKeepAliveApplications(new ArrayList<ApplicationId>());
    ns.setResponseId(RESPONSE_ID++);
    ns.setNodeHealthStatus(NodeHealthStatus.newInstance(true, "", 0));
    beatRequest.setNodeStatus(ns);
    NodeHeartbeatResponse beatResponse = rm.getResourceTrackerService().nodeHeartbeat(beatRequest);
    if (!beatResponse.getContainersToCleanup().isEmpty()) {
        // remove from queue
        synchronized (releasedContainerList) {
            for (ContainerId containerId : beatResponse.getContainersToCleanup()) {
                if (amContainerList.contains(containerId)) {
                    // AM container (not killed?, only release)
                    synchronized (amContainerList) {
                        amContainerList.remove(containerId);
                    }
                    LOG.debug(MessageFormat.format("NodeManager {0} releases " + "an AM ({1}).", node.getNodeID(), containerId));
                } else {
                    cs = runningContainers.remove(containerId);
                    containerQueue.remove(cs);
                    releasedContainerList.add(containerId);
                    LOG.debug(MessageFormat.format("NodeManager {0} releases a " + "container ({1}).", node.getNodeID(), containerId));
                }
            }
        }
    }
    if (beatResponse.getNodeAction() == NodeAction.SHUTDOWN) {
        lastStep();
    }
}

Example 2 with NodeHeartbeatRequest

use of org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest in project hadoop by apache.

the class TestNodeManagerResync method testNMSentContainerStatusOnResync.

// This is to test when NM gets the resync response from last heart beat, it
// should be able to send the already-sent-via-last-heart-beat container
// statuses again when it re-register with RM.
@Test
public void testNMSentContainerStatusOnResync() throws Exception {
    final ContainerStatus testCompleteContainer = TestNodeStatusUpdater.createContainerStatus(2, ContainerState.COMPLETE);
    final Container container = TestNodeStatusUpdater.getMockContainer(testCompleteContainer);
    NMContainerStatus report = createNMContainerStatus(2, ContainerState.COMPLETE);
    when(container.getNMContainerStatus()).thenReturn(report);
    NodeManager nm = new NodeManager() {

        int registerCount = 0;

        @Override
        protected NodeStatusUpdater createNodeStatusUpdater(Context context, Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
            return new TestNodeStatusUpdaterResync(context, dispatcher, healthChecker, metrics) {

                @Override
                protected ResourceTracker createResourceTracker() {
                    return new MockResourceTracker() {

                        @Override
                        public RegisterNodeManagerResponse registerNodeManager(RegisterNodeManagerRequest request) throws YarnException, IOException {
                            if (registerCount == 0) {
                                // first register, no containers info.
                                try {
                                    Assert.assertEquals(0, request.getNMContainerStatuses().size());
                                } catch (AssertionError error) {
                                    error.printStackTrace();
                                    assertionFailedInThread.set(true);
                                }
                                // put the completed container into the context
                                getNMContext().getContainers().put(testCompleteContainer.getContainerId(), container);
                                getNMContext().getApplications().put(testCompleteContainer.getContainerId().getApplicationAttemptId().getApplicationId(), mock(Application.class));
                            } else {
                                // second register contains the completed container info.
                                List<NMContainerStatus> statuses = request.getNMContainerStatuses();
                                try {
                                    Assert.assertEquals(1, statuses.size());
                                    Assert.assertEquals(testCompleteContainer.getContainerId(), statuses.get(0).getContainerId());
                                } catch (AssertionError error) {
                                    error.printStackTrace();
                                    assertionFailedInThread.set(true);
                                }
                            }
                            registerCount++;
                            return super.registerNodeManager(request);
                        }

                        @Override
                        public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) {
                            // first heartBeat contains the completed container info
                            List<ContainerStatus> statuses = request.getNodeStatus().getContainersStatuses();
                            try {
                                Assert.assertEquals(1, statuses.size());
                                Assert.assertEquals(testCompleteContainer.getContainerId(), statuses.get(0).getContainerId());
                            } catch (AssertionError error) {
                                error.printStackTrace();
                                assertionFailedInThread.set(true);
                            }
                            // notify RESYNC on first heartbeat.
                            return YarnServerBuilderUtils.newNodeHeartbeatResponse(1, NodeAction.RESYNC, null, null, null, null, 1000L);
                        }
                    };
                }
            };
        }
    };
    YarnConfiguration conf = createNMConfig();
    nm.init(conf);
    nm.start();
    try {
        syncBarrier.await();
    } catch (BrokenBarrierException e) {
    }
    Assert.assertFalse(assertionFailedInThread.get());
    nm.stop();
}

Example 3 with NodeHeartbeatRequest

use of org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest in project hadoop by apache.

the class TestRMNMRPCResponseId method testRPCResponseId.

@Test
public void testRPCResponseId() throws IOException, YarnException {
    String node = "localhost";
    Resource capability = BuilderUtils.newResource(1024, 1);
    RegisterNodeManagerRequest request = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
    nodeId = NodeId.newInstance(node, 1234);
    request.setNodeId(nodeId);
    request.setHttpPort(0);
    request.setResource(capability);
    RegisterNodeManagerRequest request1 = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
    request1.setNodeId(nodeId);
    request1.setHttpPort(0);
    request1.setResource(capability);
    resourceTrackerService.registerNodeManager(request1);
    org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = recordFactory.newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class);
    nodeStatus.setNodeId(nodeId);
    NodeHealthStatus nodeHealthStatus = recordFactory.newRecordInstance(NodeHealthStatus.class);
    nodeHealthStatus.setIsNodeHealthy(true);
    nodeStatus.setNodeHealthStatus(nodeHealthStatus);
    NodeHeartbeatRequest nodeHeartBeatRequest = recordFactory.newRecordInstance(NodeHeartbeatRequest.class);
    nodeHeartBeatRequest.setNodeStatus(nodeStatus);
    nodeStatus.setResponseId(0);
    NodeHeartbeatResponse response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest);
    Assert.assertTrue(response.getResponseId() == 1);
    nodeStatus.setResponseId(response.getResponseId());
    response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest);
    Assert.assertTrue(response.getResponseId() == 2);
    /* try calling with less response id */
    response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest);
    Assert.assertTrue(response.getResponseId() == 2);
    nodeStatus.setResponseId(0);
    response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest);
    Assert.assertTrue(NodeAction.RESYNC.equals(response.getNodeAction()));
    Assert.assertEquals("Too far behind rm response id:2 nm response id:0", response.getDiagnosticsMessage());
}

Example 4 with NodeHeartbeatRequest

use of org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest in project hadoop by apache.

the class TestResourceTrackerService method testNodeHeartBeatWithLabels.

@Test
public void testNodeHeartBeatWithLabels() throws Exception {
    writeToHostsFile("host2");
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile.getAbsolutePath());
    conf.set(YarnConfiguration.NODELABEL_CONFIGURATION_TYPE, YarnConfiguration.DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE);
    final RMNodeLabelsManager nodeLabelsMgr = new NullRMNodeLabelsManager();
    rm = new MockRM(conf) {

        @Override
        protected RMNodeLabelsManager createNodeLabelManager() {
            return nodeLabelsMgr;
        }
    };
    rm.start();
    // adding valid labels
    try {
        nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("A", "B", "C"));
    } catch (IOException e) {
        Assert.fail("Caught Exception while intializing");
        e.printStackTrace();
    }
    // Registering of labels and other required info to RM
    ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
    RegisterNodeManagerRequest registerReq = Records.newRecord(RegisterNodeManagerRequest.class);
    NodeId nodeId = NodeId.newInstance("host2", 1234);
    Resource capability = BuilderUtils.newResource(1024, 1);
    registerReq.setResource(capability);
    registerReq.setNodeId(nodeId);
    registerReq.setHttpPort(1234);
    registerReq.setNMVersion(YarnVersionInfo.getVersion());
    // Node register label
    registerReq.setNodeLabels(toNodeLabelSet("A"));
    RegisterNodeManagerResponse registerResponse = resourceTrackerService.registerNodeManager(registerReq);
    // modification of labels during heartbeat
    NodeHeartbeatRequest heartbeatReq = Records.newRecord(NodeHeartbeatRequest.class);
    // Node heartbeat label update
    heartbeatReq.setNodeLabels(toNodeLabelSet("B"));
    NodeStatus nodeStatusObject = getNodeStatusObject(nodeId);
    heartbeatReq.setNodeStatus(nodeStatusObject);
    heartbeatReq.setLastKnownNMTokenMasterKey(registerResponse.getNMTokenMasterKey());
    heartbeatReq.setLastKnownContainerTokenMasterKey(registerResponse.getContainerTokenMasterKey());
    NodeHeartbeatResponse nodeHeartbeatResponse = resourceTrackerService.nodeHeartbeat(heartbeatReq);
    Assert.assertEquals("InValid Node Labels were not accepted by RM", NodeAction.NORMAL, nodeHeartbeatResponse.getNodeAction());
    assertCollectionEquals(nodeLabelsMgr.getNodeLabels().get(nodeId), NodeLabelsUtils.convertToStringSet(heartbeatReq.getNodeLabels()));
    Assert.assertTrue("Valid Node Labels were not accepted by RM", nodeHeartbeatResponse.getAreNodeLabelsAcceptedByRM());
    // After modification of labels next heartbeat sends null informing no update
    Set<String> oldLabels = nodeLabelsMgr.getNodeLabels().get(nodeId);
    int responseId = nodeStatusObject.getResponseId();
    heartbeatReq = Records.newRecord(NodeHeartbeatRequest.class);
    // Node heartbeat label update
    heartbeatReq.setNodeLabels(null);
    nodeStatusObject = getNodeStatusObject(nodeId);
    nodeStatusObject.setResponseId(responseId + 2);
    heartbeatReq.setNodeStatus(nodeStatusObject);
    heartbeatReq.setLastKnownNMTokenMasterKey(registerResponse.getNMTokenMasterKey());
    heartbeatReq.setLastKnownContainerTokenMasterKey(registerResponse.getContainerTokenMasterKey());
    nodeHeartbeatResponse = resourceTrackerService.nodeHeartbeat(heartbeatReq);
    Assert.assertEquals("InValid Node Labels were not accepted by RM", NodeAction.NORMAL, nodeHeartbeatResponse.getNodeAction());
    assertCollectionEquals(nodeLabelsMgr.getNodeLabels().get(nodeId), oldLabels);
    Assert.assertFalse("Node Labels should not accepted by RM", nodeHeartbeatResponse.getAreNodeLabelsAcceptedByRM());
    rm.stop();
}

Example 5 with NodeHeartbeatRequest

use of org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest in project hadoop by apache.

the class TestResourceTrackerService method testNodeHeartBeatWithInvalidLabels.

@Test
public void testNodeHeartBeatWithInvalidLabels() throws Exception {
    writeToHostsFile("host2");
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile.getAbsolutePath());
    conf.set(YarnConfiguration.NODELABEL_CONFIGURATION_TYPE, YarnConfiguration.DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE);
    final RMNodeLabelsManager nodeLabelsMgr = new NullRMNodeLabelsManager();
    rm = new MockRM(conf) {

        @Override
        protected RMNodeLabelsManager createNodeLabelManager() {
            return nodeLabelsMgr;
        }
    };
    rm.start();
    try {
        nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("A", "B", "C"));
    } catch (IOException e) {
        Assert.fail("Caught Exception while intializing");
        e.printStackTrace();
    }
    ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
    RegisterNodeManagerRequest registerReq = Records.newRecord(RegisterNodeManagerRequest.class);
    NodeId nodeId = NodeId.newInstance("host2", 1234);
    Resource capability = BuilderUtils.newResource(1024, 1);
    registerReq.setResource(capability);
    registerReq.setNodeId(nodeId);
    registerReq.setHttpPort(1234);
    registerReq.setNMVersion(YarnVersionInfo.getVersion());
    registerReq.setNodeLabels(toNodeLabelSet("A"));
    RegisterNodeManagerResponse registerResponse = resourceTrackerService.registerNodeManager(registerReq);
    NodeHeartbeatRequest heartbeatReq = Records.newRecord(NodeHeartbeatRequest.class);
    // Invalid heart beat labels
    heartbeatReq.setNodeLabels(toNodeLabelSet("B", "#C"));
    heartbeatReq.setNodeStatus(getNodeStatusObject(nodeId));
    heartbeatReq.setLastKnownNMTokenMasterKey(registerResponse.getNMTokenMasterKey());
    heartbeatReq.setLastKnownContainerTokenMasterKey(registerResponse.getContainerTokenMasterKey());
    NodeHeartbeatResponse nodeHeartbeatResponse = resourceTrackerService.nodeHeartbeat(heartbeatReq);
    // response should be NORMAL when RM heartbeat labels are rejected
    Assert.assertEquals("Response should be NORMAL when RM heartbeat labels" + " are rejected", NodeAction.NORMAL, nodeHeartbeatResponse.getNodeAction());
    Assert.assertFalse(nodeHeartbeatResponse.getAreNodeLabelsAcceptedByRM());
    Assert.assertNotNull(nodeHeartbeatResponse.getDiagnosticsMessage());
    rm.stop();
}