Examples with RMNodeFinishedContainersPulledByAMEvent org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeFinishedContainersPulledByAMEvent used on opensource projects

Example 1 with RMNodeFinishedContainersPulledByAMEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeFinishedContainersPulledByAMEvent in project hadoop by apache.

the class AbstractYarnScheduler method updateCompletedContainers.

/**
   * Process completed container list.
   * @param completedContainers Extracted list of completed containers
   * @param releasedResources Reference resource object for completed containers
   * @param nodeId NodeId corresponding to the NodeManager
   * @return The total number of released containers
   */
protected int updateCompletedContainers(List<ContainerStatus> completedContainers, Resource releasedResources, NodeId nodeId) {
    int releasedContainers = 0;
    SchedulerNode node = getNode(nodeId);
    List<ContainerId> untrackedContainerIdList = new ArrayList<ContainerId>();
    for (ContainerStatus completedContainer : completedContainers) {
        ContainerId containerId = completedContainer.getContainerId();
        LOG.debug("Container FINISHED: " + containerId);
        RMContainer container = getRMContainer(containerId);
        completedContainer(container, completedContainer, RMContainerEventType.FINISHED);
        if (node != null) {
            node.releaseContainer(containerId, true);
        }
        if (container != null) {
            releasedContainers++;
            Resource ars = container.getAllocatedResource();
            if (ars != null) {
                Resources.addTo(releasedResources, ars);
            }
            Resource rrs = container.getReservedResource();
            if (rrs != null) {
                Resources.addTo(releasedResources, rrs);
            }
        } else {
            // Add containers which are untracked by RM.
            untrackedContainerIdList.add(containerId);
        }
    }
    // Acknowledge NM to remove RM-untracked-containers from NM context.
    if (!untrackedContainerIdList.isEmpty()) {
        this.rmContext.getDispatcher().getEventHandler().handle(new RMNodeFinishedContainersPulledByAMEvent(nodeId, untrackedContainerIdList));
    }
    return releasedContainers;
}

Example 2 with RMNodeFinishedContainersPulledByAMEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeFinishedContainersPulledByAMEvent in project hadoop by apache.

the class RMAppAttemptImpl method sendFinishedAMContainerToNM.

// Ack NM to remove finished AM container, not waiting for
// new appattempt to pull am container complete msg, new  appattempt
// may launch fail and leaves too many completed container in NM
private void sendFinishedAMContainerToNM(NodeId nodeId, ContainerId containerId) {
    List<ContainerId> containerIdList = new ArrayList<ContainerId>();
    containerIdList.add(containerId);
    eventHandler.handle(new RMNodeFinishedContainersPulledByAMEvent(nodeId, containerIdList));
}

Example 3 with RMNodeFinishedContainersPulledByAMEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeFinishedContainersPulledByAMEvent in project hadoop by apache.

the class TestRMNodeTransitions method testForHandlingDuplicatedCompltedContainers.

@Test
public void testForHandlingDuplicatedCompltedContainers() {
    // Start the node
    node.handle(new RMNodeStartedEvent(null, null, null));
    // Add info to the queue first
    node.setNextHeartBeat(false);
    ContainerId completedContainerId1 = BuilderUtils.newContainerId(BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
    RMNodeStatusEvent statusEvent1 = getMockRMNodeStatusEvent(null);
    ContainerStatus containerStatus1 = mock(ContainerStatus.class);
    doReturn(completedContainerId1).when(containerStatus1).getContainerId();
    doReturn(Collections.singletonList(containerStatus1)).when(statusEvent1).getContainers();
    verify(scheduler, times(1)).handle(any(NodeUpdateSchedulerEvent.class));
    node.handle(statusEvent1);
    verify(scheduler, times(1)).handle(any(NodeUpdateSchedulerEvent.class));
    Assert.assertEquals(1, node.getQueueSize());
    Assert.assertEquals(1, node.getCompletedContainers().size());
    // test for duplicate entries
    node.handle(statusEvent1);
    Assert.assertEquals(1, node.getQueueSize());
    // send clean up container event
    node.handle(new RMNodeFinishedContainersPulledByAMEvent(node.getNodeID(), Collections.singletonList(completedContainerId1)));
    NodeHeartbeatResponse hbrsp = Records.newRecord(NodeHeartbeatResponse.class);
    node.updateNodeHeartbeatResponseForCleanup(hbrsp);
    Assert.assertEquals(1, hbrsp.getContainersToBeRemovedFromNM().size());
    Assert.assertEquals(0, node.getCompletedContainers().size());
}

Example 4 with RMNodeFinishedContainersPulledByAMEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeFinishedContainersPulledByAMEvent in project hadoop by apache.

the class TestRMAppAttemptTransitions method testFinishedContainer.

@Test
public void testFinishedContainer() {
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    // Complete one container
    ContainerId containerId1 = BuilderUtils.newContainerId(applicationAttempt.getAppAttemptId(), 2);
    Container container1 = mock(Container.class);
    ContainerStatus containerStatus1 = mock(ContainerStatus.class);
    when(container1.getId()).thenReturn(containerId1);
    when(containerStatus1.getContainerId()).thenReturn(containerId1);
    when(container1.getNodeId()).thenReturn(NodeId.newInstance("host", 1234));
    application.handle(new RMAppRunningOnNodeEvent(application.getApplicationId(), container1.getNodeId()));
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(applicationAttempt.getAppAttemptId(), containerStatus1, container1.getNodeId()));
    ArgumentCaptor<RMNodeFinishedContainersPulledByAMEvent> captor = ArgumentCaptor.forClass(RMNodeFinishedContainersPulledByAMEvent.class);
    // Verify justFinishedContainers
    Assert.assertEquals(1, applicationAttempt.getJustFinishedContainers().size());
    Assert.assertEquals(container1.getId(), applicationAttempt.getJustFinishedContainers().get(0).getContainerId());
    Assert.assertEquals(0, getFinishedContainersSentToAM(applicationAttempt).size());
    // Verify finishedContainersSentToAM gets container after pull
    List<ContainerStatus> containerStatuses = applicationAttempt.pullJustFinishedContainers();
    Assert.assertEquals(1, containerStatuses.size());
    Mockito.verify(rmnodeEventHandler, never()).handle(Mockito.any(RMNodeEvent.class));
    Assert.assertTrue(applicationAttempt.getJustFinishedContainers().isEmpty());
    Assert.assertEquals(1, getFinishedContainersSentToAM(applicationAttempt).size());
    // Verify container is acked to NM via the RMNodeEvent after second pull
    containerStatuses = applicationAttempt.pullJustFinishedContainers();
    Assert.assertEquals(0, containerStatuses.size());
    Mockito.verify(rmnodeEventHandler).handle(captor.capture());
    Assert.assertEquals(container1.getId(), captor.getValue().getContainers().get(0));
    Assert.assertTrue(applicationAttempt.getJustFinishedContainers().isEmpty());
    Assert.assertEquals(0, getFinishedContainersSentToAM(applicationAttempt).size());
    // verify if no containers to acknowledge to NM then event should not be
    // triggered. Number of times event invoked is 1 i.e on second pull
    containerStatuses = applicationAttempt.pullJustFinishedContainers();
    Assert.assertEquals(0, containerStatuses.size());
    Mockito.verify(rmnodeEventHandler, times(1)).handle(Mockito.any(RMNodeEvent.class));
}

Example 5 with RMNodeFinishedContainersPulledByAMEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeFinishedContainersPulledByAMEvent in project hadoop by apache.

the class RMAppAttemptImpl method sendFinishedContainersToNM.

// Ack NM to remove finished containers from context.
private void sendFinishedContainersToNM() {
    for (NodeId nodeId : finishedContainersSentToAM.keySet()) {
        // Clear and get current values
        List<ContainerStatus> currentSentContainers = finishedContainersSentToAM.put(nodeId, new ArrayList<>());
        List<ContainerId> containerIdList = new ArrayList<>(currentSentContainers.size());
        for (ContainerStatus containerStatus : currentSentContainers) {
            containerIdList.add(containerStatus.getContainerId());
        }
        eventHandler.handle(new RMNodeFinishedContainersPulledByAMEvent(nodeId, containerIdList));
    }
    this.finishedContainersSentToAM.clear();
}