Examples with ContainerStatus org.apache.hadoop.yarn.api.records.ContainerStatus used on opensource projects

Example 56 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class AbstractYarnScheduler method updateCompletedContainers.

/**
   * Process completed container list.
   * @param completedContainers Extracted list of completed containers
   * @param releasedResources Reference resource object for completed containers
   * @param nodeId NodeId corresponding to the NodeManager
   * @return The total number of released containers
   */
protected int updateCompletedContainers(List<ContainerStatus> completedContainers, Resource releasedResources, NodeId nodeId) {
    int releasedContainers = 0;
    SchedulerNode node = getNode(nodeId);
    List<ContainerId> untrackedContainerIdList = new ArrayList<ContainerId>();
    for (ContainerStatus completedContainer : completedContainers) {
        ContainerId containerId = completedContainer.getContainerId();
        LOG.debug("Container FINISHED: " + containerId);
        RMContainer container = getRMContainer(containerId);
        completedContainer(container, completedContainer, RMContainerEventType.FINISHED);
        if (node != null) {
            node.releaseContainer(containerId, true);
        }
        if (container != null) {
            releasedContainers++;
            Resource ars = container.getAllocatedResource();
            if (ars != null) {
                Resources.addTo(releasedResources, ars);
            }
            Resource rrs = container.getReservedResource();
            if (rrs != null) {
                Resources.addTo(releasedResources, rrs);
            }
        } else {
            // Add containers which are untracked by RM.
            untrackedContainerIdList.add(containerId);
        }
    }
    // Acknowledge NM to remove RM-untracked-containers from NM context.
    if (!untrackedContainerIdList.isEmpty()) {
        this.rmContext.getDispatcher().getEventHandler().handle(new RMNodeFinishedContainersPulledByAMEvent(nodeId, untrackedContainerIdList));
    }
    return releasedContainers;
}

Example 57 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class RMAppAttemptImpl method transferStateFromAttempt.

public void transferStateFromAttempt(RMAppAttempt attempt) {
    this.justFinishedContainers = attempt.getJustFinishedContainersReference();
    this.finishedContainersSentToAM = attempt.getFinishedContainersSentToAMReference();
    // this msg again after am restart
    if (!this.finishedContainersSentToAM.isEmpty()) {
        for (NodeId nodeId : this.finishedContainersSentToAM.keySet()) {
            List<ContainerStatus> containerStatuses = this.finishedContainersSentToAM.get(nodeId);
            this.justFinishedContainers.putIfAbsent(nodeId, new ArrayList<>());
            this.justFinishedContainers.get(nodeId).addAll(containerStatuses);
        }
        this.finishedContainersSentToAM.clear();
    }
}

Example 58 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class TestRMNodeTransitions method testForHandlingDuplicatedCompltedContainers.

@Test
public void testForHandlingDuplicatedCompltedContainers() {
    // Start the node
    node.handle(new RMNodeStartedEvent(null, null, null));
    // Add info to the queue first
    node.setNextHeartBeat(false);
    ContainerId completedContainerId1 = BuilderUtils.newContainerId(BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
    RMNodeStatusEvent statusEvent1 = getMockRMNodeStatusEvent(null);
    ContainerStatus containerStatus1 = mock(ContainerStatus.class);
    doReturn(completedContainerId1).when(containerStatus1).getContainerId();
    doReturn(Collections.singletonList(containerStatus1)).when(statusEvent1).getContainers();
    verify(scheduler, times(1)).handle(any(NodeUpdateSchedulerEvent.class));
    node.handle(statusEvent1);
    verify(scheduler, times(1)).handle(any(NodeUpdateSchedulerEvent.class));
    Assert.assertEquals(1, node.getQueueSize());
    Assert.assertEquals(1, node.getCompletedContainers().size());
    // test for duplicate entries
    node.handle(statusEvent1);
    Assert.assertEquals(1, node.getQueueSize());
    // send clean up container event
    node.handle(new RMNodeFinishedContainersPulledByAMEvent(node.getNodeID(), Collections.singletonList(completedContainerId1)));
    NodeHeartbeatResponse hbrsp = Records.newRecord(NodeHeartbeatResponse.class);
    node.updateNodeHeartbeatResponseForCleanup(hbrsp);
    Assert.assertEquals(1, hbrsp.getContainersToBeRemovedFromNM().size());
    Assert.assertEquals(0, node.getCompletedContainers().size());
}

Example 59 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class TestRMNodeTransitions method testStatusChange.

@Test(timeout = 5000)
public void testStatusChange() {
    //Start the node
    node.handle(new RMNodeStartedEvent(null, null, null));
    //Add info to the queue first
    node.setNextHeartBeat(false);
    ContainerId completedContainerId1 = BuilderUtils.newContainerId(BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
    ContainerId completedContainerId2 = BuilderUtils.newContainerId(BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 1);
    RMNodeStatusEvent statusEvent1 = getMockRMNodeStatusEvent(null);
    RMNodeStatusEvent statusEvent2 = getMockRMNodeStatusEvent(null);
    ContainerStatus containerStatus1 = mock(ContainerStatus.class);
    ContainerStatus containerStatus2 = mock(ContainerStatus.class);
    doReturn(completedContainerId1).when(containerStatus1).getContainerId();
    doReturn(Collections.singletonList(containerStatus1)).when(statusEvent1).getContainers();
    doReturn(completedContainerId2).when(containerStatus2).getContainerId();
    doReturn(Collections.singletonList(containerStatus2)).when(statusEvent2).getContainers();
    verify(scheduler, times(1)).handle(any(NodeUpdateSchedulerEvent.class));
    node.handle(statusEvent1);
    node.handle(statusEvent2);
    verify(scheduler, times(1)).handle(any(NodeUpdateSchedulerEvent.class));
    Assert.assertEquals(2, node.getQueueSize());
    node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.EXPIRE));
    Assert.assertEquals(0, node.getQueueSize());
}

Example 60 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class TestRMNodeTransitions method testDisappearingContainer.

@Test
public void testDisappearingContainer() {
    ContainerId cid1 = BuilderUtils.newContainerId(BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(1, 1), 1), 1);
    ContainerId cid2 = BuilderUtils.newContainerId(BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(2, 2), 2), 2);
    ArrayList<ContainerStatus> containerStats = new ArrayList<ContainerStatus>();
    containerStats.add(ContainerStatus.newInstance(cid1, ContainerState.RUNNING, "", -1));
    containerStats.add(ContainerStatus.newInstance(cid2, ContainerState.RUNNING, "", -1));
    node = getRunningNode();
    node.handle(getMockRMNodeStatusEvent(containerStats));
    assertEquals("unexpected number of running containers", 2, node.getLaunchedContainers().size());
    Assert.assertTrue("first container not running", node.getLaunchedContainers().contains(cid1));
    Assert.assertTrue("second container not running", node.getLaunchedContainers().contains(cid2));
    assertEquals("already completed containers", 0, completedContainers.size());
    containerStats.remove(0);
    node.handle(getMockRMNodeStatusEvent(containerStats));
    assertEquals("expected one container to be completed", 1, completedContainers.size());
    ContainerStatus cs = completedContainers.get(0);
    assertEquals("first container not the one that completed", cid1, cs.getContainerId());
    assertEquals("completed container not marked complete", ContainerState.COMPLETE, cs.getState());
    assertEquals("completed container not marked aborted", ContainerExitStatus.ABORTED, cs.getExitStatus());
    Assert.assertTrue("completed container not marked missing", cs.getDiagnostics().contains("not reported"));
    assertEquals("unexpected number of running containers", 1, node.getLaunchedContainers().size());
    Assert.assertTrue("second container not running", node.getLaunchedContainers().contains(cid2));
}