Examples with ContainerStatus org.apache.hadoop.yarn.api.records.ContainerStatus used on opensource projects

Example 91 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class TestSchedulerUtils method testCreatePreemptedContainerStatus.

@Test
public void testCreatePreemptedContainerStatus() {
    ContainerStatus cd = SchedulerUtils.createPreemptedContainerStatus(ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), 1), "x");
    Assert.assertEquals(ContainerExitStatus.PREEMPTED, cd.getExitStatus());
}

Example 92 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class TestRMAppAttemptTransitions method testFinishedContainer.

@Test
public void testFinishedContainer() {
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    // Complete one container
    ContainerId containerId1 = BuilderUtils.newContainerId(applicationAttempt.getAppAttemptId(), 2);
    Container container1 = mock(Container.class);
    ContainerStatus containerStatus1 = mock(ContainerStatus.class);
    when(container1.getId()).thenReturn(containerId1);
    when(containerStatus1.getContainerId()).thenReturn(containerId1);
    when(container1.getNodeId()).thenReturn(NodeId.newInstance("host", 1234));
    application.handle(new RMAppRunningOnNodeEvent(application.getApplicationId(), container1.getNodeId()));
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(applicationAttempt.getAppAttemptId(), containerStatus1, container1.getNodeId()));
    ArgumentCaptor<RMNodeFinishedContainersPulledByAMEvent> captor = ArgumentCaptor.forClass(RMNodeFinishedContainersPulledByAMEvent.class);
    // Verify justFinishedContainers
    Assert.assertEquals(1, applicationAttempt.getJustFinishedContainers().size());
    Assert.assertEquals(container1.getId(), applicationAttempt.getJustFinishedContainers().get(0).getContainerId());
    Assert.assertEquals(0, getFinishedContainersSentToAM(applicationAttempt).size());
    // Verify finishedContainersSentToAM gets container after pull
    List<ContainerStatus> containerStatuses = applicationAttempt.pullJustFinishedContainers();
    Assert.assertEquals(1, containerStatuses.size());
    Mockito.verify(rmnodeEventHandler, never()).handle(Mockito.any(RMNodeEvent.class));
    Assert.assertTrue(applicationAttempt.getJustFinishedContainers().isEmpty());
    Assert.assertEquals(1, getFinishedContainersSentToAM(applicationAttempt).size());
    // Verify container is acked to NM via the RMNodeEvent after second pull
    containerStatuses = applicationAttempt.pullJustFinishedContainers();
    Assert.assertEquals(0, containerStatuses.size());
    Mockito.verify(rmnodeEventHandler).handle(captor.capture());
    Assert.assertEquals(container1.getId(), captor.getValue().getContainers().get(0));
    Assert.assertTrue(applicationAttempt.getJustFinishedContainers().isEmpty());
    Assert.assertEquals(0, getFinishedContainersSentToAM(applicationAttempt).size());
    // verify if no containers to acknowledge to NM then event should not be
    // triggered. Number of times event invoked is 1 i.e on second pull
    containerStatuses = applicationAttempt.pullJustFinishedContainers();
    Assert.assertEquals(0, containerStatuses.size());
    Mockito.verify(rmnodeEventHandler, times(1)).handle(Mockito.any(RMNodeEvent.class));
}

Example 93 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class TestRMAppAttemptTransitions method testRunningToFailed.

@Test
public void testRunningToFailed() {
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    String containerDiagMsg = "some error";
    int exitCode = 123;
    ContainerStatus cs = BuilderUtils.newContainerStatus(amContainer.getId(), ContainerState.COMPLETE, containerDiagMsg, exitCode, amContainer.getResource());
    ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
    NodeId anyNodeId = NodeId.newInstance("host", 1234);
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(appAttemptId, cs, anyNodeId));
    // ignored ContainerFinished and Expire at FinalSaving if we were supposed
    // to Failed state.
    assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState());
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(amContainer.getId(), ContainerState.COMPLETE, "", 0, amContainer.getResource()), anyNodeId));
    applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
    assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState());
    assertEquals(YarnApplicationAttemptState.RUNNING, applicationAttempt.createApplicationAttemptState());
    sendAttemptUpdateSavedEvent(applicationAttempt);
    assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState());
    assertEquals(0, applicationAttempt.getJustFinishedContainers().size());
    assertEquals(amContainer, applicationAttempt.getMasterContainer());
    assertEquals(0, application.getRanNodes().size());
    String rmAppPageUrl = pjoin(RM_WEBAPP_ADDR, "cluster", "app", applicationAttempt.getAppAttemptId().getApplicationId());
    assertEquals(rmAppPageUrl, applicationAttempt.getOriginalTrackingUrl());
    assertEquals(rmAppPageUrl, applicationAttempt.getTrackingUrl());
    verifyAMHostAndPortInvalidated();
    verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
}

Example 94 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class TestRMContainerImpl method testExpireWhileRunning.

@Test
public void testExpireWhileRunning() {
    DrainDispatcher drainDispatcher = new DrainDispatcher();
    EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
    EventHandler generic = mock(EventHandler.class);
    drainDispatcher.register(RMAppAttemptEventType.class, appAttemptEventHandler);
    drainDispatcher.register(RMNodeEventType.class, generic);
    drainDispatcher.init(new YarnConfiguration());
    drainDispatcher.start();
    NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
    ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
    ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);
    Resource resource = BuilderUtils.newResource(512, 1);
    Priority priority = BuilderUtils.newPriority(5);
    Container container = BuilderUtils.newContainer(containerId, nodeId, "host:3465", resource, priority, null);
    ConcurrentMap<ApplicationId, RMApp> appMap = new ConcurrentHashMap<>();
    RMApp rmApp = mock(RMApp.class);
    appMap.putIfAbsent(appId, rmApp);
    RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
    SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
    RMContext rmContext = mock(RMContext.class);
    when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
    when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
    when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
    when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
    YarnConfiguration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO, true);
    when(rmContext.getYarnConfiguration()).thenReturn(conf);
    when(rmContext.getRMApps()).thenReturn(appMap);
    RMContainer rmContainer = new RMContainerImpl(container, SchedulerRequestKey.extractFrom(container), appAttemptId, nodeId, "user", rmContext);
    assertEquals(RMContainerState.NEW, rmContainer.getState());
    assertEquals(resource, rmContainer.getAllocatedResource());
    assertEquals(nodeId, rmContainer.getAllocatedNode());
    assertEquals(priority, rmContainer.getAllocatedSchedulerKey().getPriority());
    verify(writer).containerStarted(any(RMContainer.class));
    verify(publisher).containerCreated(any(RMContainer.class), anyLong());
    rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START));
    drainDispatcher.await();
    assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());
    rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.ACQUIRED));
    drainDispatcher.await();
    assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());
    rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED));
    drainDispatcher.await();
    assertEquals(RMContainerState.RUNNING, rmContainer.getState());
    assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL());
    // In RUNNING state. Verify EXPIRE and associated actions.
    reset(appAttemptEventHandler);
    ContainerStatus containerStatus = SchedulerUtils.createAbnormalContainerStatus(containerId, SchedulerUtils.EXPIRED_CONTAINER);
    rmContainer.handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType.EXPIRE));
    drainDispatcher.await();
    assertEquals(RMContainerState.RUNNING, rmContainer.getState());
    verify(writer, never()).containerFinished(any(RMContainer.class));
    verify(publisher, never()).containerFinished(any(RMContainer.class), anyLong());
}

Example 95 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class TestAbstractYarnScheduler method testContainerReleasedByNode.

@Test(timeout = 60000)
public void testContainerReleasedByNode() throws Exception {
    System.out.println("Starting testContainerReleasedByNode");
    configureScheduler();
    YarnConfiguration conf = getConf();
    MockRM rm1 = new MockRM(conf);
    try {
        rm1.start();
        RMApp app1 = rm1.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), false, "default", -1, null, "Test", false, true);
        MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService());
        nm1.registerNode();
        MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
        // allocate a container that fills more than half the node
        am1.allocate("127.0.0.1", 8192, 1, new ArrayList<ContainerId>());
        nm1.nodeHeartbeat(true);
        // wait for containers to be allocated.
        List<Container> containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
        while (containers.isEmpty()) {
            Thread.sleep(10);
            nm1.nodeHeartbeat(true);
            containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
        }
        // release the container from the AM
        ContainerId cid = containers.get(0).getId();
        List<ContainerId> releasedContainers = new ArrayList<>(1);
        releasedContainers.add(cid);
        List<ContainerStatus> completedContainers = am1.allocate(new ArrayList<ResourceRequest>(), releasedContainers).getCompletedContainersStatuses();
        while (completedContainers.isEmpty()) {
            Thread.sleep(10);
            completedContainers = am1.allocate(new ArrayList<ResourceRequest>(), releasedContainers).getCompletedContainersStatuses();
        }
        // verify new container can be allocated immediately because container
        // never launched on the node
        containers = am1.allocate("127.0.0.1", 8192, 1, new ArrayList<ContainerId>()).getAllocatedContainers();
        nm1.nodeHeartbeat(true);
        while (containers.isEmpty()) {
            Thread.sleep(10);
            nm1.nodeHeartbeat(true);
            containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
        }
        // launch the container on the node
        cid = containers.get(0).getId();
        nm1.nodeHeartbeat(cid.getApplicationAttemptId(), cid.getContainerId(), ContainerState.RUNNING);
        rm1.waitForState(nm1, cid, RMContainerState.RUNNING);
        // release the container from the AM
        releasedContainers.clear();
        releasedContainers.add(cid);
        completedContainers = am1.allocate(new ArrayList<ResourceRequest>(), releasedContainers).getCompletedContainersStatuses();
        while (completedContainers.isEmpty()) {
            Thread.sleep(10);
            completedContainers = am1.allocate(new ArrayList<ResourceRequest>(), releasedContainers).getCompletedContainersStatuses();
        }
        // verify new container cannot be allocated immediately because container
        // has not been released by the node
        containers = am1.allocate("127.0.0.1", 8192, 1, new ArrayList<ContainerId>()).getAllocatedContainers();
        nm1.nodeHeartbeat(true);
        Assert.assertTrue("new container allocated before node freed old", containers.isEmpty());
        for (int i = 0; i < 10; ++i) {
            Thread.sleep(10);
            containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
            nm1.nodeHeartbeat(true);
            Assert.assertTrue("new container allocated before node freed old", containers.isEmpty());
        }
        // free the old container from the node
        nm1.nodeHeartbeat(cid.getApplicationAttemptId(), cid.getContainerId(), ContainerState.COMPLETE);
        // verify new container is now allocated
        containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
        while (containers.isEmpty()) {
            Thread.sleep(10);
            nm1.nodeHeartbeat(true);
            containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
        }
    } finally {
        rm1.stop();
        System.out.println("Stopping testContainerReleasedByNode");
    }
}