Examples with Container org.apache.hadoop.yarn.api.records.Container used on opensource projects

Example 36 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class SchedulerNode method allocateContainer.

/**
   * The Scheduler has allocated containers on this node to the given
   * application.
   * @param rmContainer Allocated container
   * @param launchedOnNode True if the container has been launched
   */
private synchronized void allocateContainer(RMContainer rmContainer, boolean launchedOnNode) {
    Container container = rmContainer.getContainer();
    if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
        deductUnallocatedResource(container.getResource());
        ++numContainers;
    }
    launchedContainers.put(container.getId(), new ContainerInfo(rmContainer, launchedOnNode));
    if (LOG.isDebugEnabled()) {
        LOG.debug("Assigned container " + container.getId() + " of capacity " + container.getResource() + " on host " + rmNode.getNodeAddress() + ", which has " + numContainers + " containers, " + getAllocatedResource() + " used and " + getUnallocatedResource() + " available after allocation");
    }
}

Example 37 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class FiCaSchedulerApp method getAllocation.

/**
   * This method produces an Allocation that includes the current view
   * of the resources that will be allocated to and preempted from this
   * application.
   *
   * @param resourceCalculator resourceCalculator
   * @param clusterResource clusterResource
   * @param minimumAllocation minimumAllocation
   * @return an allocation
   */
public Allocation getAllocation(ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumAllocation) {
    try {
        writeLock.lock();
        Set<ContainerId> currentContPreemption = Collections.unmodifiableSet(new HashSet<ContainerId>(containersToPreempt));
        containersToPreempt.clear();
        Resource tot = Resource.newInstance(0, 0);
        for (ContainerId c : currentContPreemption) {
            Resources.addTo(tot, liveContainers.get(c).getContainer().getResource());
        }
        int numCont = (int) Math.ceil(Resources.divide(rc, clusterResource, tot, minimumAllocation));
        ResourceRequest rr = ResourceRequest.newBuilder().priority(Priority.UNDEFINED).resourceName(ResourceRequest.ANY).capability(minimumAllocation).numContainers(numCont).build();
        List<Container> newlyAllocatedContainers = pullNewlyAllocatedContainers();
        List<Container> newlyIncreasedContainers = pullNewlyIncreasedContainers();
        List<Container> newlyDecreasedContainers = pullNewlyDecreasedContainers();
        List<Container> newlyPromotedContainers = pullNewlyPromotedContainers();
        List<Container> newlyDemotedContainers = pullNewlyDemotedContainers();
        List<NMToken> updatedNMTokens = pullUpdatedNMTokens();
        Resource headroom = getHeadroom();
        setApplicationHeadroomForMetrics(headroom);
        return new Allocation(newlyAllocatedContainers, headroom, null, currentContPreemption, Collections.singletonList(rr), updatedNMTokens, newlyIncreasedContainers, newlyDecreasedContainers, newlyPromotedContainers, newlyDemotedContainers);
    } finally {
        writeLock.unlock();
    }
}

Example 38 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class TestWorkPreservingRMRestart method testUAMRecoveryOnRMWorkPreservingRestart.

@Test(timeout = 600000)
public void testUAMRecoveryOnRMWorkPreservingRestart() throws Exception {
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    // start RM
    rm1 = new MockRM(conf, memStore);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    // create app and launch the UAM
    RMApp app0 = rm1.submitApp(200, true);
    MockAM am0 = MockRM.launchUAM(app0, rm1, nm1);
    am0.registerAppAttempt();
    // Allocate containers to UAM
    int numContainers = 2;
    am0.allocate("127.0.0.1", 1000, numContainers, new ArrayList<ContainerId>());
    nm1.nodeHeartbeat(true);
    List<Container> conts = am0.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
    Assert.assertTrue(conts.isEmpty());
    while (conts.size() == 0) {
        nm1.nodeHeartbeat(true);
        conts.addAll(am0.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers());
        Thread.sleep(500);
    }
    Assert.assertFalse(conts.isEmpty());
    // start new RM
    rm2 = new MockRM(conf, memStore);
    rm2.start();
    rm2.waitForState(app0.getApplicationId(), RMAppState.ACCEPTED);
    rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.LAUNCHED);
    // recover app
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());
    RMApp recoveredApp = rm2.getRMContext().getRMApps().get(app0.getApplicationId());
    NMContainerStatus container1 = TestRMRestart.createNMContainerStatus(am0.getApplicationAttemptId(), 1, ContainerState.RUNNING);
    NMContainerStatus container2 = TestRMRestart.createNMContainerStatus(am0.getApplicationAttemptId(), 2, ContainerState.RUNNING);
    nm1.registerNode(Arrays.asList(container1, container2), null);
    // Wait for RM to settle down on recovering containers;
    waitForNumContainersToRecover(2, rm2, am0.getApplicationAttemptId());
    // retry registerApplicationMaster() after RM restart.
    am0.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext());
    am0.registerAppAttempt(true);
    // Check if UAM is correctly recovered on restart
    rm2.waitForState(app0.getApplicationId(), RMAppState.RUNNING);
    rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
    // Check if containers allocated to UAM are recovered
    Map<ApplicationId, SchedulerApplication> schedulerApps = ((AbstractYarnScheduler) rm2.getResourceScheduler()).getSchedulerApplications();
    SchedulerApplication schedulerApp = schedulerApps.get(recoveredApp.getApplicationId());
    SchedulerApplicationAttempt schedulerAttempt = schedulerApp.getCurrentAppAttempt();
    Assert.assertEquals(numContainers, schedulerAttempt.getLiveContainers().size());
    // Check if UAM is able to heart beat
    Assert.assertNotNull(am0.doHeartbeat());
    // Complete the UAM
    am0.unregisterAppAttempt(false);
    rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
    rm2.waitForState(app0.getApplicationId(), RMAppState.FINISHED);
    Assert.assertEquals(FinalApplicationStatus.SUCCEEDED, recoveredApp.getFinalApplicationStatus());
    // Restart RM once more to check UAM is not re-run
    MockRM rm3 = new MockRM(conf, memStore);
    rm3.start();
    recoveredApp = rm3.getRMContext().getRMApps().get(app0.getApplicationId());
    Assert.assertEquals(RMAppState.FINISHED, recoveredApp.getState());
}

Example 39 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class TestRMAppAttemptTransitions method testRunningToKilled.

@Test
public void testRunningToKilled() {
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.KILL));
    // ignored ContainerFinished and Expire at FinalSaving if we were supposed
    // to Killed state.
    assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState());
    NodeId anyNodeId = NodeId.newInstance("host", 1234);
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(amContainer.getId(), ContainerState.COMPLETE, "", 0, amContainer.getResource()), anyNodeId));
    applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
    assertEquals(RMAppAttemptState.FINAL_SAVING, applicationAttempt.getAppAttemptState());
    assertEquals(YarnApplicationAttemptState.RUNNING, applicationAttempt.createApplicationAttemptState());
    sendAttemptUpdateSavedEvent(applicationAttempt);
    assertEquals(RMAppAttemptState.KILLED, applicationAttempt.getAppAttemptState());
    assertEquals(0, applicationAttempt.getJustFinishedContainers().size());
    assertEquals(amContainer, applicationAttempt.getMasterContainer());
    assertEquals(0, application.getRanNodes().size());
    String rmAppPageUrl = pjoin(RM_WEBAPP_ADDR, "cluster", "app", applicationAttempt.getAppAttemptId().getApplicationId());
    assertEquals(rmAppPageUrl, applicationAttempt.getOriginalTrackingUrl());
    assertEquals(rmAppPageUrl, applicationAttempt.getTrackingUrl());
    verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
    verifyAMHostAndPortInvalidated();
    verifyApplicationAttemptFinished(RMAppAttemptState.KILLED);
}

Example 40 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class TestRMAppAttemptTransitions method testContainersCleanupForLastAttempt.

@SuppressWarnings("deprecation")
@Test
public void testContainersCleanupForLastAttempt() {
    // create a failed attempt.
    applicationAttempt = new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler, masterService, submissionContext, new Configuration(), BuilderUtils.newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1), application);
    when(submissionContext.getKeepContainersAcrossApplicationAttempts()).thenReturn(true);
    when(submissionContext.getMaxAppAttempts()).thenReturn(1);
    Container amContainer = allocateApplicationAttempt();
    launchApplicationAttempt(amContainer);
    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
    ContainerStatus cs1 = ContainerStatus.newInstance(amContainer.getId(), ContainerState.COMPLETE, "some error", 123);
    ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
    NodeId anyNodeId = NodeId.newInstance("host", 1234);
    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(appAttemptId, cs1, anyNodeId));
    assertEquals(YarnApplicationAttemptState.RUNNING, applicationAttempt.createApplicationAttemptState());
    sendAttemptUpdateSavedEvent(applicationAttempt);
    assertEquals(RMAppAttemptState.FAILED, applicationAttempt.getAppAttemptState());
    assertFalse(transferStateFromPreviousAttempt);
    verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
}