Examples with Container org.apache.hadoop.yarn.api.records.Container used on opensource projects

Example 96 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class TestAMRMProxyService method testAllocateAndReleaseContainersForMultipleAM.

@Test
public void testAllocateAndReleaseContainersForMultipleAM() throws Exception {
    int numberOfApps = 5;
    for (int testAppId = 0; testAppId < numberOfApps; testAppId++) {
        RegisterApplicationMasterResponse registerResponse = registerApplicationMaster(testAppId);
        Assert.assertNotNull(registerResponse);
        List<Container> containers = getContainersAndAssert(testAppId, 10);
        releaseContainersAndAssert(testAppId, containers);
    }
    for (int testAppId = 0; testAppId < numberOfApps; testAppId++) {
        finishApplicationMaster(testAppId, FinalApplicationStatus.SUCCEEDED);
    }
}

Example 97 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class RMNodeImpl method handleReportedIncreasedContainers.

private void handleReportedIncreasedContainers(List<Container> reportedIncreasedContainers) {
    for (Container container : reportedIncreasedContainers) {
        ContainerId containerId = container.getId();
        // more about this container
        if (containersToClean.contains(containerId)) {
            LOG.info("Container " + containerId + " already scheduled for " + "cleanup, no further processing");
            continue;
        }
        ApplicationId containerAppId = containerId.getApplicationAttemptId().getApplicationId();
        if (finishedApplications.contains(containerAppId)) {
            LOG.info("Container " + containerId + " belongs to an application that is already killed," + " no further processing");
            continue;
        }
        this.nmReportedIncreasedContainers.put(containerId, container);
    }
}

Example 98 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class AbstractYarnScheduler method recoverContainersOnNode.

public void recoverContainersOnNode(List<NMContainerStatus> containerReports, RMNode nm) {
    try {
        writeLock.lock();
        if (!rmContext.isWorkPreservingRecoveryEnabled() || containerReports == null || (containerReports != null && containerReports.isEmpty())) {
            return;
        }
        for (NMContainerStatus container : containerReports) {
            ApplicationId appId = container.getContainerId().getApplicationAttemptId().getApplicationId();
            RMApp rmApp = rmContext.getRMApps().get(appId);
            if (rmApp == null) {
                LOG.error("Skip recovering container " + container + " for unknown application.");
                killOrphanContainerOnNode(nm, container);
                continue;
            }
            SchedulerApplication<T> schedulerApp = applications.get(appId);
            if (schedulerApp == null) {
                LOG.info("Skip recovering container  " + container + " for unknown SchedulerApplication. " + "Application current state is " + rmApp.getState());
                killOrphanContainerOnNode(nm, container);
                continue;
            }
            LOG.info("Recovering container " + container);
            SchedulerApplicationAttempt schedulerAttempt = schedulerApp.getCurrentAppAttempt();
            if (!rmApp.getApplicationSubmissionContext().getKeepContainersAcrossApplicationAttempts()) {
                // Do not recover containers for stopped attempt or previous attempt.
                if (schedulerAttempt.isStopped() || !schedulerAttempt.getApplicationAttemptId().equals(container.getContainerId().getApplicationAttemptId())) {
                    LOG.info("Skip recovering container " + container + " for already stopped attempt.");
                    killOrphanContainerOnNode(nm, container);
                    continue;
                }
            }
            // create container
            RMContainer rmContainer = recoverAndCreateContainer(container, nm);
            // recover RMContainer
            rmContainer.handle(new RMContainerRecoverEvent(container.getContainerId(), container));
            // recover scheduler node
            SchedulerNode schedulerNode = nodeTracker.getNode(nm.getNodeID());
            schedulerNode.recoverContainer(rmContainer);
            // recover queue: update headroom etc.
            Queue queue = schedulerAttempt.getQueue();
            queue.recoverContainer(getClusterResource(), schedulerAttempt, rmContainer);
            // recover scheduler attempt
            schedulerAttempt.recoverContainer(schedulerNode, rmContainer);
            // set master container for the current running AMContainer for this
            // attempt.
            RMAppAttempt appAttempt = rmApp.getCurrentAppAttempt();
            if (appAttempt != null) {
                Container masterContainer = appAttempt.getMasterContainer();
                // container ID stored in AppAttempt.
                if (masterContainer != null && masterContainer.getId().equals(rmContainer.getContainerId())) {
                    ((RMContainerImpl) rmContainer).setAMContainer(true);
                }
            }
            if (schedulerAttempt.getPendingRelease().remove(container.getContainerId())) {
                // release the container
                rmContainer.handle(new RMContainerFinishedEvent(container.getContainerId(), SchedulerUtils.createAbnormalContainerStatus(container.getContainerId(), SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED));
                LOG.info(container.getContainerId() + " is released by application.");
            }
        }
    } finally {
        writeLock.unlock();
    }
}

Example 99 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class AbstractYarnScheduler method getTransferredContainers.

/*
   * YARN-3136 removed synchronized lock for this method for performance
   * purposes
   */
public List<Container> getTransferredContainers(ApplicationAttemptId currentAttempt) {
    ApplicationId appId = currentAttempt.getApplicationId();
    SchedulerApplication<T> app = applications.get(appId);
    List<Container> containerList = new ArrayList<Container>();
    RMApp appImpl = this.rmContext.getRMApps().get(appId);
    if (appImpl.getApplicationSubmissionContext().getUnmanagedAM()) {
        return containerList;
    }
    if (app == null) {
        return containerList;
    }
    Collection<RMContainer> liveContainers = app.getCurrentAppAttempt().getLiveContainers();
    ContainerId amContainerId = rmContext.getRMApps().get(appId).getCurrentAppAttempt().getMasterContainer().getId();
    for (RMContainer rmContainer : liveContainers) {
        if (!rmContainer.getContainerId().equals(amContainerId)) {
            containerList.add(rmContainer.getContainer());
        }
    }
    return containerList;
}

Example 100 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class AbstractYarnScheduler method createDemotedRMContainer.

private RMContainer createDemotedRMContainer(SchedulerApplicationAttempt appAttempt, OpportunisticContainerContext oppCntxt, RMContainer rmContainer) {
    SchedulerRequestKey sk = SchedulerRequestKey.extractFrom(rmContainer.getContainer());
    Container demotedContainer = BuilderUtils.newContainer(ContainerId.newContainerId(appAttempt.getApplicationAttemptId(), oppCntxt.getContainerIdGenerator().generateContainerId()), rmContainer.getContainer().getNodeId(), rmContainer.getContainer().getNodeHttpAddress(), rmContainer.getContainer().getResource(), sk.getPriority(), null, ExecutionType.OPPORTUNISTIC, sk.getAllocationRequestId());
    demotedContainer.setVersion(rmContainer.getContainer().getVersion());
    return SchedulerUtils.createOpportunisticRmContainer(rmContext, demotedContainer, false);
}