Examples with ContainerStatus org.apache.hadoop.yarn.api.records.ContainerStatus used on opensource projects

Example 16 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class MRAMSimulator method processResponseQueue.

@Override
@SuppressWarnings("unchecked")
protected void processResponseQueue() throws Exception {
    while (!responseQueue.isEmpty()) {
        AllocateResponse response = responseQueue.take();
        // check completed containers
        if (!response.getCompletedContainersStatuses().isEmpty()) {
            for (ContainerStatus cs : response.getCompletedContainersStatuses()) {
                ContainerId containerId = cs.getContainerId();
                if (cs.getExitStatus() == ContainerExitStatus.SUCCESS) {
                    if (assignedMaps.containsKey(containerId)) {
                        LOG.debug(MessageFormat.format("Application {0} has one" + "mapper finished ({1}).", appId, containerId));
                        assignedMaps.remove(containerId);
                        mapFinished++;
                        finishedContainers++;
                    } else if (assignedReduces.containsKey(containerId)) {
                        LOG.debug(MessageFormat.format("Application {0} has one" + "reducer finished ({1}).", appId, containerId));
                        assignedReduces.remove(containerId);
                        reduceFinished++;
                        finishedContainers++;
                    } else if (amContainer.getId().equals(containerId)) {
                        // am container released event
                        isFinished = true;
                        LOG.info(MessageFormat.format("Application {0} goes to " + "finish.", appId));
                    }
                    if (mapFinished >= mapTotal && reduceFinished >= reduceTotal) {
                        lastStep();
                    }
                } else {
                    // container to be killed
                    if (assignedMaps.containsKey(containerId)) {
                        LOG.debug(MessageFormat.format("Application {0} has one " + "mapper killed ({1}).", appId, containerId));
                        pendingFailedMaps.add(assignedMaps.remove(containerId));
                    } else if (assignedReduces.containsKey(containerId)) {
                        LOG.debug(MessageFormat.format("Application {0} has one " + "reducer killed ({1}).", appId, containerId));
                        pendingFailedReduces.add(assignedReduces.remove(containerId));
                    } else if (amContainer.getId().equals(containerId)) {
                        LOG.info(MessageFormat.format("Application {0}'s AM is " + "going to be killed. Waiting for rescheduling...", appId));
                    }
                }
            }
        }
        // check finished
        if (isAMContainerRunning && (mapFinished >= mapTotal) && (reduceFinished >= reduceTotal)) {
            isAMContainerRunning = false;
            LOG.debug(MessageFormat.format("Application {0} sends out event " + "to clean up its AM container.", appId));
            isFinished = true;
            break;
        }
        // check allocated containers
        for (Container container : response.getAllocatedContainers()) {
            if (!scheduledMaps.isEmpty()) {
                ContainerSimulator cs = scheduledMaps.remove();
                LOG.debug(MessageFormat.format("Application {0} starts a " + "launch a mapper ({1}).", appId, container.getId()));
                assignedMaps.put(container.getId(), cs);
                se.getNmMap().get(container.getNodeId()).addNewContainer(container, cs.getLifeTime());
            } else if (!this.scheduledReduces.isEmpty()) {
                ContainerSimulator cs = scheduledReduces.remove();
                LOG.debug(MessageFormat.format("Application {0} starts a " + "launch a reducer ({1}).", appId, container.getId()));
                assignedReduces.put(container.getId(), cs);
                se.getNmMap().get(container.getNodeId()).addNewContainer(container, cs.getLifeTime());
            }
        }
    }
}

Example 17 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class NMSimulator method generateContainerStatusList.

/**
   * catch status of all containers located on current node
   */
private ArrayList<ContainerStatus> generateContainerStatusList() {
    ArrayList<ContainerStatus> csList = new ArrayList<ContainerStatus>();
    // add running containers
    for (ContainerSimulator container : runningContainers.values()) {
        csList.add(newContainerStatus(container.getId(), ContainerState.RUNNING, ContainerExitStatus.SUCCESS));
    }
    synchronized (amContainerList) {
        for (ContainerId cId : amContainerList) {
            csList.add(newContainerStatus(cId, ContainerState.RUNNING, ContainerExitStatus.SUCCESS));
        }
    }
    // add complete containers
    synchronized (completedContainerList) {
        for (ContainerId cId : completedContainerList) {
            LOG.debug(MessageFormat.format("NodeManager {0} completed" + " container ({1}).", node.getNodeID(), cId));
            csList.add(newContainerStatus(cId, ContainerState.COMPLETE, ContainerExitStatus.SUCCESS));
        }
        completedContainerList.clear();
    }
    // released containers
    synchronized (releasedContainerList) {
        for (ContainerId cId : releasedContainerList) {
            LOG.debug(MessageFormat.format("NodeManager {0} released container" + " ({1}).", node.getNodeID(), cId));
            csList.add(newContainerStatus(cId, ContainerState.COMPLETE, ContainerExitStatus.ABORTED));
        }
        releasedContainerList.clear();
    }
    return csList;
}

Example 18 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class SLSCapacityScheduler method updateQueueWithNodeUpdate.

private void updateQueueWithNodeUpdate(NodeUpdateSchedulerEventWrapper eventWrapper) {
    RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode();
    List<UpdatedContainerInfo> containerList = node.getContainerUpdates();
    for (UpdatedContainerInfo info : containerList) {
        for (ContainerStatus status : info.getCompletedContainers()) {
            ContainerId containerId = status.getContainerId();
            SchedulerAppReport app = super.getSchedulerAppInfo(containerId.getApplicationAttemptId());
            if (app == null) {
                // information.
                continue;
            }
            String queue = appQueueMap.get(containerId.getApplicationAttemptId());
            int releasedMemory = 0, releasedVCores = 0;
            if (status.getExitStatus() == ContainerExitStatus.SUCCESS) {
                for (RMContainer rmc : app.getLiveContainers()) {
                    if (rmc.getContainerId() == containerId) {
                        releasedMemory += rmc.getContainer().getResource().getMemorySize();
                        releasedVCores += rmc.getContainer().getResource().getVirtualCores();
                        break;
                    }
                }
            } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) {
                if (preemptionContainerMap.containsKey(containerId)) {
                    Resource preResource = preemptionContainerMap.get(containerId);
                    releasedMemory += preResource.getMemorySize();
                    releasedVCores += preResource.getVirtualCores();
                    preemptionContainerMap.remove(containerId);
                }
            }
            // update queue counters
            updateQueueMetrics(queue, releasedMemory, releasedVCores);
        }
    }
}

Example 19 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class NMSimulator method newContainerStatus.

private ContainerStatus newContainerStatus(ContainerId cId, ContainerState state, int exitState) {
    ContainerStatus cs = Records.newRecord(ContainerStatus.class);
    cs.setContainerId(cId);
    cs.setState(state);
    cs.setExitStatus(exitState);
    return cs;
}

Example 20 with ContainerStatus

use of org.apache.hadoop.yarn.api.records.ContainerStatus in project hadoop by apache.

the class TestClientRMService method getRMApp.

private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler, ApplicationId applicationId3, YarnConfiguration config, String queueName, final long memorySeconds, final long vcoreSeconds, String appNodeLabelExpression, String amNodeLabelExpression) {
    ApplicationSubmissionContext asContext = mock(ApplicationSubmissionContext.class);
    when(asContext.getMaxAppAttempts()).thenReturn(1);
    when(asContext.getNodeLabelExpression()).thenReturn(appNodeLabelExpression);
    when(asContext.getPriority()).thenReturn(Priority.newInstance(0));
    RMAppImpl app = spy(new RMAppImpl(applicationId3, rmContext, config, null, null, queueName, asContext, yarnScheduler, null, System.currentTimeMillis(), "YARN", null, BuilderUtils.newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, Resource.newInstance(1024, 1), 1)) {

        @Override
        public ApplicationReport createAndGetApplicationReport(String clientUserName, boolean allowAccess) {
            ApplicationReport report = super.createAndGetApplicationReport(clientUserName, allowAccess);
            ApplicationResourceUsageReport usageReport = report.getApplicationResourceUsageReport();
            usageReport.setMemorySeconds(memorySeconds);
            usageReport.setVcoreSeconds(vcoreSeconds);
            report.setApplicationResourceUsageReport(usageReport);
            return report;
        }
    });
    app.getAMResourceRequest().setNodeLabelExpression(amNodeLabelExpression);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(123456, 1), 1);
    RMAppAttemptImpl rmAppAttemptImpl = spy(new RMAppAttemptImpl(attemptId, rmContext, yarnScheduler, null, asContext, config, null, app));
    Container container = Container.newInstance(ContainerId.newContainerId(attemptId, 1), null, "", null, null, null);
    RMContainerImpl containerimpl = spy(new RMContainerImpl(container, SchedulerRequestKey.extractFrom(container), attemptId, null, "", rmContext));
    Map<ApplicationAttemptId, RMAppAttempt> attempts = new HashMap<ApplicationAttemptId, RMAppAttempt>();
    attempts.put(attemptId, rmAppAttemptImpl);
    when(app.getCurrentAppAttempt()).thenReturn(rmAppAttemptImpl);
    when(app.getAppAttempts()).thenReturn(attempts);
    when(app.getApplicationPriority()).thenReturn(Priority.newInstance(0));
    when(rmAppAttemptImpl.getMasterContainer()).thenReturn(container);
    ResourceScheduler rs = mock(ResourceScheduler.class);
    when(rmContext.getScheduler()).thenReturn(rs);
    when(rmContext.getScheduler().getRMContainer(any(ContainerId.class))).thenReturn(containerimpl);
    SchedulerAppReport sAppReport = mock(SchedulerAppReport.class);
    when(rmContext.getScheduler().getSchedulerAppInfo(any(ApplicationAttemptId.class))).thenReturn(sAppReport);
    List<RMContainer> rmContainers = new ArrayList<RMContainer>();
    rmContainers.add(containerimpl);
    when(rmContext.getScheduler().getSchedulerAppInfo(attemptId).getLiveContainers()).thenReturn(rmContainers);
    ContainerStatus cs = mock(ContainerStatus.class);
    when(containerimpl.completed()).thenReturn(false);
    when(containerimpl.getDiagnosticsInfo()).thenReturn("N/A");
    when(containerimpl.getContainerExitStatus()).thenReturn(0);
    when(containerimpl.getContainerState()).thenReturn(ContainerState.COMPLETE);
    return app;
}