Examples with Container org.apache.hadoop.yarn.api.records.Container used on opensource projects

Example 81 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class MRAMSimulator method processResponseQueue.

@Override
@SuppressWarnings("unchecked")
protected void processResponseQueue() throws Exception {
    while (!responseQueue.isEmpty()) {
        AllocateResponse response = responseQueue.take();
        // check completed containers
        if (!response.getCompletedContainersStatuses().isEmpty()) {
            for (ContainerStatus cs : response.getCompletedContainersStatuses()) {
                ContainerId containerId = cs.getContainerId();
                if (cs.getExitStatus() == ContainerExitStatus.SUCCESS) {
                    if (assignedMaps.containsKey(containerId)) {
                        LOG.debug(MessageFormat.format("Application {0} has one" + "mapper finished ({1}).", appId, containerId));
                        assignedMaps.remove(containerId);
                        mapFinished++;
                        finishedContainers++;
                    } else if (assignedReduces.containsKey(containerId)) {
                        LOG.debug(MessageFormat.format("Application {0} has one" + "reducer finished ({1}).", appId, containerId));
                        assignedReduces.remove(containerId);
                        reduceFinished++;
                        finishedContainers++;
                    } else if (amContainer.getId().equals(containerId)) {
                        // am container released event
                        isFinished = true;
                        LOG.info(MessageFormat.format("Application {0} goes to " + "finish.", appId));
                    }
                    if (mapFinished >= mapTotal && reduceFinished >= reduceTotal) {
                        lastStep();
                    }
                } else {
                    // container to be killed
                    if (assignedMaps.containsKey(containerId)) {
                        LOG.debug(MessageFormat.format("Application {0} has one " + "mapper killed ({1}).", appId, containerId));
                        pendingFailedMaps.add(assignedMaps.remove(containerId));
                    } else if (assignedReduces.containsKey(containerId)) {
                        LOG.debug(MessageFormat.format("Application {0} has one " + "reducer killed ({1}).", appId, containerId));
                        pendingFailedReduces.add(assignedReduces.remove(containerId));
                    } else if (amContainer.getId().equals(containerId)) {
                        LOG.info(MessageFormat.format("Application {0}'s AM is " + "going to be killed. Waiting for rescheduling...", appId));
                    }
                }
            }
        }
        // check finished
        if (isAMContainerRunning && (mapFinished >= mapTotal) && (reduceFinished >= reduceTotal)) {
            isAMContainerRunning = false;
            LOG.debug(MessageFormat.format("Application {0} sends out event " + "to clean up its AM container.", appId));
            isFinished = true;
            break;
        }
        // check allocated containers
        for (Container container : response.getAllocatedContainers()) {
            if (!scheduledMaps.isEmpty()) {
                ContainerSimulator cs = scheduledMaps.remove();
                LOG.debug(MessageFormat.format("Application {0} starts a " + "launch a mapper ({1}).", appId, container.getId()));
                assignedMaps.put(container.getId(), cs);
                se.getNmMap().get(container.getNodeId()).addNewContainer(container, cs.getLifeTime());
            } else if (!this.scheduledReduces.isEmpty()) {
                ContainerSimulator cs = scheduledReduces.remove();
                LOG.debug(MessageFormat.format("Application {0} starts a " + "launch a reducer ({1}).", appId, container.getId()));
                assignedReduces.put(container.getId(), cs);
                se.getNmMap().get(container.getNodeId()).addNewContainer(container, cs.getLifeTime());
            }
        }
    }
}

Example 82 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class RegisterApplicationMasterResponsePBImpl method initContainersPreviousAttemptList.

private void initContainersPreviousAttemptList() {
    RegisterApplicationMasterResponseProtoOrBuilder p = viaProto ? proto : builder;
    List<ContainerProto> list = p.getContainersFromPreviousAttemptsList();
    containersFromPreviousAttempts = new ArrayList<Container>();
    for (ContainerProto c : list) {
        containersFromPreviousAttempts.add(convertFromProtoFormat(c));
    }
}

Example 83 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class RegisterApplicationMasterResponsePBImpl method addContainersFromPreviousAttemptToProto.

private void addContainersFromPreviousAttemptToProto() {
    maybeInitBuilder();
    builder.clearContainersFromPreviousAttempts();
    List<ContainerProto> list = new ArrayList<ContainerProto>();
    for (Container c : containersFromPreviousAttempts) {
        list.add(convertToProtoFormat(c));
    }
    builder.addAllContainersFromPreviousAttempts(list);
}

Example 84 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class FairScheduler method completedContainerInternal.

/**
   * Clean up a completed container.
   */
@Override
protected void completedContainerInternal(RMContainer rmContainer, ContainerStatus containerStatus, RMContainerEventType event) {
    try {
        writeLock.lock();
        Container container = rmContainer.getContainer();
        // Get the application for the finished container
        FSAppAttempt application = getCurrentAttemptForContainer(container.getId());
        ApplicationId appId = container.getId().getApplicationAttemptId().getApplicationId();
        if (application == null) {
            LOG.info("Container " + container + " of" + " finished application " + appId + " completed with event " + event);
            return;
        }
        // Get the node on which the container was allocated
        FSSchedulerNode node = getFSSchedulerNode(container.getNodeId());
        if (rmContainer.getState() == RMContainerState.RESERVED) {
            application.unreserve(rmContainer.getReservedSchedulerKey(), node);
        } else {
            application.containerCompleted(rmContainer, containerStatus, event);
            node.releaseContainer(rmContainer.getContainerId(), false);
            updateRootQueueMetrics();
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("Application attempt " + application.getApplicationAttemptId() + " released container " + container.getId() + " on node: " + node + " with event: " + event);
        }
    } finally {
        writeLock.unlock();
    }
}

Example 85 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class FSAppAttempt method allocate.

public RMContainer allocate(NodeType type, FSSchedulerNode node, SchedulerRequestKey schedulerKey, PendingAsk pendingAsk, Container reservedContainer) {
    RMContainer rmContainer;
    Container container;
    try {
        writeLock.lock();
        // Update allowed locality level
        NodeType allowed = allowedLocalityLevel.get(schedulerKey);
        if (allowed != null) {
            if (allowed.equals(NodeType.OFF_SWITCH) && (type.equals(NodeType.NODE_LOCAL) || type.equals(NodeType.RACK_LOCAL))) {
                this.resetAllowedLocalityLevel(schedulerKey, type);
            } else if (allowed.equals(NodeType.RACK_LOCAL) && type.equals(NodeType.NODE_LOCAL)) {
                this.resetAllowedLocalityLevel(schedulerKey, type);
            }
        }
        // request without locking the scheduler, hence we need to check
        if (getOutstandingAsksCount(schedulerKey) <= 0) {
            return null;
        }
        container = reservedContainer;
        if (container == null) {
            container = createContainer(node, pendingAsk.getPerAllocationResource(), schedulerKey);
        }
        // Create RMContainer
        rmContainer = new RMContainerImpl(container, schedulerKey, getApplicationAttemptId(), node.getNodeID(), appSchedulingInfo.getUser(), rmContext);
        ((RMContainerImpl) rmContainer).setQueueName(this.getQueueName());
        // Add it to allContainers list.
        addToNewlyAllocatedContainers(node, rmContainer);
        liveContainers.put(container.getId(), rmContainer);
        // Update consumption and track allocations
        List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(type, node, schedulerKey, container);
        this.attemptResourceUsage.incUsed(container.getResource());
        // Update resource requests related to "request" and store in RMContainer
        ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList);
        // Inform the container
        rmContainer.handle(new RMContainerEvent(container.getId(), RMContainerEventType.START));
        if (LOG.isDebugEnabled()) {
            LOG.debug("allocate: applicationAttemptId=" + container.getId().getApplicationAttemptId() + " container=" + container.getId() + " host=" + container.getNodeId().getHost() + " type=" + type);
        }
        RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER, "SchedulerApp", getApplicationId(), container.getId(), container.getResource());
    } finally {
        writeLock.unlock();
    }
    return rmContainer;
}