Examples with RMContainer org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer used on opensource projects

Example 61 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class SLSCapacityScheduler method handle.

@Override
public void handle(SchedulerEvent schedulerEvent) {
    // metrics off
    if (!metricsON) {
        super.handle(schedulerEvent);
        return;
    }
    if (!running)
        running = true;
    // metrics on
    Timer.Context handlerTimer = null;
    Timer.Context operationTimer = null;
    NodeUpdateSchedulerEventWrapper eventWrapper;
    try {
        //if (schedulerEvent instanceof NodeUpdateSchedulerEvent) {
        if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE && schedulerEvent instanceof NodeUpdateSchedulerEvent) {
            eventWrapper = new NodeUpdateSchedulerEventWrapper((NodeUpdateSchedulerEvent) schedulerEvent);
            schedulerEvent = eventWrapper;
            updateQueueWithNodeUpdate(eventWrapper);
        } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
            // check if having AM Container, update resource usage information
            AppAttemptRemovedSchedulerEvent appRemoveEvent = (AppAttemptRemovedSchedulerEvent) schedulerEvent;
            ApplicationAttemptId appAttemptId = appRemoveEvent.getApplicationAttemptID();
            String queue = appQueueMap.get(appAttemptId);
            SchedulerAppReport app = super.getSchedulerAppInfo(appAttemptId);
            if (!app.getLiveContainers().isEmpty()) {
                // have 0 or 1
                // should have one container which is AM container
                RMContainer rmc = app.getLiveContainers().iterator().next();
                updateQueueMetrics(queue, rmc.getContainer().getResource().getMemorySize(), rmc.getContainer().getResource().getVirtualCores());
            }
        }
        handlerTimer = schedulerHandleTimer.time();
        operationTimer = schedulerHandleTimerMap.get(schedulerEvent.getType()).time();
        super.handle(schedulerEvent);
    } finally {
        if (handlerTimer != null)
            handlerTimer.stop();
        if (operationTimer != null)
            operationTimer.stop();
        schedulerHandleCounter.inc();
        schedulerHandleCounterMap.get(schedulerEvent.getType()).inc();
        if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
            SLSRunner.decreaseRemainingApps();
            AppAttemptRemovedSchedulerEvent appRemoveEvent = (AppAttemptRemovedSchedulerEvent) schedulerEvent;
            ApplicationAttemptId appAttemptId = appRemoveEvent.getApplicationAttemptID();
            appQueueMap.remove(appRemoveEvent.getApplicationAttemptID());
        } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_ADDED && schedulerEvent instanceof AppAttemptAddedSchedulerEvent) {
            AppAttemptAddedSchedulerEvent appAddEvent = (AppAttemptAddedSchedulerEvent) schedulerEvent;
            SchedulerApplication app = applications.get(appAddEvent.getApplicationAttemptId().getApplicationId());
            appQueueMap.put(appAddEvent.getApplicationAttemptId(), app.getQueue().getQueueName());
        }
    }
}

Example 62 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class SLSCapacityScheduler method updateQueueWithNodeUpdate.

private void updateQueueWithNodeUpdate(NodeUpdateSchedulerEventWrapper eventWrapper) {
    RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode();
    List<UpdatedContainerInfo> containerList = node.getContainerUpdates();
    for (UpdatedContainerInfo info : containerList) {
        for (ContainerStatus status : info.getCompletedContainers()) {
            ContainerId containerId = status.getContainerId();
            SchedulerAppReport app = super.getSchedulerAppInfo(containerId.getApplicationAttemptId());
            if (app == null) {
                // information.
                continue;
            }
            String queue = appQueueMap.get(containerId.getApplicationAttemptId());
            int releasedMemory = 0, releasedVCores = 0;
            if (status.getExitStatus() == ContainerExitStatus.SUCCESS) {
                for (RMContainer rmc : app.getLiveContainers()) {
                    if (rmc.getContainerId() == containerId) {
                        releasedMemory += rmc.getContainer().getResource().getMemorySize();
                        releasedVCores += rmc.getContainer().getResource().getVirtualCores();
                        break;
                    }
                }
            } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) {
                if (preemptionContainerMap.containsKey(containerId)) {
                    Resource preResource = preemptionContainerMap.get(containerId);
                    releasedMemory += preResource.getMemorySize();
                    releasedVCores += preResource.getVirtualCores();
                    preemptionContainerMap.remove(containerId);
                }
            }
            // update queue counters
            updateQueueMetrics(queue, releasedMemory, releasedVCores);
        }
    }
}

Example 63 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class ResourceSchedulerWrapper method handle.

@Override
public void handle(SchedulerEvent schedulerEvent) {
    // metrics off
    if (!metricsON) {
        scheduler.handle(schedulerEvent);
        return;
    }
    if (!running)
        running = true;
    // metrics on
    Timer.Context handlerTimer = null;
    Timer.Context operationTimer = null;
    NodeUpdateSchedulerEventWrapper eventWrapper;
    try {
        //if (schedulerEvent instanceof NodeUpdateSchedulerEvent) {
        if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE && schedulerEvent instanceof NodeUpdateSchedulerEvent) {
            eventWrapper = new NodeUpdateSchedulerEventWrapper((NodeUpdateSchedulerEvent) schedulerEvent);
            schedulerEvent = eventWrapper;
            updateQueueWithNodeUpdate(eventWrapper);
        } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
            // check if having AM Container, update resource usage information
            AppAttemptRemovedSchedulerEvent appRemoveEvent = (AppAttemptRemovedSchedulerEvent) schedulerEvent;
            ApplicationAttemptId appAttemptId = appRemoveEvent.getApplicationAttemptID();
            String queue = appQueueMap.get(appAttemptId.getApplicationId());
            SchedulerAppReport app = scheduler.getSchedulerAppInfo(appAttemptId);
            if (!app.getLiveContainers().isEmpty()) {
                // have 0 or 1
                // should have one container which is AM container
                RMContainer rmc = app.getLiveContainers().iterator().next();
                updateQueueMetrics(queue, rmc.getContainer().getResource().getMemorySize(), rmc.getContainer().getResource().getVirtualCores());
            }
        }
        handlerTimer = schedulerHandleTimer.time();
        operationTimer = schedulerHandleTimerMap.get(schedulerEvent.getType()).time();
        scheduler.handle(schedulerEvent);
    } finally {
        if (handlerTimer != null)
            handlerTimer.stop();
        if (operationTimer != null)
            operationTimer.stop();
        schedulerHandleCounter.inc();
        schedulerHandleCounterMap.get(schedulerEvent.getType()).inc();
        if (schedulerEvent.getType() == SchedulerEventType.APP_REMOVED && schedulerEvent instanceof AppRemovedSchedulerEvent) {
            SLSRunner.decreaseRemainingApps();
            AppRemovedSchedulerEvent appRemoveEvent = (AppRemovedSchedulerEvent) schedulerEvent;
            appQueueMap.remove(appRemoveEvent.getApplicationID());
        } else if (schedulerEvent.getType() == SchedulerEventType.APP_ADDED && schedulerEvent instanceof AppAddedSchedulerEvent) {
            AppAddedSchedulerEvent appAddEvent = (AppAddedSchedulerEvent) schedulerEvent;
            String queueName = appAddEvent.getQueue();
            appQueueMap.put(appAddEvent.getApplicationId(), queueName);
        }
    }
}

Example 64 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class FairScheduler method completedContainerInternal.

/**
   * Clean up a completed container.
   */
@Override
protected void completedContainerInternal(RMContainer rmContainer, ContainerStatus containerStatus, RMContainerEventType event) {
    try {
        writeLock.lock();
        Container container = rmContainer.getContainer();
        // Get the application for the finished container
        FSAppAttempt application = getCurrentAttemptForContainer(container.getId());
        ApplicationId appId = container.getId().getApplicationAttemptId().getApplicationId();
        if (application == null) {
            LOG.info("Container " + container + " of" + " finished application " + appId + " completed with event " + event);
            return;
        }
        // Get the node on which the container was allocated
        FSSchedulerNode node = getFSSchedulerNode(container.getNodeId());
        if (rmContainer.getState() == RMContainerState.RESERVED) {
            application.unreserve(rmContainer.getReservedSchedulerKey(), node);
        } else {
            application.containerCompleted(rmContainer, containerStatus, event);
            node.releaseContainer(rmContainer.getContainerId(), false);
            updateRootQueueMetrics();
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("Application attempt " + application.getApplicationAttemptId() + " released container " + container.getId() + " on node: " + node + " with event: " + event);
        }
    } finally {
        writeLock.unlock();
    }
}

Example 65 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class FiCaSchedulerNode method markContainerToKillable.

// According to decisions from preemption policy, mark the container to killable
public synchronized void markContainerToKillable(ContainerId containerId) {
    RMContainer c = getContainer(containerId);
    if (c != null && !killableContainers.containsKey(containerId)) {
        killableContainers.put(containerId, c);
        Resources.addTo(totalKillableResources, c.getAllocatedResource());
    }
}