Examples with RMContainer org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer used on opensource projects

Example 1 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class TestDistributedScheduling method testAMOpportunistic.

/**
   * Check if an AM can ask for opportunistic containers and get them.
   * @throws Exception
   */
@Test
public void testAMOpportunistic() throws Exception {
    // Basic container to request
    Resource capability = Resource.newInstance(1024, 1);
    Priority priority = Priority.newInstance(1);
    // Get the cluster topology
    List<NodeReport> nodeReports = rmClient.getNodeReports(NodeState.RUNNING);
    String node = nodeReports.get(0).getNodeId().getHost();
    String rack = nodeReports.get(0).getRackName();
    String[] nodes = new String[] { node };
    String[] racks = new String[] { rack };
    // Create an AM to request resources
    AMRMClient<AMRMClient.ContainerRequest> amClient = null;
    try {
        amClient = new AMRMClientImpl<AMRMClient.ContainerRequest>(client);
        amClient.init(yarnConf);
        amClient.start();
        amClient.registerApplicationMaster(NetUtils.getHostname(), 1024, "");
        // AM requests an opportunistic container
        ExecutionTypeRequest execTypeRequest = ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true);
        ContainerRequest containerRequest = new AMRMClient.ContainerRequest(capability, nodes, racks, priority, 0, true, null, execTypeRequest);
        amClient.addContainerRequest(containerRequest);
        // Wait until the container is allocated
        ContainerId opportunisticContainerId = null;
        for (int i = 0; i < 10 && opportunisticContainerId == null; i++) {
            AllocateResponse allocResponse = amClient.allocate(0.1f);
            List<Container> allocatedContainers = allocResponse.getAllocatedContainers();
            for (Container allocatedContainer : allocatedContainers) {
                // Check that this is the container we required
                assertEquals(ExecutionType.OPPORTUNISTIC, allocatedContainer.getExecutionType());
                opportunisticContainerId = allocatedContainer.getId();
            }
            sleep(100);
        }
        assertNotNull(opportunisticContainerId);
        // The RM sees the container as OPPORTUNISTIC
        ResourceScheduler scheduler = cluster.getResourceManager().getResourceScheduler();
        RMContainer rmContainer = scheduler.getRMContainer(opportunisticContainerId);
        assertEquals(ExecutionType.OPPORTUNISTIC, rmContainer.getExecutionType());
        // Release the opportunistic container
        amClient.releaseAssignedContainer(opportunisticContainerId);
        // Wait for the release container to appear
        boolean released = false;
        for (int i = 0; i < 10 && !released; i++) {
            AllocateResponse allocResponse = amClient.allocate(0.1f);
            List<ContainerStatus> completedContainers = allocResponse.getCompletedContainersStatuses();
            for (ContainerStatus completedContainer : completedContainers) {
                ContainerId completedContainerId = completedContainer.getContainerId();
                assertEquals(completedContainerId, opportunisticContainerId);
                released = true;
            }
            if (!released) {
                sleep(100);
            }
        }
        assertTrue(released);
        // The RM shouldn't see the container anymore
        rmContainer = scheduler.getRMContainer(opportunisticContainerId);
        assertNull(rmContainer);
        // Clean the AM
        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null);
    } finally {
        if (amClient != null && amClient.getServiceState() == Service.STATE.STARTED) {
            amClient.close();
        }
    }
}

Example 2 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class RMServerUtils method checkSchedContainerChangeRequest.

/**
   * Validate increase/decrease request.
   * <pre>
   * - Throw exception when any other error happens
   * </pre>
   */
public static void checkSchedContainerChangeRequest(SchedContainerChangeRequest request, boolean increase) throws InvalidResourceRequestException {
    RMContext rmContext = request.getRmContext();
    ContainerId containerId = request.getContainerId();
    RMContainer rmContainer = request.getRMContainer();
    Resource targetResource = request.getTargetCapacity();
    // Compare targetResource and original resource
    Resource originalResource = rmContainer.getAllocatedResource();
    // <20G, 8>
    if (increase) {
        if (originalResource.getMemorySize() > targetResource.getMemorySize() || originalResource.getVirtualCores() > targetResource.getVirtualCores()) {
            String msg = "Trying to increase a container, but target resource has some" + " resource < original resource, target=" + targetResource + " original=" + originalResource + " containerId=" + containerId;
            throw new InvalidResourceRequestException(msg);
        }
    } else {
        if (originalResource.getMemorySize() < targetResource.getMemorySize() || originalResource.getVirtualCores() < targetResource.getVirtualCores()) {
            String msg = "Trying to decrease a container, but target resource has " + "some resource > original resource, target=" + targetResource + " original=" + originalResource + " containerId=" + containerId;
            throw new InvalidResourceRequestException(msg);
        }
    }
    // Target resource of the increase request is more than NM can offer
    ResourceScheduler scheduler = rmContext.getScheduler();
    RMNode rmNode = request.getSchedulerNode().getRMNode();
    if (!Resources.fitsIn(scheduler.getResourceCalculator(), scheduler.getClusterResource(), targetResource, rmNode.getTotalCapability())) {
        String msg = "Target resource=" + targetResource + " of containerId=" + containerId + " is more than node's total resource=" + rmNode.getTotalCapability();
        throw new InvalidResourceRequestException(msg);
    }
}

Example 3 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class OpportunisticContainerAllocatorAMService method handleNewContainers.

private void handleNewContainers(List<Container> allocContainers, boolean isRemotelyAllocated) {
    for (Container container : allocContainers) {
        // Create RMContainer
        RMContainer rmContainer = SchedulerUtils.createOpportunisticRmContainer(rmContext, container, isRemotelyAllocated);
        rmContainer.handle(new RMContainerEvent(container.getId(), RMContainerEventType.ACQUIRED));
    }
}

Example 4 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class ClientRMService method signalToContainer.

/**
   * Signal a container.
   * After the request passes some sanity check, it will be delivered
   * to RMNodeImpl so that the next NM heartbeat will pick up the signal request
   */
@SuppressWarnings("unchecked")
@Override
public SignalContainerResponse signalToContainer(SignalContainerRequest request) throws YarnException, IOException {
    ContainerId containerId = request.getContainerId();
    UserGroupInformation callerUGI;
    try {
        callerUGI = UserGroupInformation.getCurrentUser();
    } catch (IOException ie) {
        LOG.info("Error getting UGI ", ie);
        throw RPCUtil.getRemoteException(ie);
    }
    ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
    RMApp application = this.rmContext.getRMApps().get(applicationId);
    if (application == null) {
        RMAuditLogger.logFailure(callerUGI.getUserName(), AuditConstants.SIGNAL_CONTAINER, "UNKNOWN", "ClientRMService", "Trying to signal an absent container", applicationId, containerId, null);
        throw RPCUtil.getRemoteException("Trying to signal an absent container " + containerId);
    }
    if (!checkAccess(callerUGI, application.getUser(), ApplicationAccessType.MODIFY_APP, application)) {
        RMAuditLogger.logFailure(callerUGI.getShortUserName(), AuditConstants.SIGNAL_CONTAINER, "User doesn't have permissions to " + ApplicationAccessType.MODIFY_APP.toString(), "ClientRMService", AuditConstants.UNAUTHORIZED_USER, applicationId);
        throw RPCUtil.getRemoteException(new AccessControlException("User " + callerUGI.getShortUserName() + " cannot perform operation " + ApplicationAccessType.MODIFY_APP.name() + " on " + applicationId));
    }
    RMContainer container = scheduler.getRMContainer(containerId);
    if (container != null) {
        this.rmContext.getDispatcher().getEventHandler().handle(new RMNodeSignalContainerEvent(container.getContainer().getNodeId(), request));
        RMAuditLogger.logSuccess(callerUGI.getShortUserName(), AuditConstants.SIGNAL_CONTAINER, "ClientRMService", applicationId, containerId, null);
    } else {
        RMAuditLogger.logFailure(callerUGI.getUserName(), AuditConstants.SIGNAL_CONTAINER, "UNKNOWN", "ClientRMService", "Trying to signal an absent container", applicationId, containerId, null);
        throw RPCUtil.getRemoteException("Trying to signal an absent container " + containerId);
    }
    return recordFactory.newRecordInstance(SignalContainerResponse.class);
}

Example 5 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class SLSCapacityScheduler method handle.

@Override
public void handle(SchedulerEvent schedulerEvent) {
    // metrics off
    if (!metricsON) {
        super.handle(schedulerEvent);
        return;
    }
    if (!running)
        running = true;
    // metrics on
    Timer.Context handlerTimer = null;
    Timer.Context operationTimer = null;
    NodeUpdateSchedulerEventWrapper eventWrapper;
    try {
        //if (schedulerEvent instanceof NodeUpdateSchedulerEvent) {
        if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE && schedulerEvent instanceof NodeUpdateSchedulerEvent) {
            eventWrapper = new NodeUpdateSchedulerEventWrapper((NodeUpdateSchedulerEvent) schedulerEvent);
            schedulerEvent = eventWrapper;
            updateQueueWithNodeUpdate(eventWrapper);
        } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
            // check if having AM Container, update resource usage information
            AppAttemptRemovedSchedulerEvent appRemoveEvent = (AppAttemptRemovedSchedulerEvent) schedulerEvent;
            ApplicationAttemptId appAttemptId = appRemoveEvent.getApplicationAttemptID();
            String queue = appQueueMap.get(appAttemptId);
            SchedulerAppReport app = super.getSchedulerAppInfo(appAttemptId);
            if (!app.getLiveContainers().isEmpty()) {
                // have 0 or 1
                // should have one container which is AM container
                RMContainer rmc = app.getLiveContainers().iterator().next();
                updateQueueMetrics(queue, rmc.getContainer().getResource().getMemorySize(), rmc.getContainer().getResource().getVirtualCores());
            }
        }
        handlerTimer = schedulerHandleTimer.time();
        operationTimer = schedulerHandleTimerMap.get(schedulerEvent.getType()).time();
        super.handle(schedulerEvent);
    } finally {
        if (handlerTimer != null)
            handlerTimer.stop();
        if (operationTimer != null)
            operationTimer.stop();
        schedulerHandleCounter.inc();
        schedulerHandleCounterMap.get(schedulerEvent.getType()).inc();
        if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
            SLSRunner.decreaseRemainingApps();
            AppAttemptRemovedSchedulerEvent appRemoveEvent = (AppAttemptRemovedSchedulerEvent) schedulerEvent;
            ApplicationAttemptId appAttemptId = appRemoveEvent.getApplicationAttemptID();
            appQueueMap.remove(appRemoveEvent.getApplicationAttemptID());
        } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_ADDED && schedulerEvent instanceof AppAttemptAddedSchedulerEvent) {
            AppAttemptAddedSchedulerEvent appAddEvent = (AppAttemptAddedSchedulerEvent) schedulerEvent;
            SchedulerApplication app = applications.get(appAddEvent.getApplicationAttemptId().getApplicationId());
            appQueueMap.put(appAddEvent.getApplicationAttemptId(), app.getQueue().getQueueName());
        }
    }
}