Examples with RMContainerImpl org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl used on opensource projects

Example 11 with RMContainerImpl

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl in project hadoop by apache.

the class TestReservations method testGetAppToUnreserve.

@Test
public void testGetAppToUnreserve() throws Exception {
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    setup(csConf);
    final String user_0 = "user_0";
    final ApplicationAttemptId appAttemptId_0 = TestUtils.getMockApplicationAttemptId(0, 0);
    LeafQueue a = stubLeafQueue((LeafQueue) queues.get(A));
    FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, mock(ActiveUsersManager.class), spyRMContext);
    String host_0 = "host_0";
    FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0, 8 * GB);
    String host_1 = "host_1";
    FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0, 8 * GB);
    Resource clusterResource = Resources.createResource(2 * 8 * GB);
    // Setup resource-requests
    Priority p = TestUtils.createMockPriority(5);
    SchedulerRequestKey priorityMap = toSchedulerKey(p);
    Resource capability = Resources.createResource(2 * GB, 0);
    RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
    SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
    RMContext rmContext = mock(RMContext.class);
    ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);
    DrainDispatcher drainDispatcher = new DrainDispatcher();
    when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
    when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
    when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
    when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
    when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(app_0.getApplicationId(), 1);
    ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
    Container container = TestUtils.getMockContainer(containerId, node_1.getNodeID(), Resources.createResource(2 * GB), priorityMap.getPriority());
    RMContainer rmContainer = new RMContainerImpl(container, SchedulerRequestKey.extractFrom(container), appAttemptId, node_1.getNodeID(), "user", rmContext);
    Container container_1 = TestUtils.getMockContainer(containerId, node_0.getNodeID(), Resources.createResource(1 * GB), priorityMap.getPriority());
    RMContainer rmContainer_1 = new RMContainerImpl(container_1, SchedulerRequestKey.extractFrom(container_1), appAttemptId, node_0.getNodeID(), "user", rmContext);
    // no reserved containers
    NodeId unreserveId = app_0.getNodeIdToUnreserve(priorityMap, capability, cs.getResourceCalculator(), clusterResource);
    assertEquals(null, unreserveId);
    // no reserved containers - reserve then unreserve
    app_0.reserve(node_0, priorityMap, rmContainer_1, container_1);
    app_0.unreserve(priorityMap, node_0, rmContainer_1);
    unreserveId = app_0.getNodeIdToUnreserve(priorityMap, capability, cs.getResourceCalculator(), clusterResource);
    assertEquals(null, unreserveId);
    // no container large enough is reserved
    app_0.reserve(node_0, priorityMap, rmContainer_1, container_1);
    unreserveId = app_0.getNodeIdToUnreserve(priorityMap, capability, cs.getResourceCalculator(), clusterResource);
    assertEquals(null, unreserveId);
    // reserve one that is now large enough
    app_0.reserve(node_1, priorityMap, rmContainer, container);
    unreserveId = app_0.getNodeIdToUnreserve(priorityMap, capability, cs.getResourceCalculator(), clusterResource);
    assertEquals(node_1.getNodeID(), unreserveId);
}

Example 12 with RMContainerImpl

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl in project hadoop by apache.

the class AbstractYarnScheduler method recoverAndCreateContainer.

private RMContainer recoverAndCreateContainer(NMContainerStatus status, RMNode node) {
    Container container = Container.newInstance(status.getContainerId(), node.getNodeID(), node.getHttpAddress(), status.getAllocatedResource(), status.getPriority(), null);
    container.setVersion(status.getVersion());
    ApplicationAttemptId attemptId = container.getId().getApplicationAttemptId();
    RMContainer rmContainer = new RMContainerImpl(container, SchedulerRequestKey.extractFrom(container), attemptId, node.getNodeID(), applications.get(attemptId.getApplicationId()).getUser(), rmContext, status.getCreationTime(), status.getNodeLabelExpression());
    return rmContainer;
}

Example 13 with RMContainerImpl

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl in project hadoop by apache.

the class ContainerUpdateContext method swapContainer.

/**
   * Swaps the existing RMContainer's and the temp RMContainers internal
   * container references after adjusting the resources in each.
   * @param tempRMContainer Temp RMContainer.
   * @param existingRMContainer Existing RMContainer.
   * @param updateType Update Type.
   * @return Existing RMContainer after swapping the container references.
   */
public RMContainer swapContainer(RMContainer tempRMContainer, RMContainer existingRMContainer, ContainerUpdateType updateType) {
    ContainerId matchedContainerId = existingRMContainer.getContainerId();
    // Swap updated container with the existing container
    Container tempContainer = tempRMContainer.getContainer();
    Resource updatedResource = createUpdatedResource(tempContainer, existingRMContainer.getContainer(), updateType);
    Resource resourceToRelease = createResourceToRelease(existingRMContainer.getContainer(), updateType);
    Container newContainer = Container.newInstance(matchedContainerId, existingRMContainer.getContainer().getNodeId(), existingRMContainer.getContainer().getNodeHttpAddress(), updatedResource, existingRMContainer.getContainer().getPriority(), null, tempContainer.getExecutionType());
    newContainer.setAllocationRequestId(existingRMContainer.getContainer().getAllocationRequestId());
    newContainer.setVersion(existingRMContainer.getContainer().getVersion());
    tempRMContainer.getContainer().setResource(resourceToRelease);
    tempRMContainer.getContainer().setExecutionType(existingRMContainer.getContainer().getExecutionType());
    ((RMContainerImpl) existingRMContainer).setContainer(newContainer);
    return existingRMContainer;
}

Example 14 with RMContainerImpl

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl in project hadoop by apache.

the class FifoAppAttempt method allocate.

public RMContainer allocate(NodeType type, FiCaSchedulerNode node, SchedulerRequestKey schedulerKey, Container container) {
    try {
        writeLock.lock();
        if (isStopped) {
            return null;
        }
        // request without locking the scheduler, hence we need to check
        if (getOutstandingAsksCount(schedulerKey) <= 0) {
            return null;
        }
        // Create RMContainer
        RMContainer rmContainer = new RMContainerImpl(container, schedulerKey, this.getApplicationAttemptId(), node.getNodeID(), appSchedulingInfo.getUser(), this.rmContext, node.getPartition());
        ((RMContainerImpl) rmContainer).setQueueName(this.getQueueName());
        updateAMContainerDiagnostics(AMState.ASSIGNED, null);
        // Add it to allContainers list.
        addToNewlyAllocatedContainers(node, rmContainer);
        ContainerId containerId = container.getId();
        liveContainers.put(containerId, rmContainer);
        // Update consumption and track allocations
        List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(type, node, schedulerKey, container);
        attemptResourceUsage.incUsed(node.getPartition(), container.getResource());
        // Update resource requests related to "request" and store in RMContainer
        ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList);
        // Inform the container
        rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START));
        if (LOG.isDebugEnabled()) {
            LOG.debug("allocate: applicationAttemptId=" + containerId.getApplicationAttemptId() + " container=" + containerId + " host=" + container.getNodeId().getHost() + " type=" + type);
        }
        RMAuditLogger.logSuccess(getUser(), RMAuditLogger.AuditConstants.ALLOC_CONTAINER, "SchedulerApp", getApplicationId(), containerId, container.getResource());
        return rmContainer;
    } finally {
        writeLock.unlock();
    }
}

Example 15 with RMContainerImpl

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl in project hadoop by apache.

the class FiCaSchedulerApp method apply.

public void apply(Resource cluster, ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request) {
    boolean reReservation = false;
    try {
        writeLock.lock();
        // If we allocated something
        if (request.anythingAllocatedOrReserved()) {
            ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> allocation = request.getFirstAllocatedOrReservedContainer();
            SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> schedulerContainer = allocation.getAllocatedOrReservedContainer();
            RMContainer rmContainer = schedulerContainer.getRmContainer();
            reReservation = (!schedulerContainer.isAllocated()) && (rmContainer.getState() == RMContainerState.RESERVED);
            // Or re-reservation
            if (rmContainer.getContainer().getId() == null) {
                rmContainer.setContainerId(BuilderUtils.newContainerId(getApplicationAttemptId(), getNewContainerId()));
            }
            ContainerId containerId = rmContainer.getContainerId();
            if (schedulerContainer.isAllocated()) {
                // Unreserve it first
                if (allocation.getAllocateFromReservedContainer() != null) {
                    RMContainer reservedContainer = allocation.getAllocateFromReservedContainer().getRmContainer();
                    // Handling container allocation
                    // Did we previously reserve containers at this 'priority'?
                    unreserve(schedulerContainer.getSchedulerRequestKey(), schedulerContainer.getSchedulerNode(), reservedContainer);
                }
                // Allocate a new container
                addToNewlyAllocatedContainers(schedulerContainer.getSchedulerNode(), rmContainer);
                liveContainers.put(containerId, rmContainer);
                // Deduct pending resource requests
                List<ResourceRequest> requests = appSchedulingInfo.allocate(allocation.getAllocationLocalityType(), schedulerContainer.getSchedulerNode(), schedulerContainer.getSchedulerRequestKey(), schedulerContainer.getRmContainer().getContainer());
                ((RMContainerImpl) rmContainer).setResourceRequests(requests);
                attemptResourceUsage.incUsed(schedulerContainer.getNodePartition(), allocation.getAllocatedOrReservedResource());
                rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START));
                // Inform the node
                schedulerContainer.getSchedulerNode().allocateContainer(rmContainer);
                // update locality statistics,
                incNumAllocatedContainers(allocation.getAllocationLocalityType(), allocation.getRequestLocalityType());
                if (LOG.isDebugEnabled()) {
                    LOG.debug("allocate: applicationAttemptId=" + containerId.getApplicationAttemptId() + " container=" + containerId + " host=" + rmContainer.getAllocatedNode().getHost() + " type=" + allocation.getAllocationLocalityType());
                }
                RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER, "SchedulerApp", getApplicationId(), containerId, allocation.getAllocatedOrReservedResource());
            } else {
                // If the rmContainer's state is already updated to RESERVED, this is
                // a reReservation
                reserve(schedulerContainer.getSchedulerRequestKey(), schedulerContainer.getSchedulerNode(), schedulerContainer.getRmContainer(), schedulerContainer.getRmContainer().getContainer(), reReservation);
            }
        }
    } finally {
        writeLock.unlock();
    }
    // Don't bother CS leaf queue if it is a re-reservation
    if (!reReservation) {
        getCSLeafQueue().apply(cluster, request);
    }
}