Examples with RMContainerEvent org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent used on opensource projects

Example 6 with RMContainerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent in project hadoop by apache.

the class FiCaSchedulerApp method apply.

public void apply(Resource cluster, ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request) {
    boolean reReservation = false;
    try {
        writeLock.lock();
        // If we allocated something
        if (request.anythingAllocatedOrReserved()) {
            ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> allocation = request.getFirstAllocatedOrReservedContainer();
            SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode> schedulerContainer = allocation.getAllocatedOrReservedContainer();
            RMContainer rmContainer = schedulerContainer.getRmContainer();
            reReservation = (!schedulerContainer.isAllocated()) && (rmContainer.getState() == RMContainerState.RESERVED);
            // Or re-reservation
            if (rmContainer.getContainer().getId() == null) {
                rmContainer.setContainerId(BuilderUtils.newContainerId(getApplicationAttemptId(), getNewContainerId()));
            }
            ContainerId containerId = rmContainer.getContainerId();
            if (schedulerContainer.isAllocated()) {
                // Unreserve it first
                if (allocation.getAllocateFromReservedContainer() != null) {
                    RMContainer reservedContainer = allocation.getAllocateFromReservedContainer().getRmContainer();
                    // Handling container allocation
                    // Did we previously reserve containers at this 'priority'?
                    unreserve(schedulerContainer.getSchedulerRequestKey(), schedulerContainer.getSchedulerNode(), reservedContainer);
                }
                // Allocate a new container
                addToNewlyAllocatedContainers(schedulerContainer.getSchedulerNode(), rmContainer);
                liveContainers.put(containerId, rmContainer);
                // Deduct pending resource requests
                List<ResourceRequest> requests = appSchedulingInfo.allocate(allocation.getAllocationLocalityType(), schedulerContainer.getSchedulerNode(), schedulerContainer.getSchedulerRequestKey(), schedulerContainer.getRmContainer().getContainer());
                ((RMContainerImpl) rmContainer).setResourceRequests(requests);
                attemptResourceUsage.incUsed(schedulerContainer.getNodePartition(), allocation.getAllocatedOrReservedResource());
                rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START));
                // Inform the node
                schedulerContainer.getSchedulerNode().allocateContainer(rmContainer);
                // update locality statistics,
                incNumAllocatedContainers(allocation.getAllocationLocalityType(), allocation.getRequestLocalityType());
                if (LOG.isDebugEnabled()) {
                    LOG.debug("allocate: applicationAttemptId=" + containerId.getApplicationAttemptId() + " container=" + containerId + " host=" + rmContainer.getAllocatedNode().getHost() + " type=" + allocation.getAllocationLocalityType());
                }
                RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER, "SchedulerApp", getApplicationId(), containerId, allocation.getAllocatedOrReservedResource());
            } else {
                // If the rmContainer's state is already updated to RESERVED, this is
                // a reReservation
                reserve(schedulerContainer.getSchedulerRequestKey(), schedulerContainer.getSchedulerNode(), schedulerContainer.getRmContainer(), schedulerContainer.getRmContainer().getContainer(), reReservation);
            }
        }
    } finally {
        writeLock.unlock();
    }
    // Don't bother CS leaf queue if it is a re-reservation
    if (!reReservation) {
        getCSLeafQueue().apply(cluster, request);
    }
}

Example 7 with RMContainerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent in project hadoop by apache.

the class TestApplicationMasterService method sentRMContainerLaunched.

private void sentRMContainerLaunched(MockRM rm, ContainerId containerId) {
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    RMContainer rmContainer = cs.getRMContainer(containerId);
    if (rmContainer != null) {
        rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED));
    } else {
        Assert.fail("Cannot find RMContainer");
    }
}

Example 8 with RMContainerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent in project hadoop by apache.

the class TestCapacityScheduler method testAllocateDoesNotBlockOnSchedulerLock.

@Test(timeout = 30000)
public void testAllocateDoesNotBlockOnSchedulerLock() throws Exception {
    final YarnConfiguration conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MyContainerManager containerManager = new MyContainerManager();
    final MockRMWithAMS rm = new MockRMWithAMS(conf, containerManager);
    rm.start();
    MockNM nm1 = rm.registerNode("localhost:1234", 5120);
    Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>(2);
    acls.put(ApplicationAccessType.VIEW_APP, "*");
    RMApp app = rm.submitApp(1024, "appname", "appuser", acls);
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt = app.getCurrentAppAttempt();
    ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
    int msecToWait = 10000;
    int msecToSleep = 100;
    while (attempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED && msecToWait > 0) {
        LOG.info("Waiting for AppAttempt to reach LAUNCHED state. " + "Current state is " + attempt.getAppAttemptState());
        Thread.sleep(msecToSleep);
        msecToWait -= msecToSleep;
    }
    Assert.assertEquals(attempt.getAppAttemptState(), RMAppAttemptState.LAUNCHED);
    // Create a client to the RM.
    final YarnRPC rpc = YarnRPC.create(conf);
    UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
    Credentials credentials = containerManager.getContainerCredentials();
    final InetSocketAddress rmBindAddress = rm.getApplicationMasterService().getBindAddress();
    Token<? extends TokenIdentifier> amRMToken = MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress, credentials.getAllTokens());
    currentUser.addToken(amRMToken);
    ApplicationMasterProtocol client = currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

        @Override
        public ApplicationMasterProtocol run() {
            return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rmBindAddress, conf);
        }
    });
    RegisterApplicationMasterRequest request = RegisterApplicationMasterRequest.newInstance("localhost", 12345, "");
    client.registerApplicationMaster(request);
    // Allocate a container
    List<ResourceRequest> asks = Collections.singletonList(ResourceRequest.newInstance(Priority.newInstance(1), "*", Resources.createResource(2 * GB), 1));
    AllocateRequest allocateRequest = AllocateRequest.newInstance(0, 0.0f, asks, null, null);
    client.allocate(allocateRequest);
    // Make sure the container is allocated in RM
    nm1.nodeHeartbeat(true);
    ContainerId containerId2 = ContainerId.newContainerId(applicationAttemptId, 2);
    Assert.assertTrue(rm.waitForState(nm1, containerId2, RMContainerState.ALLOCATED));
    // Acquire the container
    allocateRequest = AllocateRequest.newInstance(1, 0.0f, null, null, null);
    client.allocate(allocateRequest);
    // Launch the container
    final CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    RMContainer rmContainer = cs.getRMContainer(containerId2);
    rmContainer.handle(new RMContainerEvent(containerId2, RMContainerEventType.LAUNCHED));
    // grab the scheduler lock from another thread
    // and verify an allocate call in this thread doesn't block on it
    final CyclicBarrier barrier = new CyclicBarrier(2);
    Thread otherThread = new Thread(new Runnable() {

        @Override
        public void run() {
            synchronized (cs) {
                try {
                    barrier.await();
                    barrier.await();
                } catch (InterruptedException | BrokenBarrierException e) {
                    e.printStackTrace();
                }
            }
        }
    });
    otherThread.start();
    barrier.await();
    List<ContainerId> release = Collections.singletonList(containerId2);
    allocateRequest = AllocateRequest.newInstance(2, 0.0f, null, release, null);
    client.allocate(allocateRequest);
    barrier.await();
    otherThread.join();
    rm.stop();
}

Example 9 with RMContainerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent in project hadoop by apache.

the class SchedulerApplicationAttempt method updateContainerAndNMToken.

private Container updateContainerAndNMToken(RMContainer rmContainer, ContainerUpdateType updateType) {
    Container container = rmContainer.getContainer();
    ContainerType containerType = ContainerType.TASK;
    if (updateType != null) {
        container.setVersion(container.getVersion() + 1);
    }
    // itself is the master container.
    if (isWaitingForAMContainer()) {
        containerType = ContainerType.APPLICATION_MASTER;
    }
    try {
        // create container token and NMToken altogether.
        container.setContainerToken(rmContext.getContainerTokenSecretManager().createContainerToken(container.getId(), container.getVersion(), container.getNodeId(), getUser(), container.getResource(), container.getPriority(), rmContainer.getCreationTime(), this.logAggregationContext, rmContainer.getNodeLabelExpression(), containerType));
        updateNMToken(container);
    } catch (IllegalArgumentException e) {
        // DNS might be down, skip returning this container.
        LOG.error("Error trying to assign container token and NM token to" + " an updated container " + container.getId(), e);
        return null;
    }
    if (updateType == null || ContainerUpdateType.PROMOTE_EXECUTION_TYPE == updateType || ContainerUpdateType.DEMOTE_EXECUTION_TYPE == updateType) {
        rmContainer.handle(new RMContainerEvent(rmContainer.getContainerId(), RMContainerEventType.ACQUIRED));
    } else {
        rmContainer.handle(new RMContainerUpdatesAcquiredEvent(rmContainer.getContainerId(), ContainerUpdateType.INCREASE_RESOURCE == updateType));
        if (ContainerUpdateType.DECREASE_RESOURCE == updateType) {
            this.rmContext.getDispatcher().getEventHandler().handle(new RMNodeDecreaseContainerEvent(rmContainer.getNodeId(), Collections.singletonList(rmContainer.getContainer())));
        }
    }
    return container;
}

Example 10 with RMContainerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent in project hadoop by apache.

the class SchedulerApplicationAttempt method containerLaunchedOnNode.

@SuppressWarnings("unchecked")
public void containerLaunchedOnNode(ContainerId containerId, NodeId nodeId) {
    try {
        writeLock.lock();
        // Inform the container
        RMContainer rmContainer = getRMContainer(containerId);
        if (rmContainer == null) {
            // Some unknown container sneaked into the system. Kill it.
            rmContext.getDispatcher().getEventHandler().handle(new RMNodeCleanContainerEvent(nodeId, containerId));
            return;
        }
        rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED));
    } finally {
        writeLock.unlock();
    }
}