Examples with RMContainer org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer used on opensource projects

Example 91 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class TestFairScheduler method testMultipleCompletedEvent.

@Test
public void testMultipleCompletedEvent() throws Exception {
    // Set up a fair scheduler
    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("<queue name=\"queue1\">");
    out.println("<maxAMShare>0.2</maxAMShare>");
    out.println("</queue>");
    out.println("</allocations>");
    out.close();
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    // Create a node
    RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(20480, 20), 0, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
    NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
    scheduler.handle(nodeEvent);
    scheduler.update();
    // Launch an app
    ApplicationAttemptId attId1 = createAppAttemptId(1, 1);
    createApplicationWithAMResource(attId1, "queue1", "user1", Resource.newInstance(1024, 1));
    createSchedulingRequestExistingApplication(1024, 1, RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority(), attId1);
    FSAppAttempt app1 = scheduler.getSchedulerApp(attId1);
    scheduler.update();
    scheduler.handle(updateEvent);
    RMContainer container = app1.getLiveContainersMap().values().iterator().next();
    scheduler.completedContainer(container, SchedulerUtils.createAbnormalContainerStatus(container.getContainerId(), SchedulerUtils.LOST_CONTAINER), RMContainerEventType.KILL);
    scheduler.completedContainer(container, SchedulerUtils.createAbnormalContainerStatus(container.getContainerId(), SchedulerUtils.COMPLETED_APPLICATION), RMContainerEventType.FINISHED);
    assertEquals(Resources.none(), app1.getResourceUsage());
}

Example 92 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class TestContinuousScheduling method testSortedNodes.

@Test(timeout = 10000)
public void testSortedNodes() throws Exception {
    // Add two nodes
    RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
    scheduler.handle(nodeEvent1);
    RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 2, "127.0.0.2");
    NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
    scheduler.handle(nodeEvent2);
    // available resource
    Assert.assertEquals(scheduler.getClusterResource().getMemorySize(), 16 * 1024);
    Assert.assertEquals(scheduler.getClusterResource().getVirtualCores(), 16);
    // send application request
    ApplicationAttemptId appAttemptId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
    createMockRMApp(appAttemptId);
    scheduler.addApplication(appAttemptId.getApplicationId(), "queue11", "user11", false);
    scheduler.addApplicationAttempt(appAttemptId, false, false);
    List<ResourceRequest> ask = new ArrayList<>();
    ResourceRequest request = createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true);
    ask.add(request);
    scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
    triggerSchedulingAttempt();
    FSAppAttempt app = scheduler.getSchedulerApp(appAttemptId);
    checkAppConsumption(app, Resources.createResource(1024, 1));
    // another request
    request = createResourceRequest(1024, 1, ResourceRequest.ANY, 2, 1, true);
    ask.clear();
    ask.add(request);
    scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
    triggerSchedulingAttempt();
    checkAppConsumption(app, Resources.createResource(2048, 2));
    // 2 containers should be assigned to 2 nodes
    Set<NodeId> nodes = new HashSet<NodeId>();
    Iterator<RMContainer> it = app.getLiveContainers().iterator();
    while (it.hasNext()) {
        nodes.add(it.next().getContainer().getNodeId());
    }
    Assert.assertEquals(2, nodes.size());
}

Example 93 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class TestFairScheduler method testReservationWithMultiplePriorities.

/**
   * Reserve at a lower priority and verify the lower priority request gets
   * allocated
   */
@Test(timeout = 5000)
public void testReservationWithMultiplePriorities() throws IOException {
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    // Add a node
    RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2));
    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
    scheduler.handle(nodeEvent1);
    NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
    // Create first app and take up half resources so the second app that asks
    // for the entire node won't have enough.
    FSAppAttempt app1 = scheduler.getSchedulerApp(createSchedulingRequest(1024, 1, "queue", "user", 1));
    scheduler.update();
    scheduler.handle(updateEvent);
    assertEquals("Basic allocation failed", 1, app1.getLiveContainers().size());
    // Create another app and reserve at a lower priority first
    ApplicationAttemptId attId = createSchedulingRequest(2048, 2, "queue1", "user1", 1, 2);
    FSAppAttempt app2 = scheduler.getSchedulerApp(attId);
    scheduler.update();
    scheduler.handle(updateEvent);
    assertEquals("Reservation at lower priority failed", 1, app2.getReservedContainers().size());
    // Request container on the second app at a higher priority
    createSchedulingRequestExistingApplication(2048, 2, 1, attId);
    // Complete the first container so we can trigger allocation for app2
    ContainerId containerId = app1.getLiveContainers().iterator().next().getContainerId();
    scheduler.allocate(app1.getApplicationAttemptId(), new ArrayList<>(), Arrays.asList(containerId), null, null, NULL_UPDATE_REQUESTS);
    // Trigger allocation for app2
    scheduler.handle(updateEvent);
    // Reserved container (at lower priority) should be run
    Collection<RMContainer> liveContainers = app2.getLiveContainers();
    assertEquals("Allocation post completion failed", 1, liveContainers.size());
    assertEquals("High prio container allocated against low prio reservation", 2, liveContainers.iterator().next().getContainer().getPriority().getPriority());
}

Example 94 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class FifoAppAttempt method allocate.

public RMContainer allocate(NodeType type, FiCaSchedulerNode node, SchedulerRequestKey schedulerKey, Container container) {
    try {
        writeLock.lock();
        if (isStopped) {
            return null;
        }
        // request without locking the scheduler, hence we need to check
        if (getOutstandingAsksCount(schedulerKey) <= 0) {
            return null;
        }
        // Create RMContainer
        RMContainer rmContainer = new RMContainerImpl(container, schedulerKey, this.getApplicationAttemptId(), node.getNodeID(), appSchedulingInfo.getUser(), this.rmContext, node.getPartition());
        ((RMContainerImpl) rmContainer).setQueueName(this.getQueueName());
        updateAMContainerDiagnostics(AMState.ASSIGNED, null);
        // Add it to allContainers list.
        addToNewlyAllocatedContainers(node, rmContainer);
        ContainerId containerId = container.getId();
        liveContainers.put(containerId, rmContainer);
        // Update consumption and track allocations
        List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(type, node, schedulerKey, container);
        attemptResourceUsage.incUsed(node.getPartition(), container.getResource());
        // Update resource requests related to "request" and store in RMContainer
        ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList);
        // Inform the container
        rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START));
        if (LOG.isDebugEnabled()) {
            LOG.debug("allocate: applicationAttemptId=" + containerId.getApplicationAttemptId() + " container=" + containerId + " host=" + container.getNodeId().getHost() + " type=" + type);
        }
        RMAuditLogger.logSuccess(getUser(), RMAuditLogger.AuditConstants.ALLOC_CONTAINER, "SchedulerApp", getApplicationId(), containerId, container.getResource());
        return rmContainer;
    } finally {
        writeLock.unlock();
    }
}

Example 95 with RMContainer

use of org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer in project hadoop by apache.

the class FifoScheduler method doneApplicationAttempt.

private synchronized void doneApplicationAttempt(ApplicationAttemptId applicationAttemptId, RMAppAttemptState rmAppAttemptFinalState, boolean keepContainers) throws IOException {
    FifoAppAttempt attempt = getApplicationAttempt(applicationAttemptId);
    SchedulerApplication<FifoAppAttempt> application = applications.get(applicationAttemptId.getApplicationId());
    if (application == null || attempt == null) {
        throw new IOException("Unknown application " + applicationAttemptId + " has completed!");
    }
    // Kill all 'live' containers
    for (RMContainer container : attempt.getLiveContainers()) {
        if (keepContainers && container.getState().equals(RMContainerState.RUNNING)) {
            // do not kill the running container in the case of work-preserving AM
            // restart.
            LOG.info("Skip killing " + container.getContainerId());
            continue;
        }
        super.completedContainer(container, SchedulerUtils.createAbnormalContainerStatus(container.getContainerId(), SchedulerUtils.COMPLETED_APPLICATION), RMContainerEventType.KILL);
    }
    // Clean up pending requests, metrics etc.
    attempt.stop(rmAppAttemptFinalState);
}