Examples with SchedulerAppReport org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport used on opensource projects

Example 6 with SchedulerAppReport

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport in project hadoop by apache.

the class TestNodeLabelContainerAllocation method checkTaskContainersHost.

private void checkTaskContainersHost(ApplicationAttemptId attemptId, ContainerId containerId, ResourceManager rm, String host) {
    YarnScheduler scheduler = rm.getRMContext().getScheduler();
    SchedulerAppReport appReport = scheduler.getSchedulerAppInfo(attemptId);
    Assert.assertTrue(appReport.getLiveContainers().size() > 0);
    for (RMContainer c : appReport.getLiveContainers()) {
        if (c.getContainerId().equals(containerId)) {
            Assert.assertEquals(host, c.getAllocatedNode().getHost());
        }
    }
}

Example 7 with SchedulerAppReport

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport in project hadoop by apache.

the class TestFifoScheduler method testUpdateResourceOnNode.

@Test(timeout = 2000)
public void testUpdateResourceOnNode() throws Exception {
    AsyncDispatcher dispatcher = new InlineDispatcher();
    Configuration conf = new Configuration();
    RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf);
    containerTokenSecretManager.rollMasterKey();
    NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf);
    nmTokenSecretManager.rollMasterKey();
    RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
    FifoScheduler scheduler = new FifoScheduler();
    RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
    rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
    rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
    ((RMContextImpl) rmContext).setYarnConfiguration(new YarnConfiguration());
    NullRMNodeLabelsManager nlm = new NullRMNodeLabelsManager();
    nlm.init(new Configuration());
    rmContext.setNodeLabelManager(nlm);
    scheduler.setRMContext(rmContext);
    ((RMContextImpl) rmContext).setScheduler(scheduler);
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(new Configuration(), rmContext);
    RMNode node0 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 4), 1, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
    scheduler.handle(nodeEvent1);
    assertEquals(scheduler.getNumClusterNodes(), 1);
    Resource newResource = Resources.createResource(1024, 4);
    NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new NodeResourceUpdateSchedulerEvent(node0, ResourceOption.newInstance(newResource, ResourceOption.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT));
    scheduler.handle(node0ResourceUpdate);
    // SchedulerNode's total resource and available resource are changed.
    assertEquals(1024, scheduler.getNodeTracker().getNode(node0.getNodeID()).getTotalResource().getMemorySize());
    assertEquals(1024, scheduler.getNodeTracker().getNode(node0.getNodeID()).getUnallocatedResource().getMemorySize(), 1024);
    QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
    Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity(), 0.0f);
    int _appId = 1;
    int _appAttemptId = 1;
    ApplicationAttemptId appAttemptId = createAppAttemptId(_appId, _appAttemptId);
    createMockRMApp(appAttemptId, rmContext);
    AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue1", "user1");
    scheduler.handle(appEvent);
    AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    scheduler.handle(attemptEvent);
    int memory = 1024;
    int priority = 1;
    List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
    ResourceRequest nodeLocal = createResourceRequest(memory, node0.getHostName(), priority, 1);
    ResourceRequest rackLocal = createResourceRequest(memory, node0.getRackName(), priority, 1);
    ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority, 1);
    ask.add(nodeLocal);
    ask.add(rackLocal);
    ask.add(any);
    scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS);
    // Before the node update event, there are one local request
    Assert.assertEquals(1, nodeLocal.getNumContainers());
    NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
    // Now schedule.
    scheduler.handle(node0Update);
    // After the node update event, check no local request
    Assert.assertEquals(0, nodeLocal.getNumContainers());
    // Also check that one container was scheduled
    SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
    Assert.assertEquals(1, info.getLiveContainers().size());
    // And check the default Queue now is full.
    queueInfo = scheduler.getQueueInfo(null, false, false);
    Assert.assertEquals(1.0f, queueInfo.getCurrentCapacity(), 0.0f);
}

Example 8 with SchedulerAppReport

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport in project hadoop by apache.

the class ResourceSchedulerWrapper method updateQueueWithAllocateRequest.

private void updateQueueWithAllocateRequest(Allocation allocation, ApplicationAttemptId attemptId, List<ResourceRequest> resourceRequests, List<ContainerId> containerIds) throws IOException {
    // update queue information
    Resource pendingResource = Resources.createResource(0, 0);
    Resource allocatedResource = Resources.createResource(0, 0);
    String queueName = appQueueMap.get(attemptId.getApplicationId());
    // container requested
    for (ResourceRequest request : resourceRequests) {
        if (request.getResourceName().equals(ResourceRequest.ANY)) {
            Resources.addTo(pendingResource, Resources.multiply(request.getCapability(), request.getNumContainers()));
        }
    }
    // container allocated
    for (Container container : allocation.getContainers()) {
        Resources.addTo(allocatedResource, container.getResource());
        Resources.subtractFrom(pendingResource, container.getResource());
    }
    // container released from AM
    SchedulerAppReport report = scheduler.getSchedulerAppInfo(attemptId);
    for (ContainerId containerId : containerIds) {
        Container container = null;
        for (RMContainer c : report.getLiveContainers()) {
            if (c.getContainerId().equals(containerId)) {
                container = c.getContainer();
                break;
            }
        }
        if (container != null) {
            // released allocated containers
            Resources.subtractFrom(allocatedResource, container.getResource());
        } else {
            for (RMContainer c : report.getReservedContainers()) {
                if (c.getContainerId().equals(containerId)) {
                    container = c.getContainer();
                    break;
                }
            }
            if (container != null) {
                // released reserved containers
                Resources.subtractFrom(pendingResource, container.getResource());
            }
        }
    }
    // containers released/preemption from scheduler
    Set<ContainerId> preemptionContainers = new HashSet<ContainerId>();
    if (allocation.getContainerPreemptions() != null) {
        preemptionContainers.addAll(allocation.getContainerPreemptions());
    }
    if (allocation.getStrictContainerPreemptions() != null) {
        preemptionContainers.addAll(allocation.getStrictContainerPreemptions());
    }
    if (!preemptionContainers.isEmpty()) {
        for (ContainerId containerId : preemptionContainers) {
            if (!preemptionContainerMap.containsKey(containerId)) {
                Container container = null;
                for (RMContainer c : report.getLiveContainers()) {
                    if (c.getContainerId().equals(containerId)) {
                        container = c.getContainer();
                        break;
                    }
                }
                if (container != null) {
                    preemptionContainerMap.put(containerId, container.getResource());
                }
            }
        }
    }
    // update metrics
    SortedMap<String, Counter> counterMap = metrics.getCounters();
    String[] names = new String[] { "counter.queue." + queueName + ".pending.memory", "counter.queue." + queueName + ".pending.cores", "counter.queue." + queueName + ".allocated.memory", "counter.queue." + queueName + ".allocated.cores" };
    long[] values = new long[] { pendingResource.getMemorySize(), pendingResource.getVirtualCores(), allocatedResource.getMemorySize(), allocatedResource.getVirtualCores() };
    for (int i = names.length - 1; i >= 0; i--) {
        if (!counterMap.containsKey(names[i])) {
            metrics.counter(names[i]);
            counterMap = metrics.getCounters();
        }
        counterMap.get(names[i]).inc(values[i]);
    }
    queueLock.lock();
    try {
        if (!schedulerMetrics.isTracked(queueName)) {
            schedulerMetrics.trackQueue(queueName);
        }
    } finally {
        queueLock.unlock();
    }
}

Example 9 with SchedulerAppReport

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport in project hadoop by apache.

the class ResourceSchedulerWrapper method updateQueueWithNodeUpdate.

private void updateQueueWithNodeUpdate(NodeUpdateSchedulerEventWrapper eventWrapper) {
    RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode();
    List<UpdatedContainerInfo> containerList = node.getContainerUpdates();
    for (UpdatedContainerInfo info : containerList) {
        for (ContainerStatus status : info.getCompletedContainers()) {
            ContainerId containerId = status.getContainerId();
            SchedulerAppReport app = scheduler.getSchedulerAppInfo(containerId.getApplicationAttemptId());
            if (app == null) {
                // information.
                continue;
            }
            String queue = appQueueMap.get(containerId.getApplicationAttemptId().getApplicationId());
            int releasedMemory = 0, releasedVCores = 0;
            if (status.getExitStatus() == ContainerExitStatus.SUCCESS) {
                for (RMContainer rmc : app.getLiveContainers()) {
                    if (rmc.getContainerId() == containerId) {
                        releasedMemory += rmc.getContainer().getResource().getMemorySize();
                        releasedVCores += rmc.getContainer().getResource().getVirtualCores();
                        break;
                    }
                }
            } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) {
                if (preemptionContainerMap.containsKey(containerId)) {
                    Resource preResource = preemptionContainerMap.get(containerId);
                    releasedMemory += preResource.getMemorySize();
                    releasedVCores += preResource.getVirtualCores();
                    preemptionContainerMap.remove(containerId);
                }
            }
            // update queue counters
            updateQueueMetrics(queue, releasedMemory, releasedVCores);
        }
    }
}

Example 10 with SchedulerAppReport

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport in project hadoop by apache.

the class SLSCapacityScheduler method updateQueueWithAllocateRequest.

private void updateQueueWithAllocateRequest(Allocation allocation, ApplicationAttemptId attemptId, List<ResourceRequest> resourceRequests, List<ContainerId> containerIds) throws IOException {
    // update queue information
    Resource pendingResource = Resources.createResource(0, 0);
    Resource allocatedResource = Resources.createResource(0, 0);
    String queueName = appQueueMap.get(attemptId);
    // container requested
    for (ResourceRequest request : resourceRequests) {
        if (request.getResourceName().equals(ResourceRequest.ANY)) {
            Resources.addTo(pendingResource, Resources.multiply(request.getCapability(), request.getNumContainers()));
        }
    }
    // container allocated
    for (Container container : allocation.getContainers()) {
        Resources.addTo(allocatedResource, container.getResource());
        Resources.subtractFrom(pendingResource, container.getResource());
    }
    // container released from AM
    SchedulerAppReport report = super.getSchedulerAppInfo(attemptId);
    for (ContainerId containerId : containerIds) {
        Container container = null;
        for (RMContainer c : report.getLiveContainers()) {
            if (c.getContainerId().equals(containerId)) {
                container = c.getContainer();
                break;
            }
        }
        if (container != null) {
            // released allocated containers
            Resources.subtractFrom(allocatedResource, container.getResource());
        } else {
            for (RMContainer c : report.getReservedContainers()) {
                if (c.getContainerId().equals(containerId)) {
                    container = c.getContainer();
                    break;
                }
            }
            if (container != null) {
                // released reserved containers
                Resources.subtractFrom(pendingResource, container.getResource());
            }
        }
    }
    // containers released/preemption from scheduler
    Set<ContainerId> preemptionContainers = new HashSet<ContainerId>();
    if (allocation.getContainerPreemptions() != null) {
        preemptionContainers.addAll(allocation.getContainerPreemptions());
    }
    if (allocation.getStrictContainerPreemptions() != null) {
        preemptionContainers.addAll(allocation.getStrictContainerPreemptions());
    }
    if (!preemptionContainers.isEmpty()) {
        for (ContainerId containerId : preemptionContainers) {
            if (!preemptionContainerMap.containsKey(containerId)) {
                Container container = null;
                for (RMContainer c : report.getLiveContainers()) {
                    if (c.getContainerId().equals(containerId)) {
                        container = c.getContainer();
                        break;
                    }
                }
                if (container != null) {
                    preemptionContainerMap.put(containerId, container.getResource());
                }
            }
        }
    }
    // update metrics
    SortedMap<String, Counter> counterMap = metrics.getCounters();
    String[] names = new String[] { "counter.queue." + queueName + ".pending.memory", "counter.queue." + queueName + ".pending.cores", "counter.queue." + queueName + ".allocated.memory", "counter.queue." + queueName + ".allocated.cores" };
    long[] values = new long[] { pendingResource.getMemorySize(), pendingResource.getVirtualCores(), allocatedResource.getMemorySize(), allocatedResource.getVirtualCores() };
    for (int i = names.length - 1; i >= 0; i--) {
        if (!counterMap.containsKey(names[i])) {
            metrics.counter(names[i]);
            counterMap = metrics.getCounters();
        }
        counterMap.get(names[i]).inc(values[i]);
    }
    queueLock.lock();
    try {
        if (!schedulerMetrics.isTracked(queueName)) {
            schedulerMetrics.trackQueue(queueName);
        }
    } finally {
        queueLock.unlock();
    }
}