Examples with FiCaSchedulerApp org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp used on opensource projects

Example 96 with FiCaSchedulerApp

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp in project hadoop by apache.

the class TestLeafQueue method testFairAssignment.

@Test
public void testFairAssignment() throws Exception {
    LeafQueue a = stubLeafQueue((LeafQueue) queues.get(A));
    OrderingPolicy<FiCaSchedulerApp> schedulingOrder = new FairOrderingPolicy<FiCaSchedulerApp>();
    a.setOrderingPolicy(schedulingOrder);
    String host_0_0 = "127.0.0.1";
    String rack_0 = "rack_0";
    FiCaSchedulerNode node_0_0 = TestUtils.getMockNode(host_0_0, rack_0, 0, 16 * GB);
    final int numNodes = 4;
    Resource clusterResource = Resources.createResource(numNodes * (16 * GB), numNodes * 16);
    when(csContext.getNumClusterNodes()).thenReturn(numNodes);
    String user_0 = "user_0";
    final ApplicationAttemptId appAttemptId_0 = TestUtils.getMockApplicationAttemptId(0, 0);
    FiCaSchedulerApp app_0 = spy(new FiCaSchedulerApp(appAttemptId_0, user_0, a, mock(ActiveUsersManager.class), spyRMContext));
    a.submitApplicationAttempt(app_0, user_0);
    final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(1, 0);
    FiCaSchedulerApp app_1 = spy(new FiCaSchedulerApp(appAttemptId_1, user_0, a, mock(ActiveUsersManager.class), spyRMContext));
    a.submitApplicationAttempt(app_1, user_0);
    Map<ApplicationAttemptId, FiCaSchedulerApp> apps = ImmutableMap.of(app_0.getApplicationAttemptId(), app_0, app_1.getApplicationAttemptId(), app_1);
    Map<NodeId, FiCaSchedulerNode> nodes = ImmutableMap.of(node_0_0.getNodeID(), node_0_0);
    Priority priority = TestUtils.createMockPriority(1);
    List<ResourceRequest> app_0_requests_0 = new ArrayList<ResourceRequest>();
    List<ResourceRequest> app_1_requests_0 = new ArrayList<ResourceRequest>();
    app_0_requests_0.clear();
    app_0_requests_0.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 2 * GB, 1, true, priority, recordFactory));
    app_0.updateResourceRequests(app_0_requests_0);
    app_1_requests_0.clear();
    app_1_requests_0.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory));
    app_1.updateResourceRequests(app_1_requests_0);
    // app_0 will get containers as its submitted first.
    applyCSAssignment(clusterResource, a.assignContainers(clusterResource, node_0_0, new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
    Assert.assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize());
    applyCSAssignment(clusterResource, a.assignContainers(clusterResource, node_0_0, new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
    Assert.assertEquals(1 * GB, app_1.getCurrentConsumption().getMemorySize());
    app_0_requests_0.clear();
    app_0_requests_0.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory));
    app_0.updateResourceRequests(app_0_requests_0);
    app_1_requests_0.clear();
    app_1_requests_0.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory));
    app_1.updateResourceRequests(app_1_requests_0);
    //Since it already has more resources, app_0 will not get
    //assigned first, but app_1 will
    applyCSAssignment(clusterResource, a.assignContainers(clusterResource, node_0_0, new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
    Assert.assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize());
    Assert.assertEquals(2 * GB, app_1.getCurrentConsumption().getMemorySize());
    //and only then will app_0
    applyCSAssignment(clusterResource, a.assignContainers(clusterResource, node_0_0, new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
    Assert.assertEquals(3 * GB, app_0.getCurrentConsumption().getMemorySize());
}

Example 97 with FiCaSchedulerApp

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp in project hadoop by apache.

the class TestLeafQueue method testFifoAssignment.

@Test
public void testFifoAssignment() throws Exception {
    LeafQueue a = stubLeafQueue((LeafQueue) queues.get(A));
    a.setOrderingPolicy(new FifoOrderingPolicy<FiCaSchedulerApp>());
    String host_0_0 = "127.0.0.1";
    String rack_0 = "rack_0";
    FiCaSchedulerNode node_0_0 = TestUtils.getMockNode(host_0_0, rack_0, 0, 16 * GB);
    final int numNodes = 4;
    Resource clusterResource = Resources.createResource(numNodes * (16 * GB), numNodes * 16);
    when(csContext.getNumClusterNodes()).thenReturn(numNodes);
    String user_0 = "user_0";
    final ApplicationAttemptId appAttemptId_0 = TestUtils.getMockApplicationAttemptId(0, 0);
    FiCaSchedulerApp app_0 = spy(new FiCaSchedulerApp(appAttemptId_0, user_0, a, mock(ActiveUsersManager.class), spyRMContext, Priority.newInstance(3), false));
    a.submitApplicationAttempt(app_0, user_0);
    final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(1, 0);
    FiCaSchedulerApp app_1 = spy(new FiCaSchedulerApp(appAttemptId_1, user_0, a, mock(ActiveUsersManager.class), spyRMContext, Priority.newInstance(5), false));
    a.submitApplicationAttempt(app_1, user_0);
    Map<ApplicationAttemptId, FiCaSchedulerApp> apps = ImmutableMap.of(app_0.getApplicationAttemptId(), app_0, app_1.getApplicationAttemptId(), app_1);
    Map<NodeId, FiCaSchedulerNode> nodes = ImmutableMap.of(node_0_0.getNodeID(), node_0_0);
    Priority priority = TestUtils.createMockPriority(1);
    List<ResourceRequest> app_0_requests_0 = new ArrayList<ResourceRequest>();
    List<ResourceRequest> app_1_requests_0 = new ArrayList<ResourceRequest>();
    app_0_requests_0.clear();
    app_0_requests_0.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 2 * GB, 1, true, priority, recordFactory));
    app_0.updateResourceRequests(app_0_requests_0);
    app_1_requests_0.clear();
    app_1_requests_0.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory));
    app_1.updateResourceRequests(app_1_requests_0);
    // app_1 will get containers as it has high priority
    applyCSAssignment(clusterResource, a.assignContainers(clusterResource, node_0_0, new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
    Assert.assertEquals(1 * GB, app_1.getCurrentConsumption().getMemorySize());
    applyCSAssignment(clusterResource, a.assignContainers(clusterResource, node_0_0, new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
    Assert.assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize());
    app_0_requests_0.clear();
    app_0_requests_0.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory));
    app_0.updateResourceRequests(app_0_requests_0);
    app_1_requests_0.clear();
    app_1_requests_0.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory));
    app_1.updateResourceRequests(app_1_requests_0);
    //app_1 will still get assigned first as priority is more.
    applyCSAssignment(clusterResource, a.assignContainers(clusterResource, node_0_0, new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
    Assert.assertEquals(2 * GB, app_1.getCurrentConsumption().getMemorySize());
    Assert.assertEquals(2 * GB, app_0.getCurrentConsumption().getMemorySize());
    //and only then will app_2
    applyCSAssignment(clusterResource, a.assignContainers(clusterResource, node_0_0, new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
    Assert.assertEquals(3 * GB, app_0.getCurrentConsumption().getMemorySize());
}

Example 98 with FiCaSchedulerApp

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp in project hadoop by apache.

the class TestLeafQueue method testActivateApplicationAfterQueueRefresh.

@Test(timeout = 30000)
public void testActivateApplicationAfterQueueRefresh() throws Exception {
    // Manipulate queue 'e'
    LeafQueue e = stubLeafQueue((LeafQueue) queues.get(E));
    // Users
    final String user_e = "user_e";
    when(amResourceRequest.getCapability()).thenReturn(Resources.createResource(1 * GB, 0));
    // Submit applications
    final ApplicationAttemptId appAttemptId_0 = TestUtils.getMockApplicationAttemptId(0, 0);
    FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_e, e, mock(ActiveUsersManager.class), spyRMContext);
    e.submitApplicationAttempt(app_0, user_e);
    final ApplicationAttemptId appAttemptId_1 = TestUtils.getMockApplicationAttemptId(1, 0);
    FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_e, e, mock(ActiveUsersManager.class), spyRMContext);
    // same user
    e.submitApplicationAttempt(app_1, user_e);
    final ApplicationAttemptId appAttemptId_2 = TestUtils.getMockApplicationAttemptId(2, 0);
    FiCaSchedulerApp app_2 = new FiCaSchedulerApp(appAttemptId_2, user_e, e, mock(ActiveUsersManager.class), spyRMContext);
    // same user
    e.submitApplicationAttempt(app_2, user_e);
    // before reinitialization
    assertEquals(2, e.getNumActiveApplications());
    assertEquals(1, e.getNumPendingApplications());
    csConf.setDouble(CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT * 2);
    Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
    CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, TestUtils.spyHook);
    queues = newQueues;
    root.reinitialize(newRoot, csContext.getClusterResource());
    // after reinitialization
    assertEquals(3, e.getNumActiveApplications());
    assertEquals(0, e.getNumPendingApplications());
}

Example 99 with FiCaSchedulerApp

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp in project hadoop by apache.

the class CapacityScheduler method addApplicationAttempt.

private void addApplicationAttempt(ApplicationAttemptId applicationAttemptId, boolean transferStateFromPreviousAttempt, boolean isAttemptRecovering) {
    try {
        writeLock.lock();
        SchedulerApplication<FiCaSchedulerApp> application = applications.get(applicationAttemptId.getApplicationId());
        if (application == null) {
            LOG.warn("Application " + applicationAttemptId.getApplicationId() + " cannot be found in scheduler.");
            return;
        }
        CSQueue queue = (CSQueue) application.getQueue();
        FiCaSchedulerApp attempt = new FiCaSchedulerApp(applicationAttemptId, application.getUser(), queue, queue.getAbstractUsersManager(), rmContext, application.getPriority(), isAttemptRecovering, activitiesManager);
        if (transferStateFromPreviousAttempt) {
            attempt.transferStateFromPreviousAttempt(application.getCurrentAppAttempt());
        }
        application.setCurrentAppAttempt(attempt);
        // Update attempt priority to the latest to avoid race condition i.e
        // SchedulerApplicationAttempt is created with old priority but it is not
        // set to SchedulerApplication#setCurrentAppAttempt.
        // Scenario would occur is
        // 1. SchdulerApplicationAttempt is created with old priority.
        // 2. updateApplicationPriority() updates SchedulerApplication. Since
        // currentAttempt is null, it just return.
        // 3. ScheduelerApplcationAttempt is set in
        // SchedulerApplication#setCurrentAppAttempt.
        attempt.setPriority(application.getPriority());
        queue.submitApplicationAttempt(attempt, application.getUser());
        LOG.info("Added Application Attempt " + applicationAttemptId + " to scheduler from user " + application.getUser() + " in queue " + queue.getQueueName());
        if (isAttemptRecovering) {
            if (LOG.isDebugEnabled()) {
                LOG.debug(applicationAttemptId + " is recovering. Skipping notifying ATTEMPT_ADDED");
            }
        } else {
            rmContext.getDispatcher().getEventHandler().handle(new RMAppAttemptEvent(applicationAttemptId, RMAppAttemptEventType.ATTEMPT_ADDED));
        }
    } finally {
        writeLock.unlock();
    }
}

Example 100 with FiCaSchedulerApp

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp in project hadoop by apache.

the class CapacityScheduler method createResourceCommitRequest.

@VisibleForTesting
public ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> createResourceCommitRequest(CSAssignment csAssignment) {
    ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> allocated = null;
    ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> reserved = null;
    List<SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode>> released = null;
    if (Resources.greaterThan(calculator, getClusterResource(), csAssignment.getResource(), Resources.none())) {
        // Allocated something
        List<AssignmentInformation.AssignmentDetails> allocations = csAssignment.getAssignmentInformation().getAllocationDetails();
        if (!allocations.isEmpty()) {
            RMContainer rmContainer = allocations.get(0).rmContainer;
            allocated = new ContainerAllocationProposal<>(getSchedulerContainer(rmContainer, true), getSchedulerContainersToRelease(csAssignment), getSchedulerContainer(csAssignment.getFulfilledReservedContainer(), false), csAssignment.getType(), csAssignment.getRequestLocalityType(), csAssignment.getSchedulingMode() != null ? csAssignment.getSchedulingMode() : SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY, csAssignment.getResource());
        }
        // Reserved something
        List<AssignmentInformation.AssignmentDetails> reservation = csAssignment.getAssignmentInformation().getReservationDetails();
        if (!reservation.isEmpty()) {
            RMContainer rmContainer = reservation.get(0).rmContainer;
            reserved = new ContainerAllocationProposal<>(getSchedulerContainer(rmContainer, false), getSchedulerContainersToRelease(csAssignment), getSchedulerContainer(csAssignment.getFulfilledReservedContainer(), false), csAssignment.getType(), csAssignment.getRequestLocalityType(), csAssignment.getSchedulingMode() != null ? csAssignment.getSchedulingMode() : SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY, csAssignment.getResource());
        }
    }
    // kill all to-release containers in the request.
    if (null == allocated && null == reserved) {
        released = getSchedulerContainersToRelease(csAssignment);
    }
    if (null != allocated || null != reserved || (null != released && !released.isEmpty())) {
        List<ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode>> allocationsList = null;
        if (allocated != null) {
            allocationsList = new ArrayList<>();
            allocationsList.add(allocated);
        }
        List<ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode>> reservationsList = null;
        if (reserved != null) {
            reservationsList = new ArrayList<>();
            reservationsList.add(reserved);
        }
        return new ResourceCommitRequest<>(allocationsList, reservationsList, released);
    }
    return null;
}