Examples with SchedulerRequestKey org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey used on opensource projects

Example 11 with SchedulerRequestKey

use of org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey in project hadoop by apache.

the class AppSchedulingInfo method addToPlacementSets.

boolean addToPlacementSets(boolean recoverPreemptedRequestForAContainer, Map<SchedulerRequestKey, Map<String, ResourceRequest>> dedupRequests) {
    boolean offswitchResourcesUpdated = false;
    for (Map.Entry<SchedulerRequestKey, Map<String, ResourceRequest>> entry : dedupRequests.entrySet()) {
        SchedulerRequestKey schedulerRequestKey = entry.getKey();
        if (!schedulerKeyToPlacementSets.containsKey(schedulerRequestKey)) {
            schedulerKeyToPlacementSets.put(schedulerRequestKey, new LocalitySchedulingPlacementSet<>(this));
        }
        // Update placement set
        ResourceRequestUpdateResult pendingAmountChanges = schedulerKeyToPlacementSets.get(schedulerRequestKey).updateResourceRequests(entry.getValue().values(), recoverPreemptedRequestForAContainer);
        if (null != pendingAmountChanges) {
            updatePendingResources(pendingAmountChanges.getLastAnyResourceRequest(), pendingAmountChanges.getNewResourceRequest(), schedulerRequestKey, queue.getMetrics());
            offswitchResourcesUpdated = true;
        }
    }
    return offswitchResourcesUpdated;
}

Example 12 with SchedulerRequestKey

use of org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey in project hadoop by apache.

the class AppSchedulingInfo method getNextPendingAsk.

public PendingAsk getNextPendingAsk() {
    try {
        readLock.lock();
        SchedulerRequestKey firstRequestKey = schedulerKeys.firstKey();
        return getPendingAsk(firstRequestKey, ResourceRequest.ANY);
    } finally {
        readLock.unlock();
    }
}

Example 13 with SchedulerRequestKey

use of org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey in project hadoop by apache.

the class TestReservations method testFindNodeToUnreserve.

@Test
public void testFindNodeToUnreserve() throws Exception {
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    setup(csConf);
    final String user_0 = "user_0";
    final ApplicationAttemptId appAttemptId_0 = TestUtils.getMockApplicationAttemptId(0, 0);
    LeafQueue a = stubLeafQueue((LeafQueue) queues.get(A));
    FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, mock(ActiveUsersManager.class), spyRMContext);
    String host_1 = "host_1";
    FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0, 8 * GB);
    // Setup resource-requests
    Priority p = TestUtils.createMockPriority(5);
    SchedulerRequestKey priorityMap = toSchedulerKey(p);
    Resource capability = Resources.createResource(2 * GB, 0);
    RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
    SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
    RMContext rmContext = mock(RMContext.class);
    ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);
    DrainDispatcher drainDispatcher = new DrainDispatcher();
    when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
    when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
    when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
    when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
    when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(app_0.getApplicationId(), 1);
    ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
    Container container = TestUtils.getMockContainer(containerId, node_1.getNodeID(), Resources.createResource(2 * GB), priorityMap.getPriority());
    RMContainer rmContainer = new RMContainerImpl(container, SchedulerRequestKey.extractFrom(container), appAttemptId, node_1.getNodeID(), "user", rmContext);
    // nothing reserved
    RMContainer toUnreserveContainer = app_0.findNodeToUnreserve(csContext.getClusterResource(), node_1, priorityMap, capability);
    assertTrue(toUnreserveContainer == null);
    // reserved but scheduler doesn't know about that node.
    app_0.reserve(node_1, priorityMap, rmContainer, container);
    node_1.reserveResource(app_0, priorityMap, rmContainer);
    toUnreserveContainer = app_0.findNodeToUnreserve(csContext.getClusterResource(), node_1, priorityMap, capability);
    assertTrue(toUnreserveContainer == null);
}

Example 14 with SchedulerRequestKey

use of org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey in project hadoop by apache.

the class TestFSAppAttempt method testDelayScheduling.

@Test
public void testDelayScheduling() {
    FSLeafQueue queue = Mockito.mock(FSLeafQueue.class);
    Priority pri = Mockito.mock(Priority.class);
    SchedulerRequestKey prio = TestUtils.toSchedulerKey(pri);
    Mockito.when(pri.getPriority()).thenReturn(1);
    double nodeLocalityThreshold = .5;
    double rackLocalityThreshold = .6;
    ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
    RMContext rmContext = resourceManager.getRMContext();
    FSAppAttempt schedulerApp = new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue, null, rmContext);
    // Default level should be node-local
    assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
    // First five scheduling opportunities should remain node local
    for (int i = 0; i < 5; i++) {
        schedulerApp.addSchedulingOpportunity(prio);
        assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
    }
    // After five it should switch to rack local
    schedulerApp.addSchedulingOpportunity(prio);
    assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
    // Manually set back to node local
    schedulerApp.resetAllowedLocalityLevel(prio, NodeType.NODE_LOCAL);
    schedulerApp.resetSchedulingOpportunities(prio);
    assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
    // Now escalate again to rack-local, then to off-switch
    for (int i = 0; i < 5; i++) {
        schedulerApp.addSchedulingOpportunity(prio);
        assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
    }
    schedulerApp.addSchedulingOpportunity(prio);
    assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
    for (int i = 0; i < 6; i++) {
        schedulerApp.addSchedulingOpportunity(prio);
        assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
    }
    schedulerApp.addSchedulingOpportunity(prio);
    assertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
}

Example 15 with SchedulerRequestKey

use of org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey in project hadoop by apache.

the class TestFSAppAttempt method testLocalityLevelWithoutDelays.

@Test
public /**
   * Ensure that when negative paramaters are given (signaling delay scheduling
   * no tin use), the least restrictive locality level is returned.
   */
void testLocalityLevelWithoutDelays() {
    FSLeafQueue queue = Mockito.mock(FSLeafQueue.class);
    Priority pri = Mockito.mock(Priority.class);
    SchedulerRequestKey prio = TestUtils.toSchedulerKey(pri);
    Mockito.when(pri.getPriority()).thenReturn(1);
    RMContext rmContext = resourceManager.getRMContext();
    ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
    FSAppAttempt schedulerApp = new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue, null, rmContext);
    assertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevel(prio, 10, -1.0, -1.0));
}