Examples with CapacityScheduler org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler used on opensource projects

Example 11 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class TestRMAdminService method testAdminRefreshQueuesWithLocalConfigurationProvider.

@Test
public void testAdminRefreshQueuesWithLocalConfigurationProvider() throws IOException, YarnException {
    rm = new MockRM(configuration);
    rm.init(configuration);
    rm.start();
    CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler();
    int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications();
    try {
        rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance());
        Assert.assertEquals(maxAppsBefore, cs.getConfiguration().getMaximumSystemApplications());
    } catch (Exception ex) {
        fail("Using localConfigurationProvider. Should not get any exception.");
    }
}

Example 12 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class TestRMDispatcher method testSchedulerEventDispatcherForPreemptionEvents.

@SuppressWarnings("unchecked")
@Test(timeout = 10000)
public void testSchedulerEventDispatcherForPreemptionEvents() {
    AsyncDispatcher rmDispatcher = new AsyncDispatcher();
    CapacityScheduler sched = spy(new CapacityScheduler());
    YarnConfiguration conf = new YarnConfiguration();
    EventDispatcher schedulerDispatcher = new EventDispatcher(sched, sched.getClass().getName());
    rmDispatcher.register(SchedulerEventType.class, schedulerDispatcher);
    rmDispatcher.init(conf);
    rmDispatcher.start();
    schedulerDispatcher.init(conf);
    schedulerDispatcher.start();
    try {
        ApplicationAttemptId appAttemptId = mock(ApplicationAttemptId.class);
        RMContainer container = mock(RMContainer.class);
        ContainerPreemptEvent event1 = new ContainerPreemptEvent(appAttemptId, container, SchedulerEventType.KILL_RESERVED_CONTAINER);
        rmDispatcher.getEventHandler().handle(event1);
        ContainerPreemptEvent event2 = new ContainerPreemptEvent(appAttemptId, container, SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE);
        rmDispatcher.getEventHandler().handle(event2);
        ContainerPreemptEvent event3 = new ContainerPreemptEvent(appAttemptId, container, SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION);
        rmDispatcher.getEventHandler().handle(event3);
        // Wait for events to be processed by scheduler dispatcher.
        Thread.sleep(1000);
        verify(sched, times(3)).handle(any(SchedulerEvent.class));
        verify(sched).killReservedContainer(container);
        verify(sched).markContainerForPreemption(appAttemptId, container);
        verify(sched).markContainerForKillable(container);
    } catch (InterruptedException e) {
        Assert.fail();
    } finally {
        schedulerDispatcher.stop();
        rmDispatcher.stop();
    }
}

Example 13 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class ReservationSystemTestUtil method mockCapacityScheduler.

public CapacityScheduler mockCapacityScheduler(int numContainers) throws IOException {
    // stolen from TestCapacityScheduler
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    setupQueueConfiguration(conf);
    CapacityScheduler cs = Mockito.spy(new CapacityScheduler());
    cs.setConf(new YarnConfiguration());
    RMContext mockRmContext = createRMContext(conf);
    cs.setRMContext(mockRmContext);
    try {
        cs.serviceInit(conf);
    } catch (Exception e) {
        Assert.fail(e.getMessage());
    }
    initializeRMContext(numContainers, cs, mockRmContext);
    return cs;
}

Example 14 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class TestGreedyReservationAgent method testStress.

public void testStress(int numJobs) throws PlanningException, IOException {
    long timeWindow = 1000000L;
    Resource clusterCapacity = Resource.newInstance(500 * 100 * 1024, 500 * 32);
    step = 1000L;
    ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
    CapacityScheduler scheduler = testUtil.mockCapacityScheduler(500 * 100);
    String reservationQ = ReservationSystemTestUtil.getFullReservationQueueName();
    float instConstraint = 100;
    float avgConstraint = 100;
    ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.createConf(reservationQ, timeWindow, instConstraint, avgConstraint);
    CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
    policy.init(reservationQ, conf);
    RMContext context = ReservationSystemTestUtil.createMockRMContext();
    plan = new InMemoryPlan(scheduler.getRootQueueMetrics(), policy, agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", null, true, context);
    int acc = 0;
    List<ReservationDefinition> list = new ArrayList<ReservationDefinition>();
    for (long i = 0; i < numJobs; i++) {
        list.add(ReservationSystemTestUtil.generateRandomRR(rand, i));
    }
    long start = System.currentTimeMillis();
    for (int i = 0; i < numJobs; i++) {
        try {
            if (agent.createReservation(ReservationSystemTestUtil.getNewReservationId(), "u" + i % 100, plan, list.get(i))) {
                acc++;
            }
        } catch (PlanningException p) {
        // ignore exceptions
        }
    }
    long end = System.currentTimeMillis();
    System.out.println("Submitted " + numJobs + " jobs " + " accepted " + acc + " in " + (end - start) + "ms");
}

Example 15 with CapacityScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler in project hadoop by apache.

the class TestReservationSystem method initializeCapacityScheduler.

private CapacityScheduler initializeCapacityScheduler() {
    // stolen from TestCapacityScheduler
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    ReservationSystemTestUtil.setupQueueConfiguration(conf);
    CapacityScheduler cs = Mockito.spy(new CapacityScheduler());
    cs.setConf(conf);
    mockRMContext = ReservationSystemTestUtil.createRMContext(conf);
    cs.setRMContext(mockRMContext);
    try {
        cs.serviceInit(conf);
    } catch (Exception e) {
        Assert.fail(e.getMessage());
    }
    ReservationSystemTestUtil.initializeRMContext(10, cs, mockRMContext);
    return cs;
}