Examples with ContainerPreemptEvent org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent used on opensource projects

Example 1 with ContainerPreemptEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent in project hadoop by apache.

the class TestRMDispatcher method testSchedulerEventDispatcherForPreemptionEvents.

@SuppressWarnings("unchecked")
@Test(timeout = 10000)
public void testSchedulerEventDispatcherForPreemptionEvents() {
    AsyncDispatcher rmDispatcher = new AsyncDispatcher();
    CapacityScheduler sched = spy(new CapacityScheduler());
    YarnConfiguration conf = new YarnConfiguration();
    EventDispatcher schedulerDispatcher = new EventDispatcher(sched, sched.getClass().getName());
    rmDispatcher.register(SchedulerEventType.class, schedulerDispatcher);
    rmDispatcher.init(conf);
    rmDispatcher.start();
    schedulerDispatcher.init(conf);
    schedulerDispatcher.start();
    try {
        ApplicationAttemptId appAttemptId = mock(ApplicationAttemptId.class);
        RMContainer container = mock(RMContainer.class);
        ContainerPreemptEvent event1 = new ContainerPreemptEvent(appAttemptId, container, SchedulerEventType.KILL_RESERVED_CONTAINER);
        rmDispatcher.getEventHandler().handle(event1);
        ContainerPreemptEvent event2 = new ContainerPreemptEvent(appAttemptId, container, SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE);
        rmDispatcher.getEventHandler().handle(event2);
        ContainerPreemptEvent event3 = new ContainerPreemptEvent(appAttemptId, container, SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION);
        rmDispatcher.getEventHandler().handle(event3);
        // Wait for events to be processed by scheduler dispatcher.
        Thread.sleep(1000);
        verify(sched, times(3)).handle(any(SchedulerEvent.class));
        verify(sched).killReservedContainer(container);
        verify(sched).markContainerForPreemption(appAttemptId, container);
        verify(sched).markContainerForKillable(container);
    } catch (InterruptedException e) {
        Assert.fail();
    } finally {
        schedulerDispatcher.stop();
        rmDispatcher.stop();
    }
}

Example 2 with ContainerPreemptEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicy method testExpireKill.

@Test
public void testExpireKill() {
    final long killTime = 10000L;
    int[][] qData = new int[][] { // abs
    { 100, 40, 40, 20 }, // maxCap
    { 100, 100, 100, 100 }, // used
    { 100, 0, 60, 40 }, // pending
    { 10, 10, 0, 0 }, // reserved
    { 0, 0, 0, 0 }, // apps
    { 3, 1, 1, 1 }, // req granularity
    { -1, 1, 1, 1 }, // subqueues
    { 3, 0, 0, 0 } };
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, killTime);
    ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
    // ensure all pending rsrc from A get preempted from other queues
    when(mClock.getTime()).thenReturn(0L);
    policy.editSchedule();
    verify(mDisp, times(10)).handle(argThat(new IsPreemptionRequestFor(appC)));
    // requests reiterated
    when(mClock.getTime()).thenReturn(killTime / 2);
    policy.editSchedule();
    verify(mDisp, times(10)).handle(argThat(new IsPreemptionRequestFor(appC)));
    // kill req sent
    when(mClock.getTime()).thenReturn(killTime + 1);
    policy.editSchedule();
    verify(mDisp, times(20)).handle(evtCaptor.capture());
    List<ContainerPreemptEvent> events = evtCaptor.getAllValues();
    for (ContainerPreemptEvent e : events.subList(20, 20)) {
        assertEquals(appC, e.getAppId());
        assertEquals(MARK_CONTAINER_FOR_KILLABLE, e.getType());
    }
}

Example 3 with ContainerPreemptEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicy method preemptOrkillSelectedContainerAfterWait.

@SuppressWarnings("unchecked")
private void preemptOrkillSelectedContainerAfterWait(Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates, long currentTime) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Starting to preempt containers for selectedCandidates and size:" + selectedCandidates.size());
    }
    // preempt (or kill) the selected containers
    for (Map.Entry<ApplicationAttemptId, Set<RMContainer>> e : selectedCandidates.entrySet()) {
        ApplicationAttemptId appAttemptId = e.getKey();
        if (LOG.isDebugEnabled()) {
            LOG.debug("Send to scheduler: in app=" + appAttemptId + " #containers-to-be-preemptionCandidates=" + e.getValue().size());
        }
        for (RMContainer container : e.getValue()) {
            // if we tried to preempt this for more than maxWaitTime
            if (preemptionCandidates.get(container) != null && preemptionCandidates.get(container) + maxWaitTime <= currentTime) {
                // kill it
                rmContext.getDispatcher().getEventHandler().handle(new ContainerPreemptEvent(appAttemptId, container, SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE));
                preemptionCandidates.remove(container);
            } else {
                if (preemptionCandidates.get(container) != null) {
                    // not have to raise another event.
                    continue;
                }
                //otherwise just send preemption events
                rmContext.getDispatcher().getEventHandler().handle(new ContainerPreemptEvent(appAttemptId, container, SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION));
                preemptionCandidates.put(container, currentTime);
            }
        }
    }
}

Example 4 with ContainerPreemptEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent in project hadoop by apache.

the class FifoCandidatesSelector method preemptFrom.

/**
   * Given a target preemption for a specific application, select containers
   * to preempt (after unreserving all reservation for that app).
   */
@SuppressWarnings("unchecked")
private void preemptFrom(FiCaSchedulerApp app, Resource clusterResource, Map<String, Resource> resToObtainByPartition, List<RMContainer> skippedAMContainerlist, Resource skippedAMSize, Map<ApplicationAttemptId, Set<RMContainer>> selectedContainers, Resource totalPreemptionAllowed) {
    ApplicationAttemptId appId = app.getApplicationAttemptId();
    // first drop reserved containers towards rsrcPreempt
    List<RMContainer> reservedContainers = new ArrayList<>(app.getReservedContainers());
    for (RMContainer c : reservedContainers) {
        if (CapacitySchedulerPreemptionUtils.isContainerAlreadySelected(c, selectedContainers)) {
            continue;
        }
        if (resToObtainByPartition.isEmpty()) {
            return;
        }
        // Try to preempt this container
        CapacitySchedulerPreemptionUtils.tryPreemptContainerAndDeductResToObtain(rc, preemptionContext, resToObtainByPartition, c, clusterResource, selectedContainers, totalPreemptionAllowed);
        if (!preemptionContext.isObserveOnly()) {
            preemptionContext.getRMContext().getDispatcher().getEventHandler().handle(new ContainerPreemptEvent(appId, c, SchedulerEventType.KILL_RESERVED_CONTAINER));
        }
    }
    // if more resources are to be freed go through all live containers in
    // reverse priority and reverse allocation order and mark them for
    // preemption
    List<RMContainer> liveContainers = new ArrayList<>(app.getLiveContainers());
    sortContainers(liveContainers);
    for (RMContainer c : liveContainers) {
        if (resToObtainByPartition.isEmpty()) {
            return;
        }
        if (CapacitySchedulerPreemptionUtils.isContainerAlreadySelected(c, selectedContainers)) {
            continue;
        }
        // Skip already marked to killable containers
        if (null != preemptionContext.getKillableContainers() && preemptionContext.getKillableContainers().contains(c.getContainerId())) {
            continue;
        }
        // Skip AM Container from preemption for now.
        if (c.isAMContainer()) {
            skippedAMContainerlist.add(c);
            Resources.addTo(skippedAMSize, c.getAllocatedResource());
            continue;
        }
        // Try to preempt this container
        CapacitySchedulerPreemptionUtils.tryPreemptContainerAndDeductResToObtain(rc, preemptionContext, resToObtainByPartition, c, clusterResource, selectedContainers, totalPreemptionAllowed);
    }
}