Examples with ReservationId org.apache.hadoop.yarn.api.records.ReservationId used on opensource projects

Example 76 with ReservationId

use of org.apache.hadoop.yarn.api.records.ReservationId in project hadoop by apache.

the class TestInMemoryPlan method testArchiveCompletedReservations.

@Test
public void testArchiveCompletedReservations() {
    Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
    ReservationId reservationID1 = ReservationSystemTestUtil.getNewReservationId();
    // First add a reservation
    int[] alloc1 = { 10, 10, 10, 10, 10, 10 };
    int start = 100;
    ReservationAllocation rAllocation = createReservationAllocation(reservationID1, start, alloc1);
    Assert.assertNull(plan.getReservationById(reservationID1));
    try {
        plan.addReservation(rAllocation, false);
    } catch (PlanningException e) {
        Assert.fail(e.getMessage());
    }
    doAssertions(plan, rAllocation);
    checkAllocation(plan, alloc1, start);
    // Now add another one
    ReservationId reservationID2 = ReservationSystemTestUtil.getNewReservationId();
    int[] alloc2 = { 0, 5, 10, 5, 0 };
    rAllocation = createReservationAllocation(reservationID2, start, alloc2, true);
    Assert.assertNull(plan.getReservationById(reservationID2));
    try {
        plan.addReservation(rAllocation, false);
    } catch (PlanningException e) {
        Assert.fail(e.getMessage());
    }
    Assert.assertNotNull(plan.getReservationById(reservationID2));
    RLESparseResourceAllocation userCons = plan.getConsumptionForUserOverTime(user, start, start + alloc2.length);
    for (int i = 0; i < alloc2.length; i++) {
        Assert.assertEquals(Resource.newInstance(1024 * (alloc1[i] + alloc2[i] + i), alloc1[i] + alloc2[i] + i), plan.getTotalCommittedResources(start + i));
        Assert.assertEquals(Resource.newInstance(1024 * (alloc1[i] + alloc2[i] + i), alloc1[i] + alloc2[i] + i), userCons.getCapacityAtTime(start + i));
    }
    // Now archive completed reservations
    when(clock.getTime()).thenReturn(106L);
    when(policy.getValidWindow()).thenReturn(1L);
    try {
        // will only remove 2nd reservation as only that has fallen out of the
        // archival window
        plan.archiveCompletedReservations(clock.getTime());
    } catch (PlanningException e) {
        Assert.fail(e.getMessage());
    }
    Assert.assertNotNull(plan.getReservationById(reservationID1));
    Assert.assertNull(plan.getReservationById(reservationID2));
    checkAllocation(plan, alloc1, start);
    when(clock.getTime()).thenReturn(107L);
    try {
        // will remove 1st reservation also as it has fallen out of the archival
        // window
        plan.archiveCompletedReservations(clock.getTime());
    } catch (PlanningException e) {
        Assert.fail(e.getMessage());
    }
    userCons = plan.getConsumptionForUserOverTime(user, start, start + alloc1.length);
    Assert.assertNull(plan.getReservationById(reservationID1));
    for (int i = 0; i < alloc1.length; i++) {
        Assert.assertEquals(Resource.newInstance(0, 0), plan.getTotalCommittedResources(start + i));
        Assert.assertEquals(Resource.newInstance(0, 0), userCons.getCapacityAtTime(start + i));
    }
}

Example 77 with ReservationId

use of org.apache.hadoop.yarn.api.records.ReservationId in project hadoop by apache.

the class TestInMemoryPlan method testGetReservationsWithNoInput.

@Test
public void testGetReservationsWithNoInput() {
    Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, true, context);
    ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId();
    int[] alloc = { 10, 10, 10, 10, 10, 10 };
    int start = 100;
    ReservationAllocation rAllocation = createReservationAllocation(reservationID, start, alloc);
    Assert.assertNull(plan.getReservationById(reservationID));
    try {
        plan.addReservation(rAllocation, false);
    } catch (PlanningException e) {
        Assert.fail(e.getMessage());
    }
    // Verify that getReservations defaults to getting all reservations if no
    // reservationID, time interval, and user is provided,
    Set<ReservationAllocation> rAllocations = plan.getReservations(null, null, "");
    Assert.assertTrue(rAllocations.size() == 1);
    Assert.assertTrue(rAllocation.compareTo((ReservationAllocation) rAllocations.toArray()[0]) == 0);
}

Example 78 with ReservationId

use of org.apache.hadoop.yarn.api.records.ReservationId in project hadoop by apache.

the class TestInMemoryReservationAllocation method testBlocks.

@Test
public void testBlocks() {
    ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong());
    int[] alloc = { 10, 10, 10, 10, 10, 10 };
    int start = 100;
    ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
    Map<ReservationInterval, Resource> allocations = generateAllocation(start, alloc, false, false);
    ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc);
    doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc);
    Assert.assertFalse(rAllocation.containsGangs());
    for (int i = 0; i < alloc.length; i++) {
        Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])), rAllocation.getResourcesAtTime(start + i));
    }
}

Example 79 with ReservationId

use of org.apache.hadoop.yarn.api.records.ReservationId in project hadoop by apache.

the class TestInMemoryReservationAllocation method testSteps.

@Test
public void testSteps() {
    ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong());
    int[] alloc = { 10, 10, 10, 10, 10, 10 };
    int start = 100;
    ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
    Map<ReservationInterval, Resource> allocations = generateAllocation(start, alloc, true, false);
    ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc);
    doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc);
    Assert.assertFalse(rAllocation.containsGangs());
    for (int i = 0; i < alloc.length; i++) {
        Assert.assertEquals(Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)), rAllocation.getResourcesAtTime(start + i));
    }
}

Example 80 with ReservationId

use of org.apache.hadoop.yarn.api.records.ReservationId in project hadoop by apache.

the class TestSchedulerPlanFollowerBase method testPlanFollower.

protected void testPlanFollower(boolean isMove) throws PlanningException, InterruptedException, AccessControlException {
    // Initialize plan based on move flag
    plan = new InMemoryPlan(scheduler.getRootQueueMetrics(), policy, mAgent, scheduler.getClusterResource(), 1L, res, scheduler.getMinimumResourceCapability(), maxAlloc, "dedicated", null, isMove, context);
    // add a few reservations to the plan
    long ts = System.currentTimeMillis();
    ReservationId r1 = ReservationId.newInstance(ts, 1);
    int[] f1 = { 10, 10, 10, 10, 10 };
    ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition(0, 0 + f1.length + 1, f1.length);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r1, rDef, "u3", "dedicated", 0, 0 + f1.length, ReservationSystemTestUtil.generateAllocation(0L, 1L, f1), res, minAlloc), false));
    ReservationId r2 = ReservationId.newInstance(ts, 2);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r2, rDef, "u3", "dedicated", 3, 3 + f1.length, ReservationSystemTestUtil.generateAllocation(3L, 1L, f1), res, minAlloc), false));
    ReservationId r3 = ReservationId.newInstance(ts, 3);
    int[] f2 = { 0, 10, 20, 10, 0 };
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r3, rDef, "u4", "dedicated", 10, 10 + f2.length, ReservationSystemTestUtil.generateAllocation(10L, 1L, f2), res, minAlloc), false));
    AbstractSchedulerPlanFollower planFollower = createPlanFollower();
    when(mClock.getTime()).thenReturn(0L);
    planFollower.run();
    Queue q = getReservationQueue(r1.toString());
    assertReservationQueueExists(r1);
    // submit an app to r1
    String user_0 = "test-user";
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId_0 = ApplicationAttemptId.newInstance(appId, 0);
    AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId, q.getQueueName(), user_0);
    scheduler.handle(addAppEvent);
    AppAttemptAddedSchedulerEvent appAttemptAddedEvent = new AppAttemptAddedSchedulerEvent(appAttemptId_0, false);
    scheduler.handle(appAttemptAddedEvent);
    // initial default reservation queue should have no apps
    Queue defQ = getDefaultQueue();
    Assert.assertEquals(0, getNumberOfApplications(defQ));
    assertReservationQueueExists(r1, 0.1, 0.1);
    Assert.assertEquals(1, getNumberOfApplications(q));
    assertReservationQueueDoesNotExist(r2);
    assertReservationQueueDoesNotExist(r3);
    when(mClock.getTime()).thenReturn(3L);
    planFollower.run();
    Assert.assertEquals(0, getNumberOfApplications(defQ));
    assertReservationQueueExists(r1, 0.1, 0.1);
    Assert.assertEquals(1, getNumberOfApplications(q));
    assertReservationQueueExists(r2, 0.1, 0.1);
    assertReservationQueueDoesNotExist(r3);
    when(mClock.getTime()).thenReturn(10L);
    planFollower.run();
    q = getReservationQueue(r1.toString());
    if (isMove) {
        // app should have been moved to default reservation queue
        Assert.assertEquals(1, getNumberOfApplications(defQ));
        assertNull(q);
    } else {
        // app should be killed
        Assert.assertEquals(0, getNumberOfApplications(defQ));
        assertNotNull(q);
        AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = new AppAttemptRemovedSchedulerEvent(appAttemptId_0, RMAppAttemptState.KILLED, false);
        scheduler.handle(appAttemptRemovedEvent);
    }
    assertReservationQueueDoesNotExist(r2);
    assertReservationQueueExists(r3, 0, 1.0);
    when(mClock.getTime()).thenReturn(11L);
    planFollower.run();
    if (isMove) {
        // app should have been moved to default reservation queue
        Assert.assertEquals(1, getNumberOfApplications(defQ));
    } else {
        // app should be killed
        Assert.assertEquals(0, getNumberOfApplications(defQ));
    }
    assertReservationQueueDoesNotExist(r1);
    assertReservationQueueDoesNotExist(r2);
    assertReservationQueueExists(r3, 0.1, 0.1);
    when(mClock.getTime()).thenReturn(12L);
    planFollower.run();
    assertReservationQueueDoesNotExist(r1);
    assertReservationQueueDoesNotExist(r2);
    assertReservationQueueExists(r3, 0.2, 0.2);
    when(mClock.getTime()).thenReturn(16L);
    planFollower.run();
    assertReservationQueueDoesNotExist(r1);
    assertReservationQueueDoesNotExist(r2);
    assertReservationQueueDoesNotExist(r3);
    verifyCapacity(defQ);
}