Examples with DefaultResourceCalculator org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator used on opensource projects

Example 11 with DefaultResourceCalculator

use of org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator in project hadoop by apache.

the class TestReservationInputValidator method setUp.

@Before
public void setUp() {
    clock = mock(Clock.class);
    plan = mock(Plan.class);
    rSystem = mock(ReservationSystem.class);
    plans.put(PLAN_NAME, plan);
    rrValidator = new ReservationInputValidator(clock);
    when(clock.getTime()).thenReturn(1L);
    ResourceCalculator rCalc = new DefaultResourceCalculator();
    Resource resource = Resource.newInstance(10240, 10);
    when(plan.getResourceCalculator()).thenReturn(rCalc);
    when(plan.getTotalCapacity()).thenReturn(resource);
    when(rSystem.getQueueForReservation(any(ReservationId.class))).thenReturn(PLAN_NAME);
    when(rSystem.getPlan(PLAN_NAME)).thenReturn(plan);
}

Example 12 with DefaultResourceCalculator

use of org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator in project hadoop by apache.

the class TestUtils method applyResourceCommitRequest.

public static void applyResourceCommitRequest(Resource clusterResource, CSAssignment csAssignment, final Map<NodeId, FiCaSchedulerNode> nodes, final Map<ApplicationAttemptId, FiCaSchedulerApp> apps) throws IOException {
    CapacityScheduler cs = new CapacityScheduler() {

        @Override
        public FiCaSchedulerNode getNode(NodeId nodeId) {
            return nodes.get(nodeId);
        }

        @Override
        public FiCaSchedulerApp getApplicationAttempt(ApplicationAttemptId applicationAttemptId) {
            return apps.get(applicationAttemptId);
        }
    };
    cs.setResourceCalculator(new DefaultResourceCalculator());
    cs.submitResourceCommitRequest(clusterResource, csAssignment);
}

Example 13 with DefaultResourceCalculator

use of org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator in project hadoop by apache.

the class TestSimpleCapacityReplanner method testReplanningPlanCapacityLoss.

@Test
public void testReplanningPlanCapacityLoss() throws PlanningException {
    Resource clusterCapacity = Resource.newInstance(100 * 1024, 100);
    Resource minAlloc = Resource.newInstance(1024, 1);
    Resource maxAlloc = Resource.newInstance(1024 * 8, 8);
    ResourceCalculator res = new DefaultResourceCalculator();
    long step = 1L;
    Clock clock = mock(Clock.class);
    ReservationAgent agent = mock(ReservationAgent.class);
    SharingPolicy policy = new NoOverCommitPolicy();
    policy.init("root.dedicated", null);
    QueueMetrics queueMetrics = mock(QueueMetrics.class);
    when(clock.getTime()).thenReturn(0L);
    SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock);
    RMContext context = ReservationSystemTestUtil.createMockRMContext();
    ReservationSchedulerConfiguration conf = mock(ReservationSchedulerConfiguration.class);
    when(conf.getEnforcementWindow(any(String.class))).thenReturn(6L);
    enf.init("blah", conf);
    // Initialize the plan with more resources
    InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", enf, true, context, clock);
    // add reservation filling the plan (separating them 1ms, so we are sure
    // s2 follows s1 on acceptance
    long ts = System.currentTimeMillis();
    ReservationId r1 = ReservationId.newInstance(ts, 1);
    int[] f5 = { 20, 20, 20, 20, 20 };
    ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition(0, 0 + f5.length, f5.length);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r1, rDef, "u3", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc), false));
    when(clock.getTime()).thenReturn(1L);
    ReservationId r2 = ReservationId.newInstance(ts, 2);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r2, rDef, "u4", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc), false));
    when(clock.getTime()).thenReturn(2L);
    ReservationId r3 = ReservationId.newInstance(ts, 3);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r3, rDef, "u5", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc), false));
    when(clock.getTime()).thenReturn(3L);
    ReservationId r4 = ReservationId.newInstance(ts, 4);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r4, rDef, "u6", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc), false));
    when(clock.getTime()).thenReturn(4L);
    ReservationId r5 = ReservationId.newInstance(ts, 5);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r5, rDef, "u7", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc), false));
    int[] f6 = { 50, 50, 50, 50, 50 };
    ReservationId r6 = ReservationId.newInstance(ts, 6);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r6, rDef, "u3", "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res, minAlloc), false));
    when(clock.getTime()).thenReturn(6L);
    ReservationId r7 = ReservationId.newInstance(ts, 7);
    assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r7, rDef, "u4", "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res, minAlloc), false));
    // remove some of the resources (requires replanning)
    plan.setTotalCapacity(Resource.newInstance(70 * 1024, 70));
    when(clock.getTime()).thenReturn(0L);
    // run the replanner
    enf.plan(plan, null);
    // check which reservation are still present
    assertNotNull(plan.getReservationById(r1));
    assertNotNull(plan.getReservationById(r2));
    assertNotNull(plan.getReservationById(r3));
    assertNotNull(plan.getReservationById(r6));
    assertNotNull(plan.getReservationById(r7));
    // and which ones are removed
    assertNull(plan.getReservationById(r4));
    assertNull(plan.getReservationById(r5));
    // check resources at each moment in time no more exceed capacity
    for (int i = 0; i < 20; i++) {
        long tot = 0;
        for (ReservationAllocation r : plan.getReservationsAtTime(i)) {
            tot = r.getResourcesAtTime(i).getMemorySize();
        }
        assertTrue(tot <= 70 * 1024);
    }
}

Example 14 with DefaultResourceCalculator

use of org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator in project hadoop by apache.

the class TestCapacityOverTimePolicy method setup.

@Before
public void setup() throws Exception {
    // 24h window
    timeWindow = 86400000L;
    // 1 sec step
    step = 1000L;
    // 25% avg cap on capacity
    avgConstraint = 25;
    // 70% instantaneous cap on capacity
    instConstraint = 70;
    initTime = System.currentTimeMillis();
    minAlloc = Resource.newInstance(1024, 1);
    res = new DefaultResourceCalculator();
    maxAlloc = Resource.newInstance(1024 * 8, 8);
    mAgent = mock(ReservationAgent.class);
    QueueMetrics rootQueueMetrics = mock(QueueMetrics.class);
    String reservationQ = ReservationSystemTestUtil.getFullReservationQueueName();
    Resource clusterResource = ReservationSystemTestUtil.calculateClusterResource(totCont);
    ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.createConf(reservationQ, timeWindow, instConstraint, avgConstraint);
    CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
    policy.init(reservationQ, conf);
    RMContext context = ReservationSystemTestUtil.createMockRMContext();
    plan = new InMemoryPlan(rootQueueMetrics, policy, mAgent, clusterResource, step, res, minAlloc, maxAlloc, "dedicated", null, true, context);
}

Example 15 with DefaultResourceCalculator

use of org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator in project hadoop by apache.

the class TestCapacitySchedulerPlanFollower method setUp.

@Before
public void setUp() throws Exception {
    CapacityScheduler spyCs = new CapacityScheduler();
    cs = spy(spyCs);
    scheduler = cs;
    rmContext = TestUtils.getMockRMContext();
    spyRMContext = spy(rmContext);
    ConcurrentMap<ApplicationId, RMApp> spyApps = spy(new ConcurrentHashMap<ApplicationId, RMApp>());
    RMApp rmApp = mock(RMApp.class);
    RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
    when(rmApp.getRMAppAttempt((ApplicationAttemptId) Matchers.any())).thenReturn(rmAppAttempt);
    when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt);
    Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
    Mockito.doReturn(true).when(spyApps).containsKey((ApplicationId) Matchers.any());
    when(spyRMContext.getRMApps()).thenReturn(spyApps);
    when(spyRMContext.getScheduler()).thenReturn(scheduler);
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    ReservationSystemTestUtil.setupQueueConfiguration(csConf);
    cs.setConf(csConf);
    csContext = mock(CapacitySchedulerContext.class);
    when(csContext.getConfiguration()).thenReturn(csConf);
    when(csContext.getConf()).thenReturn(csConf);
    when(csContext.getMinimumResourceCapability()).thenReturn(minAlloc);
    when(csContext.getMaximumResourceCapability()).thenReturn(maxAlloc);
    when(csContext.getClusterResource()).thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32));
    when(scheduler.getClusterResource()).thenReturn(Resources.createResource(125 * GB, 125));
    when(csContext.getResourceCalculator()).thenReturn(new DefaultResourceCalculator());
    RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(csConf);
    containerTokenSecretManager.rollMasterKey();
    when(csContext.getContainerTokenSecretManager()).thenReturn(containerTokenSecretManager);
    cs.setRMContext(spyRMContext);
    cs.init(csConf);
    cs.start();
    setupPlanFollower();
}