Examples with RMContext org.apache.hadoop.yarn.server.resourcemanager.RMContext used on opensource projects

Example 16 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestReservationQueue method setup.

@Before
public void setup() throws IOException {
    // setup a context / conf
    csConf = new CapacitySchedulerConfiguration();
    YarnConfiguration conf = new YarnConfiguration();
    csContext = mock(CapacitySchedulerContext.class);
    when(csContext.getConfiguration()).thenReturn(csConf);
    when(csContext.getConf()).thenReturn(conf);
    when(csContext.getMinimumResourceCapability()).thenReturn(Resources.createResource(GB, 1));
    when(csContext.getMaximumResourceCapability()).thenReturn(Resources.createResource(16 * GB, 32));
    when(csContext.getClusterResource()).thenReturn(Resources.createResource(100 * 16 * GB, 100 * 32));
    when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
    RMContext mockRMContext = TestUtils.getMockRMContext();
    when(csContext.getRMContext()).thenReturn(mockRMContext);
    // create a queue
    PlanQueue pq = new PlanQueue(csContext, "root", null, null);
    reservationQueue = new ReservationQueue(csContext, "a", pq);
}

Example 17 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestMiniYarnClusterNodeUtilization method verifySimulatedUtilization.

/**
   * Verify both the RMNode and SchedulerNode have been updated with the test
   * fixture utilization data.
   */
private void verifySimulatedUtilization() throws InterruptedException {
    ResourceManager rm = cluster.getResourceManager(0);
    RMContext rmContext = rm.getRMContext();
    ResourceUtilization containersUtilization = nodeStatus.getContainersUtilization();
    ResourceUtilization nodeUtilization = nodeStatus.getNodeUtilization();
    // We check if the nodeUtilization is up to date
    for (int i = 0; i < 100; i++) {
        for (RMNode ni : rmContext.getRMNodes().values()) {
            if (ni.getNodeUtilization() != null) {
                if (ni.getNodeUtilization().equals(nodeUtilization)) {
                    break;
                }
            }
        }
        Thread.sleep(100);
    }
    // Verify the data is readable from the RM and scheduler nodes
    for (RMNode ni : rmContext.getRMNodes().values()) {
        ResourceUtilization cu = ni.getAggregatedContainersUtilization();
        assertEquals("Containers Utillization not propagated to RMNode", containersUtilization, cu);
        ResourceUtilization nu = ni.getNodeUtilization();
        assertEquals("Node Utillization not propagated to RMNode", nodeUtilization, nu);
        SchedulerNode scheduler = rmContext.getScheduler().getSchedulerNode(ni.getNodeID());
        cu = scheduler.getAggregatedContainersUtilization();
        assertEquals("Containers Utillization not propagated to SchedulerNode", containersUtilization, cu);
        nu = scheduler.getNodeUtilization();
        assertEquals("Node Utillization not propagated to SchedulerNode", nodeUtilization, nu);
    }
}

Example 18 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestRMNMRPCResponseId method setUp.

@Before
public void setUp() {
    Configuration conf = new Configuration();
    // Dispatcher that processes events inline
    Dispatcher dispatcher = new InlineDispatcher();
    dispatcher.register(SchedulerEventType.class, new EventHandler<Event>() {

        @Override
        public void handle(Event event) {
            // ignore
            ;
        }
    });
    RMContext context = new RMContextImpl(dispatcher, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), null, null);
    dispatcher.register(RMNodeEventType.class, new ResourceManager.NodeEventDispatcher(context));
    NodesListManager nodesListManager = new NodesListManager(context);
    nodesListManager.init(conf);
    context.getContainerTokenSecretManager().rollMasterKey();
    context.getNMTokenSecretManager().rollMasterKey();
    resourceTrackerService = new ResourceTrackerService(context, nodesListManager, new NMLivelinessMonitor(dispatcher), context.getContainerTokenSecretManager(), context.getNMTokenSecretManager());
    resourceTrackerService.init(conf);
}

Example 19 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestGreedyReservationAgent method setup.

@Before
public void setup() throws Exception {
    long seed = rand.nextLong();
    rand.setSeed(seed);
    Log.getLog().info("Running with seed: " + seed);
    // setting completely loose quotas
    long timeWindow = 1000000L;
    Resource clusterCapacity = Resource.newInstance(100 * 1024, 100);
    step = 1000L;
    String reservationQ = ReservationSystemTestUtil.getFullReservationQueueName();
    float instConstraint = 100;
    float avgConstraint = 100;
    ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.createConf(reservationQ, timeWindow, instConstraint, avgConstraint);
    CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
    policy.init(reservationQ, conf);
    // setting conf to
    conf.setBoolean(GreedyReservationAgent.GREEDY_FAVOR_EARLY_ALLOCATION, allocateLeft);
    agent = new GreedyReservationAgent();
    agent.init(conf);
    QueueMetrics queueMetrics = mock(QueueMetrics.class);
    RMContext context = ReservationSystemTestUtil.createMockRMContext();
    plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", null, true, context);
}

Example 20 with RMContext

use of org.apache.hadoop.yarn.server.resourcemanager.RMContext in project hadoop by apache.

the class TestGreedyReservationAgent method testStress.

public void testStress(int numJobs) throws PlanningException, IOException {
    long timeWindow = 1000000L;
    Resource clusterCapacity = Resource.newInstance(500 * 100 * 1024, 500 * 32);
    step = 1000L;
    ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
    CapacityScheduler scheduler = testUtil.mockCapacityScheduler(500 * 100);
    String reservationQ = ReservationSystemTestUtil.getFullReservationQueueName();
    float instConstraint = 100;
    float avgConstraint = 100;
    ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.createConf(reservationQ, timeWindow, instConstraint, avgConstraint);
    CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
    policy.init(reservationQ, conf);
    RMContext context = ReservationSystemTestUtil.createMockRMContext();
    plan = new InMemoryPlan(scheduler.getRootQueueMetrics(), policy, agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", null, true, context);
    int acc = 0;
    List<ReservationDefinition> list = new ArrayList<ReservationDefinition>();
    for (long i = 0; i < numJobs; i++) {
        list.add(ReservationSystemTestUtil.generateRandomRR(rand, i));
    }
    long start = System.currentTimeMillis();
    for (int i = 0; i < numJobs; i++) {
        try {
            if (agent.createReservation(ReservationSystemTestUtil.getNewReservationId(), "u" + i % 100, plan, list.get(i))) {
                acc++;
            }
        } catch (PlanningException p) {
        // ignore exceptions
        }
    }
    long end = System.currentTimeMillis();
    System.out.println("Submitted " + numJobs + " jobs " + " accepted " + acc + " in " + (end - start) + "ms");
}