Examples with RecordFactory org.apache.hadoop.yarn.factories.RecordFactory used on opensource projects

Example 21 with RecordFactory

use of org.apache.hadoop.yarn.factories.RecordFactory in project hadoop by apache.

the class TestApplicationLimits method testHeadroom.

@Test
public void testHeadroom() throws Exception {
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    csConf.setUserLimit(CapacitySchedulerConfiguration.ROOT + "." + A, 25);
    setupQueueConfiguration(csConf);
    YarnConfiguration conf = new YarnConfiguration();
    CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
    when(csContext.getConfiguration()).thenReturn(csConf);
    when(csContext.getConf()).thenReturn(conf);
    when(csContext.getMinimumResourceCapability()).thenReturn(Resources.createResource(GB));
    when(csContext.getMaximumResourceCapability()).thenReturn(Resources.createResource(16 * GB));
    when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
    when(csContext.getRMContext()).thenReturn(rmContext);
    // Say cluster has 100 nodes of 16G each
    Resource clusterResource = Resources.createResource(100 * 16 * GB);
    when(csContext.getClusterResource()).thenReturn(clusterResource);
    Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
    CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook);
    ResourceUsage queueCapacities = rootQueue.getQueueResourceUsage();
    when(csContext.getClusterResourceUsage()).thenReturn(queueCapacities);
    // Manipulate queue 'a'
    LeafQueue queue = TestLeafQueue.stubLeafQueue((LeafQueue) queues.get(A));
    queue.updateClusterResource(clusterResource, new ResourceLimits(clusterResource));
    String host_0 = "host_0";
    String rack_0 = "rack_0";
    FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, rack_0, 0, 16 * GB);
    final String user_0 = "user_0";
    final String user_1 = "user_1";
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    RMContext rmContext = TestUtils.getMockRMContext();
    RMContext spyRMContext = spy(rmContext);
    ConcurrentMap<ApplicationId, RMApp> spyApps = spy(new ConcurrentHashMap<ApplicationId, RMApp>());
    RMApp rmApp = mock(RMApp.class);
    ResourceRequest amResourceRequest = mock(ResourceRequest.class);
    Resource amResource = Resources.createResource(0, 0);
    when(amResourceRequest.getCapability()).thenReturn(amResource);
    when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
    Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
    when(spyRMContext.getRMApps()).thenReturn(spyApps);
    RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
    when(rmApp.getRMAppAttempt((ApplicationAttemptId) Matchers.any())).thenReturn(rmAppAttempt);
    when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt);
    Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
    Mockito.doReturn(true).when(spyApps).containsKey((ApplicationId) Matchers.any());
    Priority priority_1 = TestUtils.createMockPriority(1);
    // Submit first application with some resource-requests from user_0, 
    // and check headroom
    final ApplicationAttemptId appAttemptId_0_0 = TestUtils.getMockApplicationAttemptId(0, 0);
    FiCaSchedulerApp app_0_0 = new FiCaSchedulerApp(appAttemptId_0_0, user_0, queue, queue.getAbstractUsersManager(), spyRMContext);
    queue.submitApplicationAttempt(app_0_0, user_0);
    List<ResourceRequest> app_0_0_requests = new ArrayList<ResourceRequest>();
    app_0_0_requests.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 2, true, priority_1, recordFactory));
    app_0_0.updateResourceRequests(app_0_0_requests);
    // Schedule to compute 
    queue.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
    Resource expectedHeadroom = Resources.createResource(5 * 16 * GB, 1);
    assertEquals(expectedHeadroom, app_0_0.getHeadroom());
    // Submit second application from user_0, check headroom
    final ApplicationAttemptId appAttemptId_0_1 = TestUtils.getMockApplicationAttemptId(1, 0);
    FiCaSchedulerApp app_0_1 = new FiCaSchedulerApp(appAttemptId_0_1, user_0, queue, queue.getAbstractUsersManager(), spyRMContext);
    queue.submitApplicationAttempt(app_0_1, user_0);
    List<ResourceRequest> app_0_1_requests = new ArrayList<ResourceRequest>();
    app_0_1_requests.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 2, true, priority_1, recordFactory));
    app_0_1.updateResourceRequests(app_0_1_requests);
    // Schedule to compute 
    queue.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), // Schedule to compute
    SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
    assertEquals(expectedHeadroom, app_0_0.getHeadroom());
    // no change
    assertEquals(expectedHeadroom, app_0_1.getHeadroom());
    // Submit first application from user_1, check  for new headroom
    final ApplicationAttemptId appAttemptId_1_0 = TestUtils.getMockApplicationAttemptId(2, 0);
    FiCaSchedulerApp app_1_0 = new FiCaSchedulerApp(appAttemptId_1_0, user_1, queue, queue.getAbstractUsersManager(), spyRMContext);
    queue.submitApplicationAttempt(app_1_0, user_1);
    List<ResourceRequest> app_1_0_requests = new ArrayList<ResourceRequest>();
    app_1_0_requests.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 2, true, priority_1, recordFactory));
    app_1_0.updateResourceRequests(app_1_0_requests);
    // Schedule to compute 
    queue.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), // Schedule to compute
    SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
    // changes
    expectedHeadroom = Resources.createResource(10 * 16 * GB / 2, 1);
    assertEquals(expectedHeadroom, app_0_0.getHeadroom());
    assertEquals(expectedHeadroom, app_0_1.getHeadroom());
    assertEquals(expectedHeadroom, app_1_0.getHeadroom());
    // Now reduce cluster size and check for the smaller headroom
    clusterResource = Resources.createResource(90 * 16 * GB);
    // Any change is cluster resource needs to enforce user-limit recomputation.
    // In existing code, LeafQueue#updateClusterResource handled this. However
    // here that method was not used.
    queue.getUsersManager().userLimitNeedsRecompute();
    queue.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), // Schedule to compute
    SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
    // changes
    expectedHeadroom = Resources.createResource(9 * 16 * GB / 2, 1);
    assertEquals(expectedHeadroom, app_0_0.getHeadroom());
    assertEquals(expectedHeadroom, app_0_1.getHeadroom());
    assertEquals(expectedHeadroom, app_1_0.getHeadroom());
}

Example 22 with RecordFactory

use of org.apache.hadoop.yarn.factories.RecordFactory in project hadoop by apache.

the class TestAbstractYarnScheduler method testNodemanagerReconnect.

/**
   * Test the behavior of the scheduler when a node reconnects
   * with changed capabilities. This test is to catch any race conditions
   * that might occur due to the use of the RMNode object.
   * @throws Exception
   */
@Test(timeout = 60000)
public void testNodemanagerReconnect() throws Exception {
    configureScheduler();
    Configuration conf = getConf();
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, false);
        DrainDispatcher privateDispatcher = new DrainDispatcher();
        SleepHandler sleepHandler = new SleepHandler();
        ResourceTrackerService privateResourceTrackerService = getPrivateResourceTrackerService(privateDispatcher, rm, sleepHandler);
        // Register node1
        String hostname1 = "localhost1";
        Resource capability = BuilderUtils.newResource(4096, 4);
        RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
        RegisterNodeManagerRequest request1 = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
        NodeId nodeId1 = NodeId.newInstance(hostname1, 0);
        request1.setNodeId(nodeId1);
        request1.setHttpPort(0);
        request1.setResource(capability);
        privateResourceTrackerService.registerNodeManager(request1);
        privateDispatcher.await();
        Resource clusterResource = rm.getResourceScheduler().getClusterResource();
        Assert.assertEquals("Initial cluster resources don't match", capability, clusterResource);
        Resource newCapability = BuilderUtils.newResource(1024, 1);
        RegisterNodeManagerRequest request2 = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
        request2.setNodeId(nodeId1);
        request2.setHttpPort(0);
        request2.setResource(newCapability);
        // hold up the disaptcher and register the same node with lower capability
        sleepHandler.sleepFlag = true;
        privateResourceTrackerService.registerNodeManager(request2);
        privateDispatcher.await();
        Assert.assertEquals("Cluster resources don't match", newCapability, rm.getResourceScheduler().getClusterResource());
        privateResourceTrackerService.stop();
    } finally {
        rm.stop();
    }
}

Example 23 with RecordFactory

use of org.apache.hadoop.yarn.factories.RecordFactory in project hadoop by apache.

the class TestAppManager method setUp.

@SuppressWarnings("deprecation")
@Before
public void setUp() {
    long now = System.currentTimeMillis();
    rmContext = mockRMContext(1, now - 10);
    rmContext.setRMTimelineCollectorManager(mock(RMTimelineCollectorManager.class));
    ResourceScheduler scheduler = mockResourceScheduler();
    ((RMContextImpl) rmContext).setScheduler(scheduler);
    Configuration conf = new Configuration();
    ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler);
    appMonitor = new TestRMAppManager(rmContext, new ClientToAMTokenSecretManagerInRM(), scheduler, masterService, new ApplicationACLsManager(conf), conf);
    appId = MockApps.newAppID(1);
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    asContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
    asContext.setApplicationId(appId);
    asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
    asContext.setResource(mockResource());
    asContext.setPriority(Priority.newInstance(0));
    setupDispatcher(rmContext, conf);
}

Example 24 with RecordFactory

use of org.apache.hadoop.yarn.factories.RecordFactory in project hadoop by apache.

the class TestYSCRecordFactory method testPbRecordFactory.

@Test
public void testPbRecordFactory() {
    RecordFactory pbRecordFactory = RecordFactoryPBImpl.get();
    try {
        NodeHeartbeatRequest request = pbRecordFactory.newRecordInstance(NodeHeartbeatRequest.class);
        Assert.assertEquals(NodeHeartbeatRequestPBImpl.class, request.getClass());
    } catch (YarnRuntimeException e) {
        e.printStackTrace();
        Assert.fail("Failed to crete record");
    }
}

Example 25 with RecordFactory

use of org.apache.hadoop.yarn.factories.RecordFactory in project hadoop by apache.

the class TestNodeHealthService method testNodeHealthService.

@Test
public void testNodeHealthService() throws Exception {
    RecordFactory factory = RecordFactoryProvider.getRecordFactory(null);
    NodeHealthStatus healthStatus = factory.newRecordInstance(NodeHealthStatus.class);
    Configuration conf = getConfForNodeHealthScript();
    conf.writeXml(new FileOutputStream(nodeHealthConfigFile));
    conf.addResource(nodeHealthConfigFile.getName());
    writeNodeHealthScriptFile("", true);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    NodeHealthScriptRunner nodeHealthScriptRunner = spy(NodeManager.getNodeHealthScriptRunner(conf));
    NodeHealthCheckerService nodeHealthChecker = new NodeHealthCheckerService(nodeHealthScriptRunner, dirsHandler);
    nodeHealthChecker.init(conf);
    doReturn(true).when(nodeHealthScriptRunner).isHealthy();
    doReturn("").when(nodeHealthScriptRunner).getHealthReport();
    setHealthStatus(healthStatus, nodeHealthChecker.isHealthy(), nodeHealthChecker.getHealthReport(), nodeHealthChecker.getLastHealthReportTime());
    LOG.info("Checking initial healthy condition");
    // Check proper report conditions.
    Assert.assertTrue("Node health status reported unhealthy", healthStatus.getIsNodeHealthy());
    Assert.assertTrue("Node health status reported unhealthy", healthStatus.getHealthReport().equals(nodeHealthChecker.getHealthReport()));
    doReturn(false).when(nodeHealthScriptRunner).isHealthy();
    // update health status
    setHealthStatus(healthStatus, nodeHealthChecker.isHealthy(), nodeHealthChecker.getHealthReport(), nodeHealthChecker.getLastHealthReportTime());
    LOG.info("Checking Healthy--->Unhealthy");
    Assert.assertFalse("Node health status reported healthy", healthStatus.getIsNodeHealthy());
    Assert.assertTrue("Node health status reported healthy", healthStatus.getHealthReport().equals(nodeHealthChecker.getHealthReport()));
    doReturn(true).when(nodeHealthScriptRunner).isHealthy();
    setHealthStatus(healthStatus, nodeHealthChecker.isHealthy(), nodeHealthChecker.getHealthReport(), nodeHealthChecker.getLastHealthReportTime());
    LOG.info("Checking UnHealthy--->healthy");
    // Check proper report conditions.
    Assert.assertTrue("Node health status reported unhealthy", healthStatus.getIsNodeHealthy());
    Assert.assertTrue("Node health status reported unhealthy", healthStatus.getHealthReport().equals(nodeHealthChecker.getHealthReport()));
    // Healthy to timeout transition.
    doReturn(false).when(nodeHealthScriptRunner).isHealthy();
    doReturn(NodeHealthScriptRunner.NODE_HEALTH_SCRIPT_TIMED_OUT_MSG).when(nodeHealthScriptRunner).getHealthReport();
    setHealthStatus(healthStatus, nodeHealthChecker.isHealthy(), nodeHealthChecker.getHealthReport(), nodeHealthChecker.getLastHealthReportTime());
    LOG.info("Checking Healthy--->timeout");
    Assert.assertFalse("Node health status reported healthy even after timeout", healthStatus.getIsNodeHealthy());
    Assert.assertTrue("Node script time out message not propagated", healthStatus.getHealthReport().equals(NodeHealthScriptRunner.NODE_HEALTH_SCRIPT_TIMED_OUT_MSG + NodeHealthCheckerService.SEPARATOR + nodeHealthChecker.getDiskHandler().getDisksHealthReport(false)));
}