Examples with ResourceCalculatorPlugin org.apache.hadoop.yarn.util.ResourceCalculatorPlugin used on opensource projects

Example 6 with ResourceCalculatorPlugin

use of org.apache.hadoop.yarn.util.ResourceCalculatorPlugin in project hadoop by apache.

the class TestCgroupsLCEResourcesHandler method testInit.

@Test
public void testInit() throws IOException {
    LinuxContainerExecutor mockLCE = new MockLinuxContainerExecutor();
    CustomCgroupsLCEResourceHandler handler = new CustomCgroupsLCEResourceHandler();
    YarnConfiguration conf = new YarnConfiguration();
    final int numProcessors = 4;
    ResourceCalculatorPlugin plugin = Mockito.mock(ResourceCalculatorPlugin.class);
    Mockito.doReturn(numProcessors).when(plugin).getNumProcessors();
    Mockito.doReturn(numProcessors).when(plugin).getNumCores();
    handler.setConf(conf);
    handler.initConfig();
    // create mock cgroup
    File cpuCgroupMountDir = TestCGroupsHandlerImpl.createMockCgroupMount(cgroupDir, "cpu");
    // create mock mtab
    File mockMtab = TestCGroupsHandlerImpl.createMockMTab(cgroupDir);
    // setup our handler and call init()
    handler.setMtabFile(mockMtab.getAbsolutePath());
    // check values
    // in this case, we're using all cpu so the files
    // shouldn't exist(because init won't create them
    handler.init(mockLCE, plugin);
    File periodFile = new File(cpuCgroupMountDir, "cpu.cfs_period_us");
    File quotaFile = new File(cpuCgroupMountDir, "cpu.cfs_quota_us");
    Assert.assertFalse(periodFile.exists());
    Assert.assertFalse(quotaFile.exists());
    // subset of cpu being used, files should be created
    conf.setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, 75);
    handler.limits[0] = 100 * 1000;
    handler.limits[1] = 1000 * 1000;
    handler.init(mockLCE, plugin);
    int period = readIntFromFile(periodFile);
    int quota = readIntFromFile(quotaFile);
    Assert.assertEquals(100 * 1000, period);
    Assert.assertEquals(1000 * 1000, quota);
    // set cpu back to 100, quota should be -1
    conf.setInt(YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT, 100);
    handler.limits[0] = 100 * 1000;
    handler.limits[1] = 1000 * 1000;
    handler.init(mockLCE, plugin);
    quota = readIntFromFile(quotaFile);
    Assert.assertEquals(-1, quota);
    FileUtils.deleteQuietly(cgroupDir);
}

Example 7 with ResourceCalculatorPlugin

use of org.apache.hadoop.yarn.util.ResourceCalculatorPlugin in project hadoop by apache.

the class TestNodeManagerHardwareUtils method testGetContainerMemoryMB.

@Test
public void testGetContainerMemoryMB() throws Exception {
    ResourceCalculatorPlugin plugin = new TestResourceCalculatorPlugin();
    long physicalMemMB = plugin.getPhysicalMemorySize() / (1024 * 1024);
    YarnConfiguration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION, true);
    int mem = NodeManagerHardwareUtils.getContainerMemoryMB(null, conf);
    Assert.assertEquals(YarnConfiguration.DEFAULT_NM_PMEM_MB, mem);
    mem = NodeManagerHardwareUtils.getContainerMemoryMB(plugin, conf);
    int hadoopHeapSizeMB = (int) (Runtime.getRuntime().maxMemory() / (1024 * 1024));
    int calculatedMemMB = (int) (0.8 * (physicalMemMB - (2 * hadoopHeapSizeMB)));
    Assert.assertEquals(calculatedMemMB, mem);
    conf.setInt(YarnConfiguration.NM_PMEM_MB, 1024);
    mem = NodeManagerHardwareUtils.getContainerMemoryMB(conf);
    Assert.assertEquals(1024, mem);
    conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION, false);
    mem = NodeManagerHardwareUtils.getContainerMemoryMB(conf);
    Assert.assertEquals(YarnConfiguration.DEFAULT_NM_PMEM_MB, mem);
    conf.setInt(YarnConfiguration.NM_PMEM_MB, 10 * 1024);
    mem = NodeManagerHardwareUtils.getContainerMemoryMB(conf);
    Assert.assertEquals(10 * 1024, mem);
}

Example 8 with ResourceCalculatorPlugin

use of org.apache.hadoop.yarn.util.ResourceCalculatorPlugin in project hadoop by apache.

the class TestNodeManagerHardwareUtils method testGetVCores.

@Test
public void testGetVCores() {
    ResourceCalculatorPlugin plugin = new TestResourceCalculatorPlugin();
    YarnConfiguration conf = new YarnConfiguration();
    conf.setFloat(YarnConfiguration.NM_PCORES_VCORES_MULTIPLIER, 1.25f);
    int ret = NodeManagerHardwareUtils.getVCores(plugin, conf);
    Assert.assertEquals(YarnConfiguration.DEFAULT_NM_VCORES, ret);
    conf.setBoolean(YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION, true);
    ret = NodeManagerHardwareUtils.getVCores(plugin, conf);
    Assert.assertEquals(5, ret);
    conf.setBoolean(YarnConfiguration.NM_COUNT_LOGICAL_PROCESSORS_AS_CORES, true);
    ret = NodeManagerHardwareUtils.getVCores(plugin, conf);
    Assert.assertEquals(10, ret);
    conf.setInt(YarnConfiguration.NM_VCORES, 10);
    ret = NodeManagerHardwareUtils.getVCores(plugin, conf);
    Assert.assertEquals(10, ret);
    YarnConfiguration conf1 = new YarnConfiguration();
    conf1.setBoolean(YarnConfiguration.NM_ENABLE_HARDWARE_CAPABILITY_DETECTION, false);
    conf.setInt(YarnConfiguration.NM_VCORES, 10);
    ret = NodeManagerHardwareUtils.getVCores(plugin, conf);
    Assert.assertEquals(10, ret);
}