Examples with ResourceUsageMetrics org.apache.hadoop.tools.rumen.ResourceUsageMetrics used on opensource projects

Example 6 with ResourceUsageMetrics

use of org.apache.hadoop.tools.rumen.ResourceUsageMetrics in project hadoop by apache.

the class TestGridmixMemoryEmulation method testTotalHeapUsageEmulatorPlugin.

/**
   * Test {@link TotalHeapUsageEmulatorPlugin}.
   */
@Test
public void testTotalHeapUsageEmulatorPlugin() throws Exception {
    Configuration conf = new Configuration();
    // set the dummy resource calculator for testing
    ResourceCalculatorPlugin monitor = new DummyResourceCalculatorPlugin();
    // 1GB
    long maxHeapUsage = 1024 * TotalHeapUsageEmulatorPlugin.ONE_MB;
    conf.setLong(DummyResourceCalculatorPlugin.MAXPMEM_TESTING_PROPERTY, maxHeapUsage);
    monitor.setConf(conf);
    // no buffer to be reserved
    conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0F);
    // only 1 call to be made per cycle
    conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 1F);
    // 200mb
    long targetHeapUsageInMB = 200;
    // fake progress indicator
    FakeProgressive fakeProgress = new FakeProgressive();
    // fake heap usage generator
    FakeHeapUsageEmulatorCore fakeCore = new FakeHeapUsageEmulatorCore();
    // a heap usage emulator with fake core
    FakeHeapUsageEmulatorPlugin heapPlugin = new FakeHeapUsageEmulatorPlugin(fakeCore);
    // test with invalid or missing resource usage value
    ResourceUsageMetrics invalidUsage = TestResourceUsageEmulators.createMetrics(0);
    heapPlugin.initialize(conf, invalidUsage, null, null);
    // test if disabled heap emulation plugin's emulate() call is a no-operation
    // this will test if the emulation plugin is disabled or not
    int numCallsPre = fakeCore.getNumCalls();
    long heapUsagePre = fakeCore.getHeapUsageInMB();
    heapPlugin.emulate();
    int numCallsPost = fakeCore.getNumCalls();
    long heapUsagePost = fakeCore.getHeapUsageInMB();
    //  test if no calls are made heap usage emulator core
    assertEquals("Disabled heap usage emulation plugin works!", numCallsPre, numCallsPost);
    //  test if no calls are made heap usage emulator core
    assertEquals("Disabled heap usage emulation plugin works!", heapUsagePre, heapUsagePost);
    // test with get progress
    float progress = heapPlugin.getProgress();
    assertEquals("Invalid progress of disabled cumulative heap usage emulation " + "plugin!", 1.0f, progress, 0f);
    // test with wrong/invalid configuration
    Boolean failed = null;
    invalidUsage = TestResourceUsageEmulators.createMetrics(maxHeapUsage + TotalHeapUsageEmulatorPlugin.ONE_MB);
    try {
        heapPlugin.initialize(conf, invalidUsage, monitor, null);
        failed = false;
    } catch (Exception e) {
        failed = true;
    }
    assertNotNull("Fail case failure!", failed);
    assertTrue("Expected failure!", failed);
    // test with valid resource usage value
    ResourceUsageMetrics metrics = TestResourceUsageEmulators.createMetrics(targetHeapUsageInMB * TotalHeapUsageEmulatorPlugin.ONE_MB);
    // test with default emulation interval
    // in every interval, the emulator will add 100% of the expected usage 
    // (since gridmix.emulators.resource-usage.heap.load-ratio=1)
    // so at 10%, emulator will add 10% (difference), at 20% it will add 10% ...
    // So to emulate 200MB, it will add
    //   20mb + 20mb + 20mb + 20mb + .. = 200mb 
    testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 200, 10);
    // test with custom value for emulation interval of 20%
    conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_EMULATION_PROGRESS_INTERVAL, 0.2F);
    //  40mb + 40mb + 40mb + 40mb + 40mb = 200mb
    testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 200, 5);
    // test with custom value of free heap ratio and load ratio = 1
    conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 1F);
    conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0.5F);
    //  40mb + 0mb + 80mb + 0mb + 0mb = 120mb
    testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 120, 2);
    // test with custom value of heap load ratio and min free heap ratio = 0
    conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 0.5F);
    conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0F);
    // 20mb (call#1) + 20mb (call#1) + 20mb (call#2) + 20mb (call#2) +.. = 200mb
    testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 200, 10);
    // test with custom value of free heap ratio = 0.3 and load ratio = 0.5
    conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0.25F);
    conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 0.5F);
    // 20mb (call#1) + 20mb (call#1) + 30mb (call#2) + 0mb (call#2) 
    // + 30mb (call#3) + 0mb (call#3) + 35mb (call#4) + 0mb (call#4)
    // + 37mb (call#5) + 0mb (call#5) = 162mb
    testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 162, 6);
    // test if emulation interval boundary is respected
    // initialize
    fakeProgress = new FakeProgressive();
    conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0F);
    conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 1F);
    conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_EMULATION_PROGRESS_INTERVAL, 0.25F);
    heapPlugin.initialize(conf, metrics, monitor, fakeProgress);
    fakeCore.resetFake();
    // take a snapshot after the initialization
    long initHeapUsage = fakeCore.getHeapUsageInMB();
    long initNumCallsUsage = fakeCore.getNumCalls();
    // test with 0 progress
    testEmulationBoundary(0F, fakeCore, fakeProgress, heapPlugin, initHeapUsage, initNumCallsUsage, "[no-op, 0 progress]");
    // test with 24% progress
    testEmulationBoundary(0.24F, fakeCore, fakeProgress, heapPlugin, initHeapUsage, initNumCallsUsage, "[no-op, 24% progress]");
    // test with 25% progress
    testEmulationBoundary(0.25F, fakeCore, fakeProgress, heapPlugin, targetHeapUsageInMB / 4, 1, "[op, 25% progress]");
    // test with 80% progress
    testEmulationBoundary(0.80F, fakeCore, fakeProgress, heapPlugin, (targetHeapUsageInMB * 4) / 5, 2, "[op, 80% progress]");
    // now test if the final call with 100% progress ramps up the heap usage
    testEmulationBoundary(1F, fakeCore, fakeProgress, heapPlugin, targetHeapUsageInMB, 3, "[op, 100% progress]");
}

Example 7 with ResourceUsageMetrics

use of org.apache.hadoop.tools.rumen.ResourceUsageMetrics in project hadoop by apache.

the class LoadJob method buildSplits.

@Override
void buildSplits(FilePool inputDir) throws IOException {
    long mapInputBytesTotal = 0L;
    long mapOutputBytesTotal = 0L;
    long mapOutputRecordsTotal = 0L;
    final JobStory jobdesc = getJobDesc();
    if (null == jobdesc) {
        return;
    }
    final int maps = jobdesc.getNumberMaps();
    final int reds = jobdesc.getNumberReduces();
    for (int i = 0; i < maps; ++i) {
        final TaskInfo info = jobdesc.getTaskInfo(TaskType.MAP, i);
        mapInputBytesTotal += info.getInputBytes();
        mapOutputBytesTotal += info.getOutputBytes();
        mapOutputRecordsTotal += info.getOutputRecords();
    }
    final double[] reduceRecordRatio = new double[reds];
    final double[] reduceByteRatio = new double[reds];
    for (int i = 0; i < reds; ++i) {
        final TaskInfo info = jobdesc.getTaskInfo(TaskType.REDUCE, i);
        reduceByteRatio[i] = info.getInputBytes() / (1.0 * mapOutputBytesTotal);
        reduceRecordRatio[i] = info.getInputRecords() / (1.0 * mapOutputRecordsTotal);
    }
    final InputStriper striper = new InputStriper(inputDir, mapInputBytesTotal);
    final List<InputSplit> splits = new ArrayList<InputSplit>();
    for (int i = 0; i < maps; ++i) {
        final int nSpec = reds / maps + ((reds % maps) > i ? 1 : 0);
        final long[] specBytes = new long[nSpec];
        final long[] specRecords = new long[nSpec];
        final ResourceUsageMetrics[] metrics = new ResourceUsageMetrics[nSpec];
        for (int j = 0; j < nSpec; ++j) {
            final TaskInfo info = jobdesc.getTaskInfo(TaskType.REDUCE, i + j * maps);
            specBytes[j] = info.getOutputBytes();
            specRecords[j] = info.getOutputRecords();
            metrics[j] = info.getResourceUsageMetrics();
            if (LOG.isDebugEnabled()) {
                LOG.debug(String.format("SPEC(%d) %d -> %d %d %d %d %d %d %d", id(), i, i + j * maps, info.getOutputRecords(), info.getOutputBytes(), info.getResourceUsageMetrics().getCumulativeCpuUsage(), info.getResourceUsageMetrics().getPhysicalMemoryUsage(), info.getResourceUsageMetrics().getVirtualMemoryUsage(), info.getResourceUsageMetrics().getHeapUsage()));
            }
        }
        final TaskInfo info = jobdesc.getTaskInfo(TaskType.MAP, i);
        long possiblyCompressedInputBytes = info.getInputBytes();
        Configuration conf = job.getConfiguration();
        long uncompressedInputBytes = CompressionEmulationUtil.getUncompressedInputBytes(possiblyCompressedInputBytes, conf);
        splits.add(new LoadSplit(striper.splitFor(inputDir, uncompressedInputBytes, 3), maps, i, uncompressedInputBytes, info.getInputRecords(), info.getOutputBytes(), info.getOutputRecords(), reduceByteRatio, reduceRecordRatio, specBytes, specRecords, info.getResourceUsageMetrics(), metrics));
    }
    pushDescription(id(), splits);
}

Example 8 with ResourceUsageMetrics

use of org.apache.hadoop.tools.rumen.ResourceUsageMetrics in project hadoop by apache.

the class TestResourceUsageEmulators method testCumulativeCpuUsageEmulatorPlugin.

/**
   * Test {@link CumulativeCpuUsageEmulatorPlugin}.
   */
@Test
public void testCumulativeCpuUsageEmulatorPlugin() throws Exception {
    Configuration conf = new Configuration();
    long targetCpuUsage = 1000L;
    int unitCpuUsage = 50;
    // fake progress indicator
    FakeProgressive fakeProgress = new FakeProgressive();
    // fake cpu usage generator
    FakeCpuUsageEmulatorCore fakeCore = new FakeCpuUsageEmulatorCore();
    fakeCore.setUnitUsage(unitCpuUsage);
    // a cumulative cpu usage emulator with fake core
    CumulativeCpuUsageEmulatorPlugin cpuPlugin = new CumulativeCpuUsageEmulatorPlugin(fakeCore);
    // test with invalid or missing resource usage value
    ResourceUsageMetrics invalidUsage = createMetrics(0);
    cpuPlugin.initialize(conf, invalidUsage, null, null);
    // test if disabled cpu emulation plugin's emulate() call is a no-operation
    // this will test if the emulation plugin is disabled or not
    int numCallsPre = fakeCore.getNumCalls();
    long cpuUsagePre = fakeCore.getCpuUsage();
    cpuPlugin.emulate();
    int numCallsPost = fakeCore.getNumCalls();
    long cpuUsagePost = fakeCore.getCpuUsage();
    //  test if no calls are made cpu usage emulator core
    assertEquals("Disabled cumulative CPU usage emulation plugin works!", numCallsPre, numCallsPost);
    //  test if no calls are made cpu usage emulator core
    assertEquals("Disabled cumulative CPU usage emulation plugin works!", cpuUsagePre, cpuUsagePost);
    // test with get progress
    float progress = cpuPlugin.getProgress();
    assertEquals("Invalid progress of disabled cumulative CPU usage emulation " + "plugin!", 1.0f, progress, 0f);
    // test with valid resource usage value
    ResourceUsageMetrics metrics = createMetrics(targetCpuUsage);
    // fake monitor
    ResourceCalculatorPlugin monitor = new FakeResourceUsageMonitor(fakeCore);
    // test with default emulation interval
    testEmulationAccuracy(conf, fakeCore, monitor, metrics, cpuPlugin, targetCpuUsage, targetCpuUsage / unitCpuUsage);
    // test with custom value for emulation interval of 20%
    conf.setFloat(CumulativeCpuUsageEmulatorPlugin.CPU_EMULATION_PROGRESS_INTERVAL, 0.2F);
    testEmulationAccuracy(conf, fakeCore, monitor, metrics, cpuPlugin, targetCpuUsage, targetCpuUsage / unitCpuUsage);
    // test if emulation interval boundary is respected (unit usage = 1)
    //  test the case where the current progress is less than threshold
    // initialize
    fakeProgress = new FakeProgressive();
    fakeCore.reset();
    fakeCore.setUnitUsage(1);
    conf.setFloat(CumulativeCpuUsageEmulatorPlugin.CPU_EMULATION_PROGRESS_INTERVAL, 0.25F);
    cpuPlugin.initialize(conf, metrics, monitor, fakeProgress);
    // take a snapshot after the initialization
    long initCpuUsage = monitor.getCumulativeCpuTime();
    long initNumCalls = fakeCore.getNumCalls();
    // test with 0 progress
    testEmulationBoundary(0F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[no-op, 0 progress]");
    // test with 24% progress
    testEmulationBoundary(0.24F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[no-op, 24% progress]");
    // test with 25% progress
    //  target = 1000ms, target emulation at 25% = 250ms, 
    //  weighed target = 1000 * 0.25^4 (we are using progress^4 as the weight)
    //                 ~ 4
    //  but current usage = init-usage = 100, hence expected = 100
    testEmulationBoundary(0.25F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[op, 25% progress]");
    // test with 80% progress
    //  target = 1000ms, target emulation at 80% = 800ms, 
    //  weighed target = 1000 * 0.25^4 (we are using progress^4 as the weight)
    //                 ~ 410
    //  current-usage = init-usage = 100, hence expected-usage = 410
    testEmulationBoundary(0.80F, fakeCore, fakeProgress, cpuPlugin, 410, 410, "[op, 80% progress]");
    // now test if the final call with 100% progress ramps up the CPU usage
    testEmulationBoundary(1F, fakeCore, fakeProgress, cpuPlugin, targetCpuUsage, targetCpuUsage, "[op, 100% progress]");
    // test if emulation interval boundary is respected (unit usage = 50)
    //  test the case where the current progress is less than threshold
    // initialize
    fakeProgress = new FakeProgressive();
    fakeCore.reset();
    fakeCore.setUnitUsage(unitCpuUsage);
    conf.setFloat(CumulativeCpuUsageEmulatorPlugin.CPU_EMULATION_PROGRESS_INTERVAL, 0.40F);
    cpuPlugin.initialize(conf, metrics, monitor, fakeProgress);
    // take a snapshot after the initialization
    initCpuUsage = monitor.getCumulativeCpuTime();
    initNumCalls = fakeCore.getNumCalls();
    // test with 0 progress
    testEmulationBoundary(0F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[no-op, 0 progress]");
    // test with 39% progress
    testEmulationBoundary(0.39F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[no-op, 39% progress]");
    // test with 40% progress
    //  target = 1000ms, target emulation at 40% = 4000ms, 
    //  weighed target = 1000 * 0.40^4 (we are using progress^4 as the weight)
    //                 ~ 26
    // current-usage = init-usage = 100, hence expected-usage = 100
    testEmulationBoundary(0.40F, fakeCore, fakeProgress, cpuPlugin, initCpuUsage, initNumCalls, "[op, 40% progress]");
    // test with 90% progress
    //  target = 1000ms, target emulation at 90% = 900ms, 
    //  weighed target = 1000 * 0.90^4 (we are using progress^4 as the weight)
    //                 ~ 657
    //  current-usage = init-usage = 100, hence expected-usage = 657 but 
    //  the fake-core increases in steps of 50, hence final target = 700
    testEmulationBoundary(0.90F, fakeCore, fakeProgress, cpuPlugin, 700, 700 / unitCpuUsage, "[op, 90% progress]");
    // now test if the final call with 100% progress ramps up the CPU usage
    testEmulationBoundary(1F, fakeCore, fakeProgress, cpuPlugin, targetCpuUsage, targetCpuUsage / unitCpuUsage, "[op, 100% progress]");
}