Examples with FakeProgressive org.apache.hadoop.mapred.gridmix.TestResourceUsageEmulators.FakeProgressive used on opensource projects

Example 1 with FakeProgressive

use of org.apache.hadoop.mapred.gridmix.TestResourceUsageEmulators.FakeProgressive in project hadoop by apache.

the class TestGridmixMemoryEmulation method testTotalHeapUsageEmulatorPlugin.

/**
   * Test {@link TotalHeapUsageEmulatorPlugin}.
   */
@Test
public void testTotalHeapUsageEmulatorPlugin() throws Exception {
    Configuration conf = new Configuration();
    // set the dummy resource calculator for testing
    ResourceCalculatorPlugin monitor = new DummyResourceCalculatorPlugin();
    // 1GB
    long maxHeapUsage = 1024 * TotalHeapUsageEmulatorPlugin.ONE_MB;
    conf.setLong(DummyResourceCalculatorPlugin.MAXPMEM_TESTING_PROPERTY, maxHeapUsage);
    monitor.setConf(conf);
    // no buffer to be reserved
    conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0F);
    // only 1 call to be made per cycle
    conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 1F);
    // 200mb
    long targetHeapUsageInMB = 200;
    // fake progress indicator
    FakeProgressive fakeProgress = new FakeProgressive();
    // fake heap usage generator
    FakeHeapUsageEmulatorCore fakeCore = new FakeHeapUsageEmulatorCore();
    // a heap usage emulator with fake core
    FakeHeapUsageEmulatorPlugin heapPlugin = new FakeHeapUsageEmulatorPlugin(fakeCore);
    // test with invalid or missing resource usage value
    ResourceUsageMetrics invalidUsage = TestResourceUsageEmulators.createMetrics(0);
    heapPlugin.initialize(conf, invalidUsage, null, null);
    // test if disabled heap emulation plugin's emulate() call is a no-operation
    // this will test if the emulation plugin is disabled or not
    int numCallsPre = fakeCore.getNumCalls();
    long heapUsagePre = fakeCore.getHeapUsageInMB();
    heapPlugin.emulate();
    int numCallsPost = fakeCore.getNumCalls();
    long heapUsagePost = fakeCore.getHeapUsageInMB();
    //  test if no calls are made heap usage emulator core
    assertEquals("Disabled heap usage emulation plugin works!", numCallsPre, numCallsPost);
    //  test if no calls are made heap usage emulator core
    assertEquals("Disabled heap usage emulation plugin works!", heapUsagePre, heapUsagePost);
    // test with get progress
    float progress = heapPlugin.getProgress();
    assertEquals("Invalid progress of disabled cumulative heap usage emulation " + "plugin!", 1.0f, progress, 0f);
    // test with wrong/invalid configuration
    Boolean failed = null;
    invalidUsage = TestResourceUsageEmulators.createMetrics(maxHeapUsage + TotalHeapUsageEmulatorPlugin.ONE_MB);
    try {
        heapPlugin.initialize(conf, invalidUsage, monitor, null);
        failed = false;
    } catch (Exception e) {
        failed = true;
    }
    assertNotNull("Fail case failure!", failed);
    assertTrue("Expected failure!", failed);
    // test with valid resource usage value
    ResourceUsageMetrics metrics = TestResourceUsageEmulators.createMetrics(targetHeapUsageInMB * TotalHeapUsageEmulatorPlugin.ONE_MB);
    // test with default emulation interval
    // in every interval, the emulator will add 100% of the expected usage 
    // (since gridmix.emulators.resource-usage.heap.load-ratio=1)
    // so at 10%, emulator will add 10% (difference), at 20% it will add 10% ...
    // So to emulate 200MB, it will add
    //   20mb + 20mb + 20mb + 20mb + .. = 200mb 
    testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 200, 10);
    // test with custom value for emulation interval of 20%
    conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_EMULATION_PROGRESS_INTERVAL, 0.2F);
    //  40mb + 40mb + 40mb + 40mb + 40mb = 200mb
    testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 200, 5);
    // test with custom value of free heap ratio and load ratio = 1
    conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 1F);
    conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0.5F);
    //  40mb + 0mb + 80mb + 0mb + 0mb = 120mb
    testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 120, 2);
    // test with custom value of heap load ratio and min free heap ratio = 0
    conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 0.5F);
    conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0F);
    // 20mb (call#1) + 20mb (call#1) + 20mb (call#2) + 20mb (call#2) +.. = 200mb
    testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 200, 10);
    // test with custom value of free heap ratio = 0.3 and load ratio = 0.5
    conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0.25F);
    conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 0.5F);
    // 20mb (call#1) + 20mb (call#1) + 30mb (call#2) + 0mb (call#2) 
    // + 30mb (call#3) + 0mb (call#3) + 35mb (call#4) + 0mb (call#4)
    // + 37mb (call#5) + 0mb (call#5) = 162mb
    testEmulationAccuracy(conf, fakeCore, monitor, metrics, heapPlugin, 162, 6);
    // test if emulation interval boundary is respected
    // initialize
    fakeProgress = new FakeProgressive();
    conf.setFloat(TotalHeapUsageEmulatorPlugin.MIN_HEAP_FREE_RATIO, 0F);
    conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_LOAD_RATIO, 1F);
    conf.setFloat(TotalHeapUsageEmulatorPlugin.HEAP_EMULATION_PROGRESS_INTERVAL, 0.25F);
    heapPlugin.initialize(conf, metrics, monitor, fakeProgress);
    fakeCore.resetFake();
    // take a snapshot after the initialization
    long initHeapUsage = fakeCore.getHeapUsageInMB();
    long initNumCallsUsage = fakeCore.getNumCalls();
    // test with 0 progress
    testEmulationBoundary(0F, fakeCore, fakeProgress, heapPlugin, initHeapUsage, initNumCallsUsage, "[no-op, 0 progress]");
    // test with 24% progress
    testEmulationBoundary(0.24F, fakeCore, fakeProgress, heapPlugin, initHeapUsage, initNumCallsUsage, "[no-op, 24% progress]");
    // test with 25% progress
    testEmulationBoundary(0.25F, fakeCore, fakeProgress, heapPlugin, targetHeapUsageInMB / 4, 1, "[op, 25% progress]");
    // test with 80% progress
    testEmulationBoundary(0.80F, fakeCore, fakeProgress, heapPlugin, (targetHeapUsageInMB * 4) / 5, 2, "[op, 80% progress]");
    // now test if the final call with 100% progress ramps up the heap usage
    testEmulationBoundary(1F, fakeCore, fakeProgress, heapPlugin, targetHeapUsageInMB, 3, "[op, 100% progress]");
}

Example 2 with FakeProgressive

use of org.apache.hadoop.mapred.gridmix.TestResourceUsageEmulators.FakeProgressive in project hadoop by apache.

the class TestGridmixMemoryEmulation method testEmulationAccuracy.

// test whether the heap usage emulator achieves the desired target using
// desired calls to the underling core engine.
private static void testEmulationAccuracy(Configuration conf, FakeHeapUsageEmulatorCore fakeCore, ResourceCalculatorPlugin monitor, ResourceUsageMetrics metrics, TotalHeapUsageEmulatorPlugin heapPlugin, long expectedTotalHeapUsageInMB, long expectedTotalNumCalls) throws Exception {
    FakeProgressive fakeProgress = new FakeProgressive();
    fakeCore.resetFake();
    heapPlugin.initialize(conf, metrics, monitor, fakeProgress);
    int numLoops = 0;
    while (fakeProgress.getProgress() < 1) {
        ++numLoops;
        float progress = numLoops / 100.0F;
        fakeProgress.setProgress(progress);
        heapPlugin.emulate();
    }
    // test if the resource plugin shows the expected usage
    assertEquals("Cumulative heap usage emulator plugin failed (total usage)!", expectedTotalHeapUsageInMB, fakeCore.getHeapUsageInMB(), 1L);
    // test if the resource plugin shows the expected num calls
    assertEquals("Cumulative heap usage emulator plugin failed (num calls)!", expectedTotalNumCalls, fakeCore.getNumCalls(), 0L);
}