Examples with ChoreService org.apache.hadoop.hbase.ChoreService used on opensource projects

Example 11 with ChoreService

use of org.apache.hadoop.hbase.ChoreService in project hbase by apache.

the class TestHeapMemoryManager method testWhenClusterIsHavingMoreWritesThanReads.

@Test
public void testWhenClusterIsHavingMoreWritesThanReads() throws Exception {
    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
    Configuration conf = HBaseConfiguration.create();
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
    // Both memstore and block cache are nearly filled
    blockCache.setTestBlockSize(0);
    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
    // Let the system start with default values for memstore heap and block cache size.
    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), regionServerAccounting);
    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
    long oldBlockCacheSize = blockCache.maxSize;
    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
    heapMemoryManager.start(choreService);
    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
    memStoreFlusher.requestFlush(null, false);
    memStoreFlusher.requestFlush(null, false);
    memStoreFlusher.requestFlush(null, false);
    blockCache.evictBlock(null);
    // Allow the tuner to run once and do necessary memory up
    Thread.sleep(1500);
    // No changes should happen as there is undefined increase in flushes and evictions
    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
    assertEquals(oldBlockCacheSize, blockCache.maxSize);
    // Do some more flushes before the next run of HeapMemoryTuner
    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
    memStoreFlusher.requestFlush(null, false);
    memStoreFlusher.requestFlush(null, false);
    memStoreFlusher.requestFlush(null, false);
    // Allow the tuner to run once and do necessary memory up
    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldBlockCacheSize, blockCache.maxSize);
}

Example 12 with ChoreService

use of org.apache.hadoop.hbase.ChoreService in project hbase by apache.

the class TestHeapMemoryManager method testHeapMemoryManagerWithOffheapMemstoreAndMixedWorkload.

@Test
public void testHeapMemoryManagerWithOffheapMemstoreAndMixedWorkload() throws Exception {
    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
    Configuration conf = HBaseConfiguration.create();
    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true);
    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
    // Empty memstore and but nearly filled block cache
    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
    regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
    // Let the system start with default values for memstore heap and block cache size.
    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), regionServerAccounting);
    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
    long oldBlockCacheSize = blockCache.maxSize;
    float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE;
    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
    heapMemoryManager.start(choreService);
    blockCache.evictBlock(null);
    blockCache.evictBlock(null);
    blockCache.evictBlock(null);
    // do some offheap flushes also. So there should be decrease in memstore but
    // not as that when we don't have offheap flushes
    memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
    memStoreFlusher.requestFlush(null, false);
    memStoreFlusher.requestFlush(null, false);
    memStoreFlusher.requestFlush(null, false);
    memStoreFlusher.requestFlush(null, false);
    // Allow the tuner to run once and do necessary memory up
    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
    oldBlockCacheSize = blockCache.maxSize;
    // change memstore size
    // regionServerAccounting.setTestMemstoreSize((long)(maxHeapSize * 0.4 * 0.8));
    // The memstore size would have decreased. Now again do some flushes and ensure the
    // flushes are due to onheap overhead. This should once again call for increase in
    // memstore size but that increase should be to the safe size
    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK;
    memStoreFlusher.requestFlush(null, false);
    memStoreFlusher.requestFlush(null, false);
    memStoreFlusher.requestFlush(null, false);
    memStoreFlusher.requestFlush(null, false);
    // Allow the tuner to run once and do necessary memory up
    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
    assertHeapSpaceDelta(maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
    assertHeapSpaceDelta(-maxStepValue, oldBlockCacheSize, blockCache.maxSize);
}

Example 13 with ChoreService

use of org.apache.hadoop.hbase.ChoreService in project hbase by apache.

the class TestHeapMemoryManager method testPluggingInHeapMemoryTuner.

@Test
public void testPluggingInHeapMemoryTuner() throws Exception {
    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
    Configuration conf = HBaseConfiguration.create();
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.78f);
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.02f);
    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, HeapMemoryTuner.class);
    // Let the system start with default values for memstore heap and block cache size.
    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), new RegionServerAccountingStub(conf));
    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
    heapMemoryManager.start(choreService);
    // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner
    CustomHeapMemoryTuner.memstoreSize = 0.78f;
    CustomHeapMemoryTuner.blockCacheSize = 0.02f;
    // Allow the tuner to run once and do necessary memory up
    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
    // Memstore
    assertHeapSpace(0.78f, memStoreFlusher.memstoreSize);
    // BlockCache
    assertHeapSpace(0.02f, blockCache.maxSize);
    // Now we wants to be in read mode. Set bigger memstore size from CustomHeapMemoryTuner
    CustomHeapMemoryTuner.blockCacheSize = 0.75f;
    CustomHeapMemoryTuner.memstoreSize = 0.05f;
    // Allow the tuner to run once and do necessary memory up
    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
    // BlockCache
    assertHeapSpace(0.75f, blockCache.maxSize);
    // Memstore
    assertHeapSpace(0.05f, memStoreFlusher.memstoreSize);
}

Example 14 with ChoreService

use of org.apache.hadoop.hbase.ChoreService in project hbase by apache.

the class TestHeapMemoryManager method testWhenClusterIsReadHeavyWithEmptyBlockCache.

@Test
public void testWhenClusterIsReadHeavyWithEmptyBlockCache() throws Exception {
    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
    Configuration conf = HBaseConfiguration.create();
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
    // Empty block cache and memstore
    blockCache.setTestBlockSize(0);
    regionServerAccounting.setTestMemstoreSize(0);
    // Let the system start with default values for memstore heap and block cache size.
    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), regionServerAccounting);
    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
    long oldBlockCacheSize = blockCache.maxSize;
    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
    heapMemoryManager.start(choreService);
    blockCache.evictBlock(null);
    blockCache.evictBlock(null);
    blockCache.evictBlock(null);
    // Allow the tuner to run once and do necessary memory up
    Thread.sleep(1500);
    // No changes should be made by tuner as we already have lot of empty space
    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
    assertEquals(oldBlockCacheSize, blockCache.maxSize);
}

Example 15 with ChoreService

use of org.apache.hadoop.hbase.ChoreService in project hbase by apache.

the class TestHeapMemoryManager method testWhenSizeGivenByHeapTunerGoesOutsideRange.

@Test
public void testWhenSizeGivenByHeapTunerGoesOutsideRange() throws Exception {
    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
    Configuration conf = HBaseConfiguration.create();
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, HeapMemoryTuner.class);
    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), new RegionServerAccountingStub(conf));
    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
    heapMemoryManager.start(choreService);
    CustomHeapMemoryTuner.memstoreSize = 0.78f;
    CustomHeapMemoryTuner.blockCacheSize = 0.02f;
    // Allow the tuner to run once and do necessary memory up
    Thread.sleep(1500);
    // Even if the tuner says to set the memstore to 78%, HBase makes it as 70% as that is the
    // upper bound. Same with block cache as 10% is the lower bound.
    assertHeapSpace(0.7f, memStoreFlusher.memstoreSize);
    assertHeapSpace(0.1f, blockCache.maxSize);
}