Examples with ChoreService org.apache.hadoop.hbase.ChoreService used on opensource projects

Example 1 with ChoreService

use of org.apache.hadoop.hbase.ChoreService in project hbase by apache.

the class TestEndToEndSplitTransaction method testFromClientSideWhileSplitting.

/**
   * Tests that the client sees meta table changes as atomic during splits
   */
@Test
public void testFromClientSideWhileSplitting() throws Throwable {
    LOG.info("Starting testFromClientSideWhileSplitting");
    final TableName tableName = TableName.valueOf(name.getMethodName());
    final byte[] FAMILY = Bytes.toBytes("family");
    //SplitTransaction will update the meta table by offlining the parent region, and adding info
    //for daughters.
    Table table = TEST_UTIL.createTable(tableName, FAMILY);
    Stoppable stopper = new StoppableImplementation();
    RegionSplitter regionSplitter = new RegionSplitter(table);
    RegionChecker regionChecker = new RegionChecker(CONF, stopper, tableName);
    final ChoreService choreService = new ChoreService("TEST_SERVER");
    choreService.scheduleChore(regionChecker);
    regionSplitter.start();
    //wait until the splitter is finished
    regionSplitter.join();
    stopper.stop(null);
    if (regionChecker.ex != null) {
        throw new AssertionError("regionChecker", regionChecker.ex);
    }
    if (regionSplitter.ex != null) {
        throw new AssertionError("regionSplitter", regionSplitter.ex);
    }
    //one final check
    regionChecker.verify();
}

Example 2 with ChoreService

use of org.apache.hadoop.hbase.ChoreService in project hbase by apache.

the class TestHeapMemoryManager method testWhenCombinedHeapSizesFromTunerGoesOutSideMaxLimit.

@Test
public void testWhenCombinedHeapSizesFromTunerGoesOutSideMaxLimit() throws Exception {
    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
    Configuration conf = HBaseConfiguration.create();
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, HeapMemoryTuner.class);
    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), new RegionServerAccountingStub(conf));
    long oldMemstoreSize = memStoreFlusher.memstoreSize;
    long oldBlockCacheSize = blockCache.maxSize;
    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
    heapMemoryManager.start(choreService);
    CustomHeapMemoryTuner.memstoreSize = 0.7f;
    CustomHeapMemoryTuner.blockCacheSize = 0.3f;
    // Allow the tuner to run once and do necessary memory up
    Thread.sleep(1500);
    assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize);
    assertEquals(oldBlockCacheSize, blockCache.maxSize);
}

Example 3 with ChoreService

use of org.apache.hadoop.hbase.ChoreService in project hbase by apache.

the class TestHeapMemoryManager method testHeapMemoryManagerWhenOffheapFlushesHappenUnderReadHeavyCase.

@Test
public void testHeapMemoryManagerWhenOffheapFlushesHappenUnderReadHeavyCase() throws Exception {
    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
    Configuration conf = HBaseConfiguration.create();
    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf, true);
    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
    // Empty memstore and but nearly filled block cache
    blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
    regionServerAccounting.setTestMemstoreSize(0);
    // Let the system start with default values for memstore heap and block cache size.
    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), regionServerAccounting);
    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
    long oldBlockCacheSize = blockCache.maxSize;
    float maxStepValue = DefaultHeapMemoryTuner.DEFAULT_MIN_STEP_VALUE;
    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
    heapMemoryManager.start(choreService);
    blockCache.evictBlock(null);
    blockCache.evictBlock(null);
    blockCache.evictBlock(null);
    // do some offheap flushes also. So there should be decrease in memstore but
    // not as that when we don't have offheap flushes
    memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
    memStoreFlusher.requestFlush(null, false);
    memStoreFlusher.requestFlush(null, false);
    memStoreFlusher.requestFlush(null, false);
    memStoreFlusher.requestFlush(null, false);
    // Allow the tuner to run once and do necessary memory up
    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
    oldBlockCacheSize = blockCache.maxSize;
    // Do some more evictions before the next run of HeapMemoryTuner
    blockCache.evictBlock(null);
    // Allow the tuner to run once and do necessary memory up
    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
}

Example 4 with ChoreService

use of org.apache.hadoop.hbase.ChoreService in project hbase by apache.

the class TestHeapMemoryManager method testWhenL2BlockCacheIsOnHeap.

@Test
public void testWhenL2BlockCacheIsOnHeap() throws Exception {
    HeapMemoryManager heapMemoryManager = null;
    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.3));
    Configuration conf = HBaseConfiguration.create();
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
    conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.4F);
    conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.3F);
    conf.setFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 0.1F);
    conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "heap");
    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, HeapMemoryTuner.class);
    try {
        heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), new RegionServerAccountingStub(conf));
        fail("Should have failed as the collective heap memory need is above 80%");
    } catch (Exception e) {
    }
    // Change the max/min ranges for memstore and bock cache so as to pass the criteria check
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.6f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.6f);
    heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), new RegionServerAccountingStub(conf));
    long oldMemstoreSize = memStoreFlusher.memstoreSize;
    long oldBlockCacheSize = blockCache.maxSize;
    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
    heapMemoryManager.start(choreService);
    CustomHeapMemoryTuner.memstoreSize = 0.4f;
    CustomHeapMemoryTuner.blockCacheSize = 0.4f;
    // Allow the tuner to run once and do necessary memory up
    Thread.sleep(1500);
    // The size should not get changes as the collection of memstore size and L1 and L2 block cache
    // size will cross the ax allowed 80% mark
    assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize);
    assertEquals(oldBlockCacheSize, blockCache.maxSize);
    CustomHeapMemoryTuner.memstoreSize = 0.1f;
    CustomHeapMemoryTuner.blockCacheSize = 0.5f;
    // Allow the tuner to run once and do necessary memory up
    waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
    assertHeapSpace(0.1f, memStoreFlusher.memstoreSize);
    assertHeapSpace(0.5f, blockCache.maxSize);
}

Example 5 with ChoreService

use of org.apache.hadoop.hbase.ChoreService in project hbase by apache.

the class TestHeapMemoryManager method testWhenClusterIsWriteHeavyWithEmptyMemstore.

@Test
public void testWhenClusterIsWriteHeavyWithEmptyMemstore() throws Exception {
    Configuration conf = HBaseConfiguration.create();
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
    RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
    // Empty block cache and memstore
    blockCache.setTestBlockSize(0);
    regionServerAccounting.setTestMemstoreSize(0);
    conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
    // Let the system start with default values for memstore heap and block cache size.
    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf), regionServerAccounting);
    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
    long oldBlockCacheSize = blockCache.maxSize;
    final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
    heapMemoryManager.start(choreService);
    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK;
    memStoreFlusher.requestFlush(null, false);
    memStoreFlusher.requestFlush(null, false);
    memStoreFlusher.requestFlush(null, false);
    memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
    memStoreFlusher.requestFlush(null, false);
    // Allow the tuner to run once and do necessary memory up
    Thread.sleep(1500);
    // No changes should be made by tuner as we already have lot of empty space
    assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
    assertEquals(oldBlockCacheSize, blockCache.maxSize);
}