use of org.apache.accumulo.core.spi.cache.BlockCacheManager in project accumulo by apache.
the class TestLruBlockCache method testCacheEvictionSimple.
@Test
public void testCacheEvictionSimple() throws Exception {
long maxSize = 100000;
long blockSize = calculateBlockSizeDefault(maxSize, 10);
DefaultConfiguration dc = DefaultConfiguration.getInstance();
ConfigurationCopy cc = new ConfigurationCopy(dc);
cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
BlockCacheManager manager = BlockCacheManagerFactory.getInstance(cc);
cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize));
cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize));
LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).buildMap().forEach(cc::set);
manager.start(new BlockCacheConfiguration(cc));
LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX);
Block[] blocks = generateFixedBlocks(10, blockSize, "block");
long expectedCacheSize = cache.heapSize();
// Add all the blocks
for (Block block : blocks) {
cache.cacheBlock(block.blockName, block.buf);
expectedCacheSize += block.heapSize();
}
// A single eviction run should have occurred
assertEquals(1, cache.getEvictionCount());
// Our expected size overruns acceptable limit
assertTrue(expectedCacheSize > (maxSize * LruBlockCacheConfiguration.DEFAULT_ACCEPTABLE_FACTOR));
// But the cache did not grow beyond max
assertTrue(cache.heapSize() < maxSize);
// And is still below the acceptable limit
assertTrue(cache.heapSize() < (maxSize * LruBlockCacheConfiguration.DEFAULT_ACCEPTABLE_FACTOR));
// All blocks except block 0 and 1 should be in the cache
assertNull(cache.getBlock(blocks[0].blockName));
assertNull(cache.getBlock(blocks[1].blockName));
for (int i = 2; i < blocks.length; i++) {
assertTrue(Arrays.equals(cache.getBlock(blocks[i].blockName).getBuffer(), blocks[i].buf));
}
manager.stop();
}
use of org.apache.accumulo.core.spi.cache.BlockCacheManager in project accumulo by apache.
the class TestLruBlockCache method testCacheSimple.
@Test
public void testCacheSimple() throws Exception {
long maxSize = 1000000;
long blockSize = calculateBlockSizeDefault(maxSize, 101);
DefaultConfiguration dc = DefaultConfiguration.getInstance();
ConfigurationCopy cc = new ConfigurationCopy(dc);
cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
BlockCacheManager manager = BlockCacheManagerFactory.getInstance(cc);
cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize));
cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize));
manager.start(new BlockCacheConfiguration(cc));
LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX);
Block[] blocks = generateRandomBlocks(100, blockSize);
long expectedCacheSize = cache.heapSize();
// Confirm empty
for (Block block : blocks) {
assertNull(cache.getBlock(block.blockName));
}
// Add blocks
for (Block block : blocks) {
cache.cacheBlock(block.blockName, block.buf);
expectedCacheSize += block.heapSize();
}
// Verify correctly calculated cache heap size
assertEquals(expectedCacheSize, cache.heapSize());
// Check if all blocks are properly cached and retrieved
for (Block block : blocks) {
CacheEntry ce = cache.getBlock(block.blockName);
assertNotNull(ce);
assertEquals(ce.getBuffer().length, block.buf.length);
}
// Verify correctly calculated cache heap size
assertEquals(expectedCacheSize, cache.heapSize());
// Check if all blocks are properly cached and retrieved
for (Block block : blocks) {
CacheEntry ce = cache.getBlock(block.blockName);
assertNotNull(ce);
assertEquals(ce.getBuffer().length, block.buf.length);
}
// Expect no evictions
assertEquals(0, cache.getEvictionCount());
// Thread t = new LruBlockCache.StatisticsThread(cache);
// t.start();
// t.join();
manager.stop();
}
use of org.apache.accumulo.core.spi.cache.BlockCacheManager in project accumulo by apache.
the class TestLruBlockCache method testCacheEvictionTwoPriorities.
@Test
public void testCacheEvictionTwoPriorities() throws Exception {
long maxSize = 100000;
long blockSize = calculateBlockSizeDefault(maxSize, 10);
DefaultConfiguration dc = DefaultConfiguration.getInstance();
ConfigurationCopy cc = new ConfigurationCopy(dc);
cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
BlockCacheManager manager = BlockCacheManagerFactory.getInstance(cc);
cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize));
cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize));
LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).minFactor(0.98f).acceptableFactor(0.99f).singleFactor(0.25f).multiFactor(0.50f).memoryFactor(0.25f).buildMap().forEach(cc::set);
manager.start(new BlockCacheConfiguration(cc));
LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX);
Block[] singleBlocks = generateFixedBlocks(5, 10000, "single");
Block[] multiBlocks = generateFixedBlocks(5, 10000, "multi");
long expectedCacheSize = cache.heapSize();
// Add and get the multi blocks
for (Block block : multiBlocks) {
cache.cacheBlock(block.blockName, block.buf);
expectedCacheSize += block.heapSize();
assertTrue(Arrays.equals(cache.getBlock(block.blockName).getBuffer(), block.buf));
}
// Add the single blocks (no get)
for (Block block : singleBlocks) {
cache.cacheBlock(block.blockName, block.buf);
expectedCacheSize += block.heapSize();
}
// A single eviction run should have occurred
assertEquals(cache.getEvictionCount(), 1);
// We expect two entries evicted
assertEquals(cache.getEvictedCount(), 2);
// Our expected size overruns acceptable limit
assertTrue(expectedCacheSize > (maxSize * LruBlockCacheConfiguration.DEFAULT_ACCEPTABLE_FACTOR));
// But the cache did not grow beyond max
assertTrue(cache.heapSize() <= maxSize);
// And is now below the acceptable limit
assertTrue(cache.heapSize() <= (maxSize * LruBlockCacheConfiguration.DEFAULT_ACCEPTABLE_FACTOR));
// We expect fairness across the two priorities.
// This test makes multi go barely over its limit, in-memory
// empty, and the rest in single. Two single evictions and
// one multi eviction expected.
assertNull(cache.getBlock(singleBlocks[0].blockName));
assertNull(cache.getBlock(multiBlocks[0].blockName));
// And all others to be cached
for (int i = 1; i < 4; i++) {
assertTrue(Arrays.equals(cache.getBlock(singleBlocks[i].blockName).getBuffer(), singleBlocks[i].buf));
assertTrue(Arrays.equals(cache.getBlock(multiBlocks[i].blockName).getBuffer(), multiBlocks[i].buf));
}
manager.stop();
}
use of org.apache.accumulo.core.spi.cache.BlockCacheManager in project accumulo by apache.
the class TestLruBlockCache method testBackgroundEvictionThread.
@Test
public void testBackgroundEvictionThread() throws Exception {
long maxSize = 100000;
// room for 9, will evict
long blockSize = calculateBlockSizeDefault(maxSize, 9);
DefaultConfiguration dc = DefaultConfiguration.getInstance();
ConfigurationCopy cc = new ConfigurationCopy(dc);
cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
BlockCacheManager manager = BlockCacheManagerFactory.getInstance(cc);
cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize));
cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize));
manager.start(new BlockCacheConfiguration(cc));
LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX);
Block[] blocks = generateFixedBlocks(10, blockSize, "block");
// Add all the blocks
for (Block block : blocks) {
cache.cacheBlock(block.blockName, block.buf);
}
// Let the eviction run
int n = 0;
while (cache.getEvictionCount() == 0) {
Thread.sleep(1000);
assertTrue(n++ < 1);
}
// A single eviction run should have occurred
assertEquals(cache.getEvictionCount(), 1);
manager.stop();
}
use of org.apache.accumulo.core.spi.cache.BlockCacheManager in project accumulo by apache.
the class TestLruBlockCache method testScanResistance.
// test scan resistance
@Test
public void testScanResistance() throws Exception {
long maxSize = 100000;
long blockSize = calculateBlockSize(maxSize, 10);
DefaultConfiguration dc = DefaultConfiguration.getInstance();
ConfigurationCopy cc = new ConfigurationCopy(dc);
cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
BlockCacheManager manager = BlockCacheManagerFactory.getInstance(cc);
cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize));
cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize));
LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).minFactor(0.66f).acceptableFactor(0.99f).singleFactor(0.33f).multiFactor(0.33f).memoryFactor(0.34f).buildMap().forEach(cc::set);
manager.start(new BlockCacheConfiguration(cc));
LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX);
Block[] singleBlocks = generateFixedBlocks(20, blockSize, "single");
Block[] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
// Add 5 multi blocks
for (Block block : multiBlocks) {
cache.cacheBlock(block.blockName, block.buf);
cache.getBlock(block.blockName);
}
// Add 5 single blocks
for (int i = 0; i < 5; i++) {
cache.cacheBlock(singleBlocks[i].blockName, singleBlocks[i].buf);
}
// An eviction ran
assertEquals(1, cache.getEvictionCount());
// To drop down to 2/3 capacity, we'll need to evict 4 blocks
assertEquals(4, cache.getEvictedCount());
// Should have been taken off equally from single and multi
assertNull(cache.getBlock(singleBlocks[0].blockName));
assertNull(cache.getBlock(singleBlocks[1].blockName));
assertNull(cache.getBlock(multiBlocks[0].blockName));
assertNull(cache.getBlock(multiBlocks[1].blockName));
for (int i = 5; i < 18; i++) {
cache.cacheBlock(singleBlocks[i].blockName, singleBlocks[i].buf);
}
// 4 total evictions, 16 total evicted
assertEquals(4, cache.getEvictionCount());
assertEquals(16, cache.getEvictedCount());
// Should now have 7 total blocks
assertEquals(7, cache.size());
manager.stop();
}
Aggregations