Examples with LruBlockCache org.apache.accumulo.core.file.blockfile.cache.lru.LruBlockCache used on opensource projects

Example 1 with LruBlockCache

use of org.apache.accumulo.core.file.blockfile.cache.lru.LruBlockCache in project accumulo by apache.

the class TestLruBlockCache method testCacheSimple.

public void testCacheSimple() throws Exception {
    long maxSize = 1000000;
    long blockSize = calculateBlockSizeDefault(maxSize, 101);
    DefaultConfiguration dc = DefaultConfiguration.getInstance();
    ConfigurationCopy cc = new ConfigurationCopy(dc);
    cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
    BlockCacheManager manager = BlockCacheManagerFactory.getInstance(cc);
    cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize));
    cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize));
    manager.start(new BlockCacheConfiguration(cc));
    LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX);
    Block[] blocks = generateRandomBlocks(100, blockSize);
    long expectedCacheSize = cache.heapSize();
    // Confirm empty
    for (Block block : blocks) {
        assertTrue(cache.getBlock(block.blockName) == null);
    }
    // Add blocks
    for (Block block : blocks) {
        cache.cacheBlock(block.blockName, block.buf);
        expectedCacheSize += block.heapSize();
    }
    // Verify correctly calculated cache heap size
    assertEquals(expectedCacheSize, cache.heapSize());
    // Check if all blocks are properly cached and retrieved
    for (Block block : blocks) {
        CacheEntry ce = cache.getBlock(block.blockName);
        assertTrue(ce != null);
        assertEquals(ce.getBuffer().length, block.buf.length);
    }
    // Verify correctly calculated cache heap size
    assertEquals(expectedCacheSize, cache.heapSize());
    // Check if all blocks are properly cached and retrieved
    for (Block block : blocks) {
        CacheEntry ce = cache.getBlock(block.blockName);
        assertTrue(ce != null);
        assertEquals(ce.getBuffer().length, block.buf.length);
    }
    // Expect no evictions
    assertEquals(0, cache.getEvictionCount());
    // Thread t = new LruBlockCache.StatisticsThread(cache);
    // t.start();
    // t.join();
    manager.stop();
}

Example 2 with LruBlockCache

use of org.apache.accumulo.core.file.blockfile.cache.lru.LruBlockCache in project accumulo by apache.

the class TestLruBlockCache method testCacheEvictionSimple.

public void testCacheEvictionSimple() throws Exception {
    long maxSize = 100000;
    long blockSize = calculateBlockSizeDefault(maxSize, 10);
    DefaultConfiguration dc = DefaultConfiguration.getInstance();
    ConfigurationCopy cc = new ConfigurationCopy(dc);
    cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
    BlockCacheManager manager = BlockCacheManagerFactory.getInstance(cc);
    cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize));
    cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize));
    LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).buildMap().forEach(cc::set);
    manager.start(new BlockCacheConfiguration(cc));
    LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX);
    Block[] blocks = generateFixedBlocks(10, blockSize, "block");
    long expectedCacheSize = cache.heapSize();
    // Add all the blocks
    for (Block block : blocks) {
        cache.cacheBlock(block.blockName, block.buf);
        expectedCacheSize += block.heapSize();
    }
    // A single eviction run should have occurred
    assertEquals(1, cache.getEvictionCount());
    // Our expected size overruns acceptable limit
    assertTrue(expectedCacheSize > (maxSize * LruBlockCacheConfiguration.DEFAULT_ACCEPTABLE_FACTOR));
    // But the cache did not grow beyond max
    assertTrue(cache.heapSize() < maxSize);
    // And is still below the acceptable limit
    assertTrue(cache.heapSize() < (maxSize * LruBlockCacheConfiguration.DEFAULT_ACCEPTABLE_FACTOR));
    // All blocks except block 0 and 1 should be in the cache
    assertTrue(cache.getBlock(blocks[0].blockName) == null);
    assertTrue(cache.getBlock(blocks[1].blockName) == null);
    for (int i = 2; i < blocks.length; i++) {
        assertTrue(Arrays.equals(cache.getBlock(blocks[i].blockName).getBuffer(), blocks[i].buf));
    }
    manager.stop();
}

Example 3 with LruBlockCache

use of org.apache.accumulo.core.file.blockfile.cache.lru.LruBlockCache in project accumulo by apache.

the class TestLruBlockCache method testScanResistance.

// test scan resistance
public void testScanResistance() throws Exception {
    long maxSize = 100000;
    long blockSize = calculateBlockSize(maxSize, 10);
    DefaultConfiguration dc = DefaultConfiguration.getInstance();
    ConfigurationCopy cc = new ConfigurationCopy(dc);
    cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
    BlockCacheManager manager = BlockCacheManagerFactory.getInstance(cc);
    cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize));
    cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize));
    LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).minFactor(0.66f).acceptableFactor(0.99f).singleFactor(0.33f).multiFactor(0.33f).memoryFactor(0.34f).buildMap().forEach(cc::set);
    manager.start(new BlockCacheConfiguration(cc));
    LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX);
    Block[] singleBlocks = generateFixedBlocks(20, blockSize, "single");
    Block[] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
    // Add 5 multi blocks
    for (Block block : multiBlocks) {
        cache.cacheBlock(block.blockName, block.buf);
        cache.getBlock(block.blockName);
    }
    // Add 5 single blocks
    for (int i = 0; i < 5; i++) {
        cache.cacheBlock(singleBlocks[i].blockName, singleBlocks[i].buf);
    }
    // An eviction ran
    assertEquals(1, cache.getEvictionCount());
    // To drop down to 2/3 capacity, we'll need to evict 4 blocks
    assertEquals(4, cache.getEvictedCount());
    // Should have been taken off equally from single and multi
    assertEquals(null, cache.getBlock(singleBlocks[0].blockName));
    assertEquals(null, cache.getBlock(singleBlocks[1].blockName));
    assertEquals(null, cache.getBlock(multiBlocks[0].blockName));
    assertEquals(null, cache.getBlock(multiBlocks[1].blockName));
    for (int i = 5; i < 18; i++) {
        cache.cacheBlock(singleBlocks[i].blockName, singleBlocks[i].buf);
    }
    // 4 total evictions, 16 total evicted
    assertEquals(4, cache.getEvictionCount());
    assertEquals(16, cache.getEvictedCount());
    // Should now have 7 total blocks
    assertEquals(7, cache.size());
    manager.stop();
}

Example 4 with LruBlockCache

use of org.apache.accumulo.core.file.blockfile.cache.lru.LruBlockCache in project accumulo by apache.

the class TestLruBlockCache method testBackgroundEvictionThread.

public void testBackgroundEvictionThread() throws Exception {
    long maxSize = 100000;
    // room for 9, will evict
    long blockSize = calculateBlockSizeDefault(maxSize, 9);
    DefaultConfiguration dc = DefaultConfiguration.getInstance();
    ConfigurationCopy cc = new ConfigurationCopy(dc);
    cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
    BlockCacheManager manager = BlockCacheManagerFactory.getInstance(cc);
    cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize));
    cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize));
    manager.start(new BlockCacheConfiguration(cc));
    LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX);
    Block[] blocks = generateFixedBlocks(10, blockSize, "block");
    // Add all the blocks
    for (Block block : blocks) {
        cache.cacheBlock(block.blockName, block.buf);
    }
    // Let the eviction run
    int n = 0;
    while (cache.getEvictionCount() == 0) {
        Thread.sleep(1000);
        assertTrue(n++ < 1);
    }
    // A single eviction run should have occurred
    assertEquals(cache.getEvictionCount(), 1);
    manager.stop();
}

Example 5 with LruBlockCache

use of org.apache.accumulo.core.file.blockfile.cache.lru.LruBlockCache in project accumulo by apache.

the class TestLruBlockCache method testCacheEvictionTwoPriorities.

public void testCacheEvictionTwoPriorities() throws Exception {
    long maxSize = 100000;
    long blockSize = calculateBlockSizeDefault(maxSize, 10);
    DefaultConfiguration dc = DefaultConfiguration.getInstance();
    ConfigurationCopy cc = new ConfigurationCopy(dc);
    cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
    BlockCacheManager manager = BlockCacheManagerFactory.getInstance(cc);
    cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize));
    cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize));
    LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).minFactor(0.98f).acceptableFactor(0.99f).singleFactor(0.25f).multiFactor(0.50f).memoryFactor(0.25f).buildMap().forEach(cc::set);
    manager.start(new BlockCacheConfiguration(cc));
    LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX);
    Block[] singleBlocks = generateFixedBlocks(5, 10000, "single");
    Block[] multiBlocks = generateFixedBlocks(5, 10000, "multi");
    long expectedCacheSize = cache.heapSize();
    // Add and get the multi blocks
    for (Block block : multiBlocks) {
        cache.cacheBlock(block.blockName, block.buf);
        expectedCacheSize += block.heapSize();
        assertTrue(Arrays.equals(cache.getBlock(block.blockName).getBuffer(), block.buf));
    }
    // Add the single blocks (no get)
    for (Block block : singleBlocks) {
        cache.cacheBlock(block.blockName, block.buf);
        expectedCacheSize += block.heapSize();
    }
    // A single eviction run should have occurred
    assertEquals(cache.getEvictionCount(), 1);
    // We expect two entries evicted
    assertEquals(cache.getEvictedCount(), 2);
    // Our expected size overruns acceptable limit
    assertTrue(expectedCacheSize > (maxSize * LruBlockCacheConfiguration.DEFAULT_ACCEPTABLE_FACTOR));
    // But the cache did not grow beyond max
    assertTrue(cache.heapSize() <= maxSize);
    // And is now below the acceptable limit
    assertTrue(cache.heapSize() <= (maxSize * LruBlockCacheConfiguration.DEFAULT_ACCEPTABLE_FACTOR));
    // We expect fairness across the two priorities.
    // This test makes multi go barely over its limit, in-memory
    // empty, and the rest in single. Two single evictions and
    // one multi eviction expected.
    assertTrue(cache.getBlock(singleBlocks[0].blockName) == null);
    assertTrue(cache.getBlock(multiBlocks[0].blockName) == null);
    // And all others to be cached
    for (int i = 1; i < 4; i++) {
        assertTrue(Arrays.equals(cache.getBlock(singleBlocks[i].blockName).getBuffer(), singleBlocks[i].buf));
        assertTrue(Arrays.equals(cache.getBlock(multiBlocks[i].blockName).getBuffer(), multiBlocks[i].buf));
    }
    manager.stop();
}