Examples with BlockCacheConfiguration org.apache.accumulo.core.file.blockfile.cache.impl.BlockCacheConfiguration used on opensource projects

Example 6 with BlockCacheConfiguration

use of org.apache.accumulo.core.file.blockfile.cache.impl.BlockCacheConfiguration in project accumulo by apache.

the class TestLruBlockCache method testConfiguration.

public void testConfiguration() {
    ConfigurationCopy cc = new ConfigurationCopy();
    cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
    cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(1019));
    cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(1000023));
    cc.set(Property.TSERV_DATACACHE_SIZE, Long.toString(1000027));
    cc.set(Property.TSERV_SUMMARYCACHE_SIZE, Long.toString(1000029));
    LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).minFactor(0.93f).acceptableFactor(0.97f).singleFactor(0.20f).multiFactor(0.30f).memoryFactor(0.50f).mapConcurrencyLevel(5).buildMap().forEach(cc::set);
    String defaultPrefix = BlockCacheManager.CACHE_PROPERTY_BASE + LruBlockCacheConfiguration.PROPERTY_PREFIX + ".default.";
    // this should be overridden by cache type specific setting
    cc.set(defaultPrefix + LruBlockCacheConfiguration.MEMORY_FACTOR_PROPERTY, "0.6");
    // this is not set for the cache type, so should fall back to default
    cc.set(defaultPrefix + LruBlockCacheConfiguration.MAP_LOAD_PROPERTY, "0.53");
    BlockCacheConfiguration bcc = new BlockCacheConfiguration(cc);
    LruBlockCacheConfiguration lbcc = new LruBlockCacheConfiguration(bcc, CacheType.INDEX);
    assertEquals(false, lbcc.isUseEvictionThread());
    assertEquals(0.93f, lbcc.getMinFactor());
    assertEquals(0.97f, lbcc.getAcceptableFactor());
    assertEquals(0.20f, lbcc.getSingleFactor());
    assertEquals(0.30f, lbcc.getMultiFactor());
    assertEquals(0.50f, lbcc.getMemoryFactor());
    assertEquals(0.53f, lbcc.getMapLoadFactor());
    assertEquals(5, lbcc.getMapConcurrencyLevel());
    assertEquals(1019, lbcc.getBlockSize());
    assertEquals(1000023, lbcc.getMaxSize());
}

Example 7 with BlockCacheConfiguration

use of org.apache.accumulo.core.file.blockfile.cache.impl.BlockCacheConfiguration in project accumulo by apache.

the class TestLruBlockCache method testCacheEvictionTwoPriorities.

public void testCacheEvictionTwoPriorities() throws Exception {
    long maxSize = 100000;
    long blockSize = calculateBlockSizeDefault(maxSize, 10);
    DefaultConfiguration dc = DefaultConfiguration.getInstance();
    ConfigurationCopy cc = new ConfigurationCopy(dc);
    cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
    BlockCacheManager manager = BlockCacheManagerFactory.getInstance(cc);
    cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize));
    cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize));
    LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).minFactor(0.98f).acceptableFactor(0.99f).singleFactor(0.25f).multiFactor(0.50f).memoryFactor(0.25f).buildMap().forEach(cc::set);
    manager.start(new BlockCacheConfiguration(cc));
    LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX);
    Block[] singleBlocks = generateFixedBlocks(5, 10000, "single");
    Block[] multiBlocks = generateFixedBlocks(5, 10000, "multi");
    long expectedCacheSize = cache.heapSize();
    // Add and get the multi blocks
    for (Block block : multiBlocks) {
        cache.cacheBlock(block.blockName, block.buf);
        expectedCacheSize += block.heapSize();
        assertTrue(Arrays.equals(cache.getBlock(block.blockName).getBuffer(), block.buf));
    }
    // Add the single blocks (no get)
    for (Block block : singleBlocks) {
        cache.cacheBlock(block.blockName, block.buf);
        expectedCacheSize += block.heapSize();
    }
    // A single eviction run should have occurred
    assertEquals(cache.getEvictionCount(), 1);
    // We expect two entries evicted
    assertEquals(cache.getEvictedCount(), 2);
    // Our expected size overruns acceptable limit
    assertTrue(expectedCacheSize > (maxSize * LruBlockCacheConfiguration.DEFAULT_ACCEPTABLE_FACTOR));
    // But the cache did not grow beyond max
    assertTrue(cache.heapSize() <= maxSize);
    // And is now below the acceptable limit
    assertTrue(cache.heapSize() <= (maxSize * LruBlockCacheConfiguration.DEFAULT_ACCEPTABLE_FACTOR));
    // We expect fairness across the two priorities.
    // This test makes multi go barely over its limit, in-memory
    // empty, and the rest in single. Two single evictions and
    // one multi eviction expected.
    assertTrue(cache.getBlock(singleBlocks[0].blockName) == null);
    assertTrue(cache.getBlock(multiBlocks[0].blockName) == null);
    // And all others to be cached
    for (int i = 1; i < 4; i++) {
        assertTrue(Arrays.equals(cache.getBlock(singleBlocks[i].blockName).getBuffer(), singleBlocks[i].buf));
        assertTrue(Arrays.equals(cache.getBlock(multiBlocks[i].blockName).getBuffer(), multiBlocks[i].buf));
    }
    manager.stop();
}

Example 8 with BlockCacheConfiguration

use of org.apache.accumulo.core.file.blockfile.cache.impl.BlockCacheConfiguration in project accumulo by apache.

the class TestLruBlockCache method testCacheEvictionThreePriorities.

public void testCacheEvictionThreePriorities() throws Exception {
    long maxSize = 100000;
    long blockSize = calculateBlockSize(maxSize, 10);
    DefaultConfiguration dc = DefaultConfiguration.getInstance();
    ConfigurationCopy cc = new ConfigurationCopy(dc);
    cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
    BlockCacheManager manager = BlockCacheManagerFactory.getInstance(cc);
    cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize));
    cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize));
    LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).minFactor(0.98f).acceptableFactor(0.99f).singleFactor(0.33f).multiFactor(0.33f).memoryFactor(0.34f).buildMap().forEach(cc::set);
    manager.start(new BlockCacheConfiguration(cc));
    LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX);
    Block[] singleBlocks = generateFixedBlocks(5, blockSize, "single");
    Block[] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
    Block[] memoryBlocks = generateFixedBlocks(5, blockSize, "memory");
    long expectedCacheSize = cache.heapSize();
    // Add 3 blocks from each priority
    for (int i = 0; i < 3; i++) {
        // Just add single blocks
        cache.cacheBlock(singleBlocks[i].blockName, singleBlocks[i].buf);
        expectedCacheSize += singleBlocks[i].heapSize();
        // Add and get multi blocks
        cache.cacheBlock(multiBlocks[i].blockName, multiBlocks[i].buf);
        expectedCacheSize += multiBlocks[i].heapSize();
        cache.getBlock(multiBlocks[i].blockName);
        // Add memory blocks as such
        cache.cacheBlock(memoryBlocks[i].blockName, memoryBlocks[i].buf, true);
        expectedCacheSize += memoryBlocks[i].heapSize();
    }
    // Do not expect any evictions yet
    assertEquals(0, cache.getEvictionCount());
    // Verify cache size
    assertEquals(expectedCacheSize, cache.heapSize());
    // Insert a single block, oldest single should be evicted
    cache.cacheBlock(singleBlocks[3].blockName, singleBlocks[3].buf);
    // Single eviction, one thing evicted
    assertEquals(1, cache.getEvictionCount());
    assertEquals(1, cache.getEvictedCount());
    // Verify oldest single block is the one evicted
    assertEquals(null, cache.getBlock(singleBlocks[0].blockName));
    // Change the oldest remaining single block to a multi
    cache.getBlock(singleBlocks[1].blockName);
    // Insert another single block
    cache.cacheBlock(singleBlocks[4].blockName, singleBlocks[4].buf);
    // Two evictions, two evicted.
    assertEquals(2, cache.getEvictionCount());
    assertEquals(2, cache.getEvictedCount());
    // Oldest multi block should be evicted now
    assertEquals(null, cache.getBlock(multiBlocks[0].blockName));
    // Insert another memory block
    cache.cacheBlock(memoryBlocks[3].blockName, memoryBlocks[3].buf, true);
    // Three evictions, three evicted.
    assertEquals(3, cache.getEvictionCount());
    assertEquals(3, cache.getEvictedCount());
    // Oldest memory block should be evicted now
    assertEquals(null, cache.getBlock(memoryBlocks[0].blockName));
    // Add a block that is twice as big (should force two evictions)
    Block[] bigBlocks = generateFixedBlocks(3, blockSize * 3, "big");
    cache.cacheBlock(bigBlocks[0].blockName, bigBlocks[0].buf);
    // Four evictions, six evicted (inserted block 3X size, expect +3 evicted)
    assertEquals(4, cache.getEvictionCount());
    assertEquals(6, cache.getEvictedCount());
    // Expect three remaining singles to be evicted
    assertEquals(null, cache.getBlock(singleBlocks[2].blockName));
    assertEquals(null, cache.getBlock(singleBlocks[3].blockName));
    assertEquals(null, cache.getBlock(singleBlocks[4].blockName));
    // Make the big block a multi block
    cache.getBlock(bigBlocks[0].blockName);
    // Cache another single big block
    cache.cacheBlock(bigBlocks[1].blockName, bigBlocks[1].buf);
    // Five evictions, nine evicted (3 new)
    assertEquals(5, cache.getEvictionCount());
    assertEquals(9, cache.getEvictedCount());
    // Expect three remaining multis to be evicted
    assertEquals(null, cache.getBlock(singleBlocks[1].blockName));
    assertEquals(null, cache.getBlock(multiBlocks[1].blockName));
    assertEquals(null, cache.getBlock(multiBlocks[2].blockName));
    // Cache a big memory block
    cache.cacheBlock(bigBlocks[2].blockName, bigBlocks[2].buf, true);
    // Six evictions, twelve evicted (3 new)
    assertEquals(6, cache.getEvictionCount());
    assertEquals(12, cache.getEvictedCount());
    // Expect three remaining in-memory to be evicted
    assertEquals(null, cache.getBlock(memoryBlocks[1].blockName));
    assertEquals(null, cache.getBlock(memoryBlocks[2].blockName));
    assertEquals(null, cache.getBlock(memoryBlocks[3].blockName));
    manager.stop();
}