Examples with BlockCacheManager org.apache.accumulo.core.spi.cache.BlockCacheManager used on opensource projects

Example 6 with BlockCacheManager

use of org.apache.accumulo.core.spi.cache.BlockCacheManager in project accumulo by apache.

the class BlockCacheFactoryTest method testStartWithDefault.

@Test
public void testStartWithDefault() throws Exception {
    DefaultConfiguration dc = DefaultConfiguration.getInstance();
    BlockCacheManager manager = BlockCacheManagerFactory.getInstance(dc);
    manager.start(new BlockCacheConfiguration(dc));
    assertNotNull(manager.getBlockCache(CacheType.INDEX));
}

Example 7 with BlockCacheManager

use of org.apache.accumulo.core.spi.cache.BlockCacheManager in project accumulo by apache.

the class TestLruBlockCache method testCacheEvictionThreePriorities.

@Test
public void testCacheEvictionThreePriorities() throws Exception {
    long maxSize = 100000;
    long blockSize = calculateBlockSize(maxSize, 10);
    DefaultConfiguration dc = DefaultConfiguration.getInstance();
    ConfigurationCopy cc = new ConfigurationCopy(dc);
    cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
    BlockCacheManager manager = BlockCacheManagerFactory.getInstance(cc);
    cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize));
    cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize));
    LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).minFactor(0.98f).acceptableFactor(0.99f).singleFactor(0.33f).multiFactor(0.33f).memoryFactor(0.34f).buildMap().forEach(cc::set);
    manager.start(new BlockCacheConfiguration(cc));
    LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX);
    Block[] singleBlocks = generateFixedBlocks(5, blockSize, "single");
    Block[] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
    Block[] memoryBlocks = generateFixedBlocks(5, blockSize, "memory");
    long expectedCacheSize = cache.heapSize();
    // Add 3 blocks from each priority
    for (int i = 0; i < 3; i++) {
        // Just add single blocks
        cache.cacheBlock(singleBlocks[i].blockName, singleBlocks[i].buf);
        expectedCacheSize += singleBlocks[i].heapSize();
        // Add and get multi blocks
        cache.cacheBlock(multiBlocks[i].blockName, multiBlocks[i].buf);
        expectedCacheSize += multiBlocks[i].heapSize();
        cache.getBlock(multiBlocks[i].blockName);
        // Add memory blocks as such
        cache.cacheBlock(memoryBlocks[i].blockName, memoryBlocks[i].buf, true);
        expectedCacheSize += memoryBlocks[i].heapSize();
    }
    // Do not expect any evictions yet
    assertEquals(0, cache.getEvictionCount());
    // Verify cache size
    assertEquals(expectedCacheSize, cache.heapSize());
    // Insert a single block, oldest single should be evicted
    cache.cacheBlock(singleBlocks[3].blockName, singleBlocks[3].buf);
    // Single eviction, one thing evicted
    assertEquals(1, cache.getEvictionCount());
    assertEquals(1, cache.getEvictedCount());
    // Verify oldest single block is the one evicted
    assertNull(cache.getBlock(singleBlocks[0].blockName));
    // Change the oldest remaining single block to a multi
    cache.getBlock(singleBlocks[1].blockName);
    // Insert another single block
    cache.cacheBlock(singleBlocks[4].blockName, singleBlocks[4].buf);
    // Two evictions, two evicted.
    assertEquals(2, cache.getEvictionCount());
    assertEquals(2, cache.getEvictedCount());
    // Oldest multi block should be evicted now
    assertNull(cache.getBlock(multiBlocks[0].blockName));
    // Insert another memory block
    cache.cacheBlock(memoryBlocks[3].blockName, memoryBlocks[3].buf, true);
    // Three evictions, three evicted.
    assertEquals(3, cache.getEvictionCount());
    assertEquals(3, cache.getEvictedCount());
    // Oldest memory block should be evicted now
    assertNull(cache.getBlock(memoryBlocks[0].blockName));
    // Add a block that is twice as big (should force two evictions)
    Block[] bigBlocks = generateFixedBlocks(3, blockSize * 3, "big");
    cache.cacheBlock(bigBlocks[0].blockName, bigBlocks[0].buf);
    // Four evictions, six evicted (inserted block 3X size, expect +3 evicted)
    assertEquals(4, cache.getEvictionCount());
    assertEquals(6, cache.getEvictedCount());
    // Expect three remaining singles to be evicted
    assertNull(cache.getBlock(singleBlocks[2].blockName));
    assertNull(cache.getBlock(singleBlocks[3].blockName));
    assertNull(cache.getBlock(singleBlocks[4].blockName));
    // Make the big block a multi block
    cache.getBlock(bigBlocks[0].blockName);
    // Cache another single big block
    cache.cacheBlock(bigBlocks[1].blockName, bigBlocks[1].buf);
    // Five evictions, nine evicted (3 new)
    assertEquals(5, cache.getEvictionCount());
    assertEquals(9, cache.getEvictedCount());
    // Expect three remaining multis to be evicted
    assertNull(cache.getBlock(singleBlocks[1].blockName));
    assertNull(cache.getBlock(multiBlocks[1].blockName));
    assertNull(cache.getBlock(multiBlocks[2].blockName));
    // Cache a big memory block
    cache.cacheBlock(bigBlocks[2].blockName, bigBlocks[2].buf, true);
    // Six evictions, twelve evicted (3 new)
    assertEquals(6, cache.getEvictionCount());
    assertEquals(12, cache.getEvictedCount());
    // Expect three remaining in-memory to be evicted
    assertNull(cache.getBlock(memoryBlocks[1].blockName));
    assertNull(cache.getBlock(memoryBlocks[2].blockName));
    assertNull(cache.getBlock(memoryBlocks[3].blockName));
    manager.stop();
}

Example 8 with BlockCacheManager

use of org.apache.accumulo.core.spi.cache.BlockCacheManager in project accumulo by apache.

the class RFileTest method runVersionTest.

private void runVersionTest(int version, ConfigurationCopy aconf) throws Exception {
    InputStream in = this.getClass().getClassLoader().getResourceAsStream("org/apache/accumulo/core/file/rfile/ver_" + version + ".rf");
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    byte[] buf = new byte[1024];
    int read;
    while ((read = in.read(buf)) > 0) baos.write(buf, 0, read);
    byte[] data = baos.toByteArray();
    SeekableByteArrayInputStream bais = new SeekableByteArrayInputStream(data);
    FSDataInputStream in2 = new FSDataInputStream(bais);
    aconf.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
    aconf.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(100000));
    aconf.set(Property.TSERV_DATACACHE_SIZE, Long.toString(100000000));
    aconf.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(100000000));
    BlockCacheManager manager = BlockCacheManagerFactory.getInstance(aconf);
    manager.start(new BlockCacheConfiguration(aconf));
    CachableBuilder cb = new CachableBuilder().input(in2, "cache-1").length(data.length).conf(hadoopConf).cryptoService(CryptoServiceFactory.newInstance(aconf, ClassloaderType.JAVA)).cacheProvider(new BasicCacheProvider(manager.getBlockCache(CacheType.INDEX), manager.getBlockCache(CacheType.DATA)));
    Reader reader = new RFile.Reader(cb);
    checkIndex(reader);
    ColumnFamilySkippingIterator iter = new ColumnFamilySkippingIterator(reader);
    for (int start : new int[] { 0, 10, 100, 998 }) {
        for (int cf = 1; cf <= 4; cf++) {
            if (start == 0)
                iter.seek(new Range(), newColFamByteSequence(formatString("cf_", cf)), true);
            else
                iter.seek(new Range(formatString("r_", start), null), newColFamByteSequence(formatString("cf_", cf)), true);
            for (int i = start; i < 1000; i++) {
                assertTrue(iter.hasTop());
                assertEquals(newKey(formatString("r_", i), formatString("cf_", cf), formatString("cq_", 0), "", 1000 - i), iter.getTopKey());
                assertEquals(newValue(i + ""), iter.getTopValue());
                iter.next();
            }
            assertFalse(iter.hasTop());
        }
        if (start == 0)
            iter.seek(new Range(), newColFamByteSequence(), false);
        else
            iter.seek(new Range(formatString("r_", start), null), newColFamByteSequence(), false);
        for (int i = start; i < 1000; i++) {
            for (int cf = 1; cf <= 4; cf++) {
                assertTrue(iter.hasTop());
                assertEquals(newKey(formatString("r_", i), formatString("cf_", cf), formatString("cq_", 0), "", 1000 - i), iter.getTopKey());
                assertEquals(newValue(i + ""), iter.getTopValue());
                iter.next();
            }
        }
        assertFalse(iter.hasTop());
    }
    manager.stop();
    reader.close();
}