Examples with ByteBuffAllocator org.apache.hadoop.hbase.io.ByteBuffAllocator used on opensource projects

Example 6 with ByteBuffAllocator

use of org.apache.hadoop.hbase.io.ByteBuffAllocator in project hbase by apache.

the class TestBucketCacheRefCnt method testInBucketCache.

@Test
public void testInBucketCache() throws IOException {
    ByteBuffAllocator alloc = ByteBuffAllocator.create(HBaseConfiguration.create(), true);
    cache = create(1, 1000);
    try {
        HFileBlock blk = createBlock(200, 1020, alloc);
        BlockCacheKey key = createKey("testHFile-00", 200);
        cache.cacheBlock(key, blk);
        assertTrue(blk.refCnt() == 1 || blk.refCnt() == 2);
        Cacheable block1 = cache.getBlock(key, false, false, false);
        assertTrue(block1.refCnt() >= 2);
        assertTrue(((HFileBlock) block1).getByteBuffAllocator() == alloc);
        Cacheable block2 = cache.getBlock(key, false, false, false);
        assertTrue(((HFileBlock) block2).getByteBuffAllocator() == alloc);
        assertTrue(block2.refCnt() >= 3);
        cache.evictBlock(key);
        assertTrue(blk.refCnt() >= 1);
        assertTrue(block1.refCnt() >= 2);
        assertTrue(block2.refCnt() >= 2);
        // Get key again
        Cacheable block3 = cache.getBlock(key, false, false, false);
        if (block3 != null) {
            assertTrue(((HFileBlock) block3).getByteBuffAllocator() == alloc);
            assertTrue(block3.refCnt() >= 3);
            assertFalse(block3.release());
        }
        blk.release();
        boolean ret1 = block1.release();
        boolean ret2 = block2.release();
        assertTrue(ret1 || ret2);
        assertEquals(0, blk.refCnt());
        assertEquals(0, block1.refCnt());
        assertEquals(0, block2.refCnt());
    } finally {
        cache.shutdown();
    }
}

Example 7 with ByteBuffAllocator

use of org.apache.hadoop.hbase.io.ByteBuffAllocator in project hbase by apache.

the class TestBucketCacheRefCnt method testReplacingBlockAndGettingBlockConcurrently.

/**
 * <pre>
 * This test is for HBASE-26281,
 * test two threads for replacing Block and getting Block execute concurrently.
 * The threads sequence is:
 * 1. Block1 was cached successfully,the {@link RefCnt} of Block1 is 1.
 * 2. Thread1 caching the same {@link BlockCacheKey} with Block2 satisfied
 *    {@link BlockCacheUtil#shouldReplaceExistingCacheBlock}, so Block2 would
 *    replace Block1, but thread1 stopping before {@link BucketCache#cacheBlockWithWaitInternal}
 * 3. Thread2 invoking {@link BucketCache#getBlock} with the same {@link BlockCacheKey},
 *    which returned Block1, the {@link RefCnt} of Block1 is 2.
 * 4. Thread1 continues caching Block2, in {@link BucketCache.WriterThread#putIntoBackingMap},
 *    the old Block1 is freed directly which {@link RefCnt} is 2, but the Block1 is still used
 *    by Thread2 and the content of Block1 would be overwritten after it is freed, which may
 *    cause a serious error.
 * </pre>
 * @throws Exception
 */
@Test
public void testReplacingBlockAndGettingBlockConcurrently() throws Exception {
    ByteBuffAllocator byteBuffAllocator = ByteBuffAllocator.create(HBaseConfiguration.create(), true);
    final MyBucketCache myBucketCache = createMyBucketCache(1, 1000);
    try {
        HFileBlock hfileBlock = createBlock(200, 1020, byteBuffAllocator);
        final BlockCacheKey blockCacheKey = createKey("testTwoThreadConcurrent", 200);
        myBucketCache.cacheBlock(blockCacheKey, hfileBlock);
        waitUntilFlushedToCache(myBucketCache, blockCacheKey);
        assertEquals(1, hfileBlock.refCnt());
        assertTrue(!myBucketCache.ramCache.containsKey(blockCacheKey));
        final AtomicReference<Throwable> exceptionRef = new AtomicReference<Throwable>();
        Thread cacheBlockThread = new Thread(() -> {
            try {
                HFileBlock newHFileBlock = createBlock(200, 1020, byteBuffAllocator);
                myBucketCache.cacheBlock(blockCacheKey, newHFileBlock);
                waitUntilFlushedToCache(myBucketCache, blockCacheKey);
            } catch (Throwable exception) {
                exceptionRef.set(exception);
            }
        });
        cacheBlockThread.setName(MyBucketCache.CACHE_BLOCK_THREAD_NAME);
        cacheBlockThread.start();
        String oldThreadName = Thread.currentThread().getName();
        HFileBlock gotHFileBlock = null;
        try {
            Thread.currentThread().setName(MyBucketCache.GET_BLOCK_THREAD_NAME);
            gotHFileBlock = (HFileBlock) (myBucketCache.getBlock(blockCacheKey, false, false, false));
            assertTrue(gotHFileBlock.equals(hfileBlock));
            assertTrue(gotHFileBlock.getByteBuffAllocator() == byteBuffAllocator);
            assertEquals(2, gotHFileBlock.refCnt());
            /**
             * Release the second cyclicBarrier.await in
             * {@link MyBucketCache#cacheBlockWithWaitInternal}
             */
            myBucketCache.cyclicBarrier.await();
        } finally {
            Thread.currentThread().setName(oldThreadName);
        }
        cacheBlockThread.join();
        assertTrue(exceptionRef.get() == null);
        assertEquals(1, gotHFileBlock.refCnt());
        assertTrue(gotHFileBlock.equals(hfileBlock));
        assertTrue(myBucketCache.overwiteByteBuff == null);
        assertTrue(myBucketCache.freeBucketEntryCounter.get() == 0);
        gotHFileBlock.release();
        assertEquals(0, gotHFileBlock.refCnt());
        assertTrue(myBucketCache.overwiteByteBuff != null);
        assertTrue(myBucketCache.freeBucketEntryCounter.get() == 1);
        assertTrue(myBucketCache.replaceCounter.get() == 1);
        assertTrue(myBucketCache.blockEvictCounter.get() == 1);
    } finally {
        myBucketCache.shutdown();
    }
}

Example 8 with ByteBuffAllocator

use of org.apache.hadoop.hbase.io.ByteBuffAllocator in project hbase by apache.

the class TestBucketCacheRefCnt method testBlockInBackingMap.

@Test
public void testBlockInBackingMap() throws Exception {
    ByteBuffAllocator alloc = ByteBuffAllocator.create(HBaseConfiguration.create(), true);
    cache = create(1, 1000);
    try {
        HFileBlock blk = createBlock(200, 1020, alloc);
        BlockCacheKey key = createKey("testHFile-00", 200);
        cache.cacheBlock(key, blk);
        waitUntilFlushedToCache(cache, key);
        assertEquals(1, blk.refCnt());
        Cacheable block = cache.getBlock(key, false, false, false);
        assertTrue(block instanceof HFileBlock);
        assertTrue(((HFileBlock) block).getByteBuffAllocator() == alloc);
        assertEquals(2, block.refCnt());
        block.retain();
        assertEquals(3, block.refCnt());
        Cacheable newBlock = cache.getBlock(key, false, false, false);
        assertTrue(newBlock instanceof HFileBlock);
        assertTrue(((HFileBlock) newBlock).getByteBuffAllocator() == alloc);
        assertEquals(4, newBlock.refCnt());
        // release the newBlock
        assertFalse(newBlock.release());
        assertEquals(3, newBlock.refCnt());
        assertEquals(3, block.refCnt());
        // Evict the key
        cache.evictBlock(key);
        assertEquals(2, block.refCnt());
        // Evict again, shouldn't change the refCnt.
        cache.evictBlock(key);
        assertEquals(2, block.refCnt());
        assertFalse(block.release());
        assertEquals(1, block.refCnt());
        /**
         * The key was evicted from {@link BucketCache#backingMap} and {@link BucketCache#ramCache},
         * so {@link BucketCache#getBlock} return null.
         */
        Cacheable newestBlock = cache.getBlock(key, false, false, false);
        assertNull(newestBlock);
        assertEquals(1, block.refCnt());
        assertTrue(((HFileBlock) newBlock).getByteBuffAllocator() == alloc);
        // Release the block
        assertTrue(block.release());
        assertEquals(0, block.refCnt());
        assertEquals(0, newBlock.refCnt());
    } finally {
        cache.shutdown();
    }
}

Example 9 with ByteBuffAllocator

use of org.apache.hadoop.hbase.io.ByteBuffAllocator in project hbase by apache.

the class TestFileIOEngine method testReadFailedShouldReleaseByteBuff.

@Test
public void testReadFailedShouldReleaseByteBuff() {
    ByteBuffAllocator alloc = Mockito.mock(ByteBuffAllocator.class);
    final RefCnt refCnt = RefCnt.create();
    Mockito.when(alloc.allocate(Mockito.anyInt())).thenAnswer(new Answer<ByteBuff>() {

        @Override
        public ByteBuff answer(InvocationOnMock invocation) throws Throwable {
            int len = invocation.getArgument(0);
            return ByteBuff.wrap(new ByteBuffer[] { ByteBuffer.allocate(len + 1) }, refCnt);
        }
    });
    int len = 10;
    byte[] data1 = new byte[len];
    assertEquals(1, refCnt.refCnt());
    try {
        fileIOEngine.write(ByteBuffer.wrap(data1), 0);
        BucketEntry be = createBucketEntry(0, len, alloc);
        fileIOEngine.read(be);
        fail();
    } catch (IOException ioe) {
    // expected exception.
    }
    assertEquals(0, refCnt.refCnt());
}

Example 10 with ByteBuffAllocator

use of org.apache.hadoop.hbase.io.ByteBuffAllocator in project hbase by apache.

the class TestHFile method testReaderCombinedCache.

private void testReaderCombinedCache(final String l1CachePolicy) throws Exception {
    int bufCount = 1024;
    int blockSize = 64 * 1024;
    ByteBuffAllocator alloc = initAllocator(true, bufCount, blockSize, 0);
    fillByteBuffAllocator(alloc, bufCount);
    Path storeFilePath = writeStoreFile();
    // Open the file reader with CombinedBlockCache
    BlockCache combined = initCombinedBlockCache(l1CachePolicy);
    conf.setBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, true);
    CacheConfig cacheConfig = new CacheConfig(conf, null, combined, alloc);
    HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig, true, conf);
    long offset = 0;
    Cacheable cachedBlock = null;
    while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
        BlockCacheKey key = new BlockCacheKey(storeFilePath.getName(), offset);
        HFileBlock block = reader.readBlock(offset, -1, true, true, false, true, null, null);
        offset += block.getOnDiskSizeWithHeader();
        // Read the cached block.
        cachedBlock = combined.getBlock(key, false, false, true);
        try {
            Assert.assertNotNull(cachedBlock);
            Assert.assertTrue(cachedBlock instanceof HFileBlock);
            HFileBlock hfb = (HFileBlock) cachedBlock;
            // Data block will be cached in BucketCache, so it should be an off-heap block.
            if (hfb.getBlockType().isData()) {
                Assert.assertTrue(hfb.isSharedMem());
            } else if (!l1CachePolicy.equals("TinyLfu")) {
                Assert.assertFalse(hfb.isSharedMem());
            }
        } finally {
            cachedBlock.release();
        }
        // return back the ByteBuffer back to allocator.
        block.release();
    }
    reader.close();
    combined.shutdown();
    if (cachedBlock != null) {
        Assert.assertEquals(0, cachedBlock.refCnt());
    }
    Assert.assertEquals(bufCount, alloc.getFreeBufferCount());
    alloc.clean();
}