Examples with ByteBuffAllocator org.apache.hadoop.hbase.io.ByteBuffAllocator used on opensource projects

Example 1 with ByteBuffAllocator

use of org.apache.hadoop.hbase.io.ByteBuffAllocator in project hbase by apache.

the class TestHFile method testReaderWithoutBlockCache.

@Test
public void testReaderWithoutBlockCache() throws Exception {
    int bufCount = 32;
    // AllByteBuffers will be allocated from the buffers.
    ByteBuffAllocator alloc = initAllocator(true, 64 * 1024, bufCount, 0);
    fillByteBuffAllocator(alloc, bufCount);
    // start write to store file.
    Path path = writeStoreFile();
    try {
        readStoreFile(path, conf, alloc);
    } catch (Exception e) {
        // fail test
        assertTrue(false);
    }
    Assert.assertEquals(bufCount, alloc.getFreeBufferCount());
    alloc.clean();
}

Example 2 with ByteBuffAllocator

use of org.apache.hadoop.hbase.io.ByteBuffAllocator in project hbase by apache.

the class TestHFile method testReaderWithLRUBlockCache.

/**
 * Test case for HBASE-22127 in LruBlockCache.
 */
@Test
public void testReaderWithLRUBlockCache() throws Exception {
    int bufCount = 1024, blockSize = 64 * 1024;
    ByteBuffAllocator alloc = initAllocator(true, bufCount, blockSize, 0);
    fillByteBuffAllocator(alloc, bufCount);
    Path storeFilePath = writeStoreFile();
    // Open the file reader with LRUBlockCache
    BlockCache lru = new LruBlockCache(1024 * 1024 * 32, blockSize, true, conf);
    CacheConfig cacheConfig = new CacheConfig(conf, null, lru, alloc);
    HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConfig, true, conf);
    long offset = 0;
    while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
        BlockCacheKey key = new BlockCacheKey(storeFilePath.getName(), offset);
        HFileBlock block = reader.readBlock(offset, -1, true, true, false, true, null, null);
        offset += block.getOnDiskSizeWithHeader();
        // Ensure the block is an heap one.
        Cacheable cachedBlock = lru.getBlock(key, false, false, true);
        Assert.assertNotNull(cachedBlock);
        Assert.assertTrue(cachedBlock instanceof HFileBlock);
        Assert.assertFalse(((HFileBlock) cachedBlock).isSharedMem());
        // Should never allocate off-heap block from allocator because ensure that it's LRU.
        Assert.assertEquals(bufCount, alloc.getFreeBufferCount());
        // return back the ByteBuffer back to allocator.
        block.release();
    }
    reader.close();
    Assert.assertEquals(bufCount, alloc.getFreeBufferCount());
    alloc.clean();
    lru.shutdown();
}

Example 3 with ByteBuffAllocator

use of org.apache.hadoop.hbase.io.ByteBuffAllocator in project hbase by apache.

the class TestBucketCache method testCacheBlockNextBlockMetadataMissing.

@Test
public void testCacheBlockNextBlockMetadataMissing() throws Exception {
    int size = 100;
    int length = HConstants.HFILEBLOCK_HEADER_SIZE + size;
    ByteBuffer buf1 = ByteBuffer.allocate(size), buf2 = ByteBuffer.allocate(size);
    HFileContext meta = new HFileContextBuilder().build();
    ByteBuffAllocator allocator = ByteBuffAllocator.HEAP;
    HFileBlock blockWithNextBlockMetadata = new HFileBlock(BlockType.DATA, size, size, -1, ByteBuff.wrap(buf1), HFileBlock.FILL_HEADER, -1, 52, -1, meta, allocator);
    HFileBlock blockWithoutNextBlockMetadata = new HFileBlock(BlockType.DATA, size, size, -1, ByteBuff.wrap(buf2), HFileBlock.FILL_HEADER, -1, -1, -1, meta, allocator);
    BlockCacheKey key = new BlockCacheKey("testCacheBlockNextBlockMetadataMissing", 0);
    ByteBuffer actualBuffer = ByteBuffer.allocate(length);
    ByteBuffer block1Buffer = ByteBuffer.allocate(length);
    ByteBuffer block2Buffer = ByteBuffer.allocate(length);
    blockWithNextBlockMetadata.serialize(block1Buffer, true);
    blockWithoutNextBlockMetadata.serialize(block2Buffer, true);
    // Add blockWithNextBlockMetadata, expect blockWithNextBlockMetadata back.
    CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithNextBlockMetadata, actualBuffer, block1Buffer);
    waitUntilFlushedToBucket(cache, key);
    assertNotNull(cache.backingMap.get(key));
    assertEquals(1, cache.backingMap.get(key).refCnt());
    assertEquals(1, blockWithNextBlockMetadata.getBufferReadOnly().refCnt());
    assertEquals(1, blockWithoutNextBlockMetadata.getBufferReadOnly().refCnt());
    // Add blockWithoutNextBlockMetada, expect blockWithNextBlockMetadata back.
    CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithoutNextBlockMetadata, actualBuffer, block1Buffer);
    assertEquals(1, blockWithNextBlockMetadata.getBufferReadOnly().refCnt());
    assertEquals(1, blockWithoutNextBlockMetadata.getBufferReadOnly().refCnt());
    assertEquals(1, cache.backingMap.get(key).refCnt());
    // Clear and add blockWithoutNextBlockMetadata
    assertTrue(cache.evictBlock(key));
    assertEquals(1, blockWithNextBlockMetadata.getBufferReadOnly().refCnt());
    assertEquals(1, blockWithoutNextBlockMetadata.getBufferReadOnly().refCnt());
    assertNull(cache.getBlock(key, false, false, false));
    CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithoutNextBlockMetadata, actualBuffer, block2Buffer);
    waitUntilFlushedToBucket(cache, key);
    assertEquals(1, blockWithNextBlockMetadata.getBufferReadOnly().refCnt());
    assertEquals(1, blockWithoutNextBlockMetadata.getBufferReadOnly().refCnt());
    // Add blockWithNextBlockMetadata, expect blockWithNextBlockMetadata to replace.
    CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithNextBlockMetadata, actualBuffer, block1Buffer);
    waitUntilFlushedToBucket(cache, key);
    assertEquals(1, blockWithNextBlockMetadata.getBufferReadOnly().refCnt());
    assertEquals(1, blockWithoutNextBlockMetadata.getBufferReadOnly().refCnt());
}

Example 4 with ByteBuffAllocator

use of org.apache.hadoop.hbase.io.ByteBuffAllocator in project hbase by apache.

the class TestBucketCacheRefCnt method testEvictingBlockCachingBlockGettingBlockConcurrently.

/**
 * <pre>
 * This test also is for HBASE-26281,
 * test three threads for evicting Block,caching Block and getting Block
 * execute concurrently.
 * 1. Thread1 caching Block1, stopping after {@link BucketCache.WriterThread#putIntoBackingMap},
 *    the {@link RefCnt} of Block1 is 1.
 * 2. Thread2 invoking {@link BucketCache#evictBlock} with the same {@link BlockCacheKey},
 *    but stopping after {@link BucketCache#removeFromRamCache}.
 * 3. Thread3 invoking {@link BucketCache#getBlock} with the same {@link BlockCacheKey},
 *    which returned Block1, the {@link RefCnt} of Block1 is 2.
 * 4. Thread1 continues caching block1,but finding that {@link BucketCache.RAMCache#remove}
 *    returning false, so invoking {@link BucketCache#blockEvicted} to free the the Block1
 *    directly which {@link RefCnt} is 2 and the Block1 is still used by Thread3.
 * </pre>
 */
@Test
public void testEvictingBlockCachingBlockGettingBlockConcurrently() throws Exception {
    ByteBuffAllocator byteBuffAllocator = ByteBuffAllocator.create(HBaseConfiguration.create(), true);
    final MyBucketCache2 myBucketCache2 = createMyBucketCache2(1, 1000);
    try {
        final HFileBlock hfileBlock = createBlock(200, 1020, byteBuffAllocator);
        final BlockCacheKey blockCacheKey = createKey("testThreeThreadConcurrent", 200);
        final AtomicReference<Throwable> cacheBlockThreadExceptionRef = new AtomicReference<Throwable>();
        Thread cacheBlockThread = new Thread(() -> {
            try {
                myBucketCache2.cacheBlock(blockCacheKey, hfileBlock);
                /**
                 * Wait for Caching Block completed.
                 */
                myBucketCache2.writeThreadDoneCyclicBarrier.await();
            } catch (Throwable exception) {
                cacheBlockThreadExceptionRef.set(exception);
            }
        });
        cacheBlockThread.setName(MyBucketCache2.CACHE_BLOCK_THREAD_NAME);
        cacheBlockThread.start();
        final AtomicReference<Throwable> evictBlockThreadExceptionRef = new AtomicReference<Throwable>();
        Thread evictBlockThread = new Thread(() -> {
            try {
                myBucketCache2.evictBlock(blockCacheKey);
            } catch (Throwable exception) {
                evictBlockThreadExceptionRef.set(exception);
            }
        });
        evictBlockThread.setName(MyBucketCache2.EVICT_BLOCK_THREAD_NAME);
        evictBlockThread.start();
        String oldThreadName = Thread.currentThread().getName();
        HFileBlock gotHFileBlock = null;
        try {
            Thread.currentThread().setName(MyBucketCache2.GET_BLOCK_THREAD_NAME);
            gotHFileBlock = (HFileBlock) (myBucketCache2.getBlock(blockCacheKey, false, false, false));
            assertTrue(gotHFileBlock.equals(hfileBlock));
            assertTrue(gotHFileBlock.getByteBuffAllocator() == byteBuffAllocator);
            assertEquals(2, gotHFileBlock.refCnt());
            try {
                /**
                 * Release the second cyclicBarrier.await in {@link MyBucketCache2#putIntoBackingMap} for
                 * {@link BucketCache.WriterThread},getBlock completed,{@link BucketCache.WriterThread}
                 * could continue.
                 */
                myBucketCache2.putCyclicBarrier.await();
            } catch (Throwable e) {
                throw new RuntimeException(e);
            }
        } finally {
            Thread.currentThread().setName(oldThreadName);
        }
        cacheBlockThread.join();
        evictBlockThread.join();
        assertTrue(cacheBlockThreadExceptionRef.get() == null);
        assertTrue(evictBlockThreadExceptionRef.get() == null);
        assertTrue(gotHFileBlock.equals(hfileBlock));
        assertEquals(1, gotHFileBlock.refCnt());
        assertTrue(myBucketCache2.overwiteByteBuff == null);
        assertTrue(myBucketCache2.freeBucketEntryCounter.get() == 0);
        gotHFileBlock.release();
        assertEquals(0, gotHFileBlock.refCnt());
        assertTrue(myBucketCache2.overwiteByteBuff != null);
        assertTrue(myBucketCache2.freeBucketEntryCounter.get() == 1);
        assertTrue(myBucketCache2.blockEvictCounter.get() == 1);
    } finally {
        myBucketCache2.shutdown();
    }
}

Example 5 with ByteBuffAllocator

use of org.apache.hadoop.hbase.io.ByteBuffAllocator in project hbase by apache.

the class TestHFileBlock method createOffHeapAlloc.

private ByteBuffAllocator createOffHeapAlloc() {
    Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
    conf.setInt(ByteBuffAllocator.MAX_BUFFER_COUNT_KEY, MAX_BUFFER_COUNT);
    conf.setInt(ByteBuffAllocator.MIN_ALLOCATE_SIZE_KEY, 0);
    ByteBuffAllocator alloc = ByteBuffAllocator.create(conf, true);
    // Fill the allocator
    List<ByteBuff> bufs = new ArrayList<>();
    for (int i = 0; i < MAX_BUFFER_COUNT; i++) {
        ByteBuff bb = alloc.allocateOneBuffer();
        assertTrue(!bb.hasArray());
        bufs.add(bb);
    }
    bufs.forEach(ByteBuff::release);
    return alloc;
}