Examples with BucketCache org.apache.hadoop.hbase.io.hfile.bucket.BucketCache used on opensource projects

Example 11 with BucketCache

use of org.apache.hadoop.hbase.io.hfile.bucket.BucketCache in project hbase by apache.

the class TestBlockEvictionFromClient method testScanWithMultipleColumnFamilies.

@Test
public void testScanWithMultipleColumnFamilies() throws IOException, InterruptedException {
    Table table = null;
    try {
        latch = new CountDownLatch(1);
        // Check if get() returns blocks on its close() itself
        final TableName tableName = TableName.valueOf(name.getMethodName());
        // Create KV that will give you two blocks
        // Create a table with block size as 1024
        byte[][] fams = new byte[10][];
        fams[0] = FAMILY;
        for (int i = 1; i < 10; i++) {
            fams[i] = (Bytes.toBytes("testFamily" + i));
        }
        table = TEST_UTIL.createTable(tableName, fams, 1, 1024, CustomInnerRegionObserver.class.getName());
        // get the block cache and region
        RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
        String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
        Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getFromOnlineRegions(regionName);
        BlockCache cache = setCacheProperties(region);
        Put put = new Put(ROW);
        put.addColumn(FAMILY, QUALIFIER, data);
        table.put(put);
        region.flush(true);
        put = new Put(ROW1);
        put.addColumn(FAMILY, QUALIFIER, data);
        table.put(put);
        region.flush(true);
        for (int i = 1; i < 10; i++) {
            put = new Put(ROW);
            put.addColumn(Bytes.toBytes("testFamily" + i), Bytes.toBytes("testQualifier" + i), data2);
            table.put(put);
            if (i % 2 == 0) {
                region.flush(true);
            }
        }
        region.flush(true);
        byte[] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER);
        put = new Put(ROW);
        put.addColumn(FAMILY, QUALIFIER2, data2);
        table.put(put);
        region.flush(true);
        // flush the data
        System.out.println("Flushing cache");
        // Should create one Hfile with 2 blocks
        // Create three sets of gets
        ScanThread[] scanThreads = initiateScan(table, true);
        Thread.sleep(200);
        Iterator<CachedBlock> iterator = cache.iterator();
        boolean usedBlocksFound = false;
        int refCount = 0;
        int noOfBlocksWithRef = 0;
        while (iterator.hasNext()) {
            CachedBlock next = iterator.next();
            BlockCacheKey cacheKey = new BlockCacheKey(next.getFilename(), next.getOffset());
            if (cache instanceof BucketCache) {
                refCount = ((BucketCache) cache).getRefCount(cacheKey);
            } else if (cache instanceof CombinedBlockCache) {
                refCount = ((CombinedBlockCache) cache).getRefCount(cacheKey);
            } else {
                continue;
            }
            if (refCount != 0) {
                // Blocks will be with count 3
                System.out.println("The refCount is " + refCount);
                assertEquals(NO_OF_THREADS, refCount);
                usedBlocksFound = true;
                noOfBlocksWithRef++;
            }
        }
        assertTrue(usedBlocksFound);
        // the number of blocks referred
        assertEquals(12, noOfBlocksWithRef);
        CustomInnerRegionObserver.getCdl().get().countDown();
        for (ScanThread thread : scanThreads) {
            thread.join();
        }
        // giving some time for the block to be decremented
        checkForBlockEviction(cache, true, false);
    } finally {
        if (table != null) {
            table.close();
        }
    }
}

Example 12 with BucketCache

use of org.apache.hadoop.hbase.io.hfile.bucket.BucketCache in project hbase by apache.

the class TestCacheConfig method doBucketCacheConfigTest.

private void doBucketCacheConfigTest() {
    final int bcSize = 100;
    this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, bcSize);
    CacheConfig cc = new CacheConfig(this.conf);
    basicBlockCacheOps(cc, false, false);
    assertTrue(cc.getBlockCache() instanceof CombinedBlockCache);
    // TODO: Assert sizes allocated are right and proportions.
    CombinedBlockCache cbc = (CombinedBlockCache) cc.getBlockCache();
    BlockCache[] bcs = cbc.getBlockCaches();
    assertTrue(bcs[0] instanceof LruBlockCache);
    LruBlockCache lbc = (LruBlockCache) bcs[0];
    assertEquals(MemorySizeUtil.getLruCacheSize(this.conf), lbc.getMaxSize());
    assertTrue(bcs[1] instanceof BucketCache);
    BucketCache bc = (BucketCache) bcs[1];
    // getMaxSize comes back in bytes but we specified size in MB
    assertEquals(bcSize, bc.getMaxSize() / (1024 * 1024));
}