Examples with CacheConfig org.apache.hadoop.hbase.io.hfile.CacheConfig used on opensource projects

Example 36 with CacheConfig

use of org.apache.hadoop.hbase.io.hfile.CacheConfig in project hbase by apache.

the class TestBlockEvictionFromClient method testScanWithException.

@Test
public void testScanWithException() throws IOException, InterruptedException {
    Table table = null;
    try {
        latch = new CountDownLatch(1);
        exceptionLatch = new CountDownLatch(1);
        final TableName tableName = TableName.valueOf(name.getMethodName());
        // Create KV that will give you two blocks
        // Create a table with block size as 1024
        table = TEST_UTIL.createTable(tableName, FAMILIES_1, 1, 1024, CustomInnerRegionObserverWrapper.class.getName());
        // get the block cache and region
        RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
        String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
        Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getFromOnlineRegions(regionName);
        Store store = region.getStores().iterator().next();
        CacheConfig cacheConf = store.getCacheConfig();
        cacheConf.setCacheDataOnWrite(true);
        cacheConf.setEvictOnClose(true);
        BlockCache cache = cacheConf.getBlockCache();
        // insert data. 2 Rows are added
        insertData(table);
        // flush the data
        System.out.println("Flushing cache");
        // Should create one Hfile with 2 blocks
        region.flush(true);
        // CustomInnerRegionObserver.sleepTime.set(5000);
        CustomInnerRegionObserver.throwException.set(true);
        ScanThread[] scanThreads = initiateScan(table, false);
        // The block would have been decremented for the scan case as it was
        // wrapped
        // before even the postNext hook gets executed.
        // giving some time for the block to be decremented
        Thread.sleep(100);
        Iterator<CachedBlock> iterator = cache.iterator();
        boolean usedBlocksFound = false;
        int refCount = 0;
        while (iterator.hasNext()) {
            CachedBlock next = iterator.next();
            BlockCacheKey cacheKey = new BlockCacheKey(next.getFilename(), next.getOffset());
            if (cache instanceof BucketCache) {
                refCount = ((BucketCache) cache).getRefCount(cacheKey);
            } else if (cache instanceof CombinedBlockCache) {
                refCount = ((CombinedBlockCache) cache).getRefCount(cacheKey);
            } else {
                continue;
            }
            if (refCount != 0) {
                // Blocks will be with count 3
                assertEquals(NO_OF_THREADS, refCount);
                usedBlocksFound = true;
            }
        }
        assertTrue(usedBlocksFound);
        exceptionLatch.countDown();
        // countdown the latch
        CustomInnerRegionObserver.getCdl().get().countDown();
        for (ScanThread thread : scanThreads) {
            thread.join();
        }
        iterator = cache.iterator();
        usedBlocksFound = false;
        refCount = 0;
        while (iterator.hasNext()) {
            CachedBlock next = iterator.next();
            BlockCacheKey cacheKey = new BlockCacheKey(next.getFilename(), next.getOffset());
            if (cache instanceof BucketCache) {
                refCount = ((BucketCache) cache).getRefCount(cacheKey);
            } else if (cache instanceof CombinedBlockCache) {
                refCount = ((CombinedBlockCache) cache).getRefCount(cacheKey);
            } else {
                continue;
            }
            if (refCount != 0) {
                // Blocks will be with count 3
                assertEquals(NO_OF_THREADS, refCount);
                usedBlocksFound = true;
            }
        }
        assertFalse(usedBlocksFound);
        // you should always see 0 ref count. since after HBASE-16604 we always recreate the scanner
        assertEquals(0, refCount);
    } finally {
        if (table != null) {
            table.close();
        }
    }
}

Example 37 with CacheConfig

use of org.apache.hadoop.hbase.io.hfile.CacheConfig in project hbase by apache.

the class HFileTestUtil method createHFile.

/**
   * Create an HFile with the given number of rows between a given
   * start key and end key @ family:qualifier.
   * If withTag is true, we add the rowKey as the tag value for
   * tagtype MOB_TABLE_NAME_TAG_TYPE
   */
public static void createHFile(Configuration configuration, FileSystem fs, Path path, DataBlockEncoding encoding, byte[] family, byte[] qualifier, byte[] startKey, byte[] endKey, int numRows, boolean withTag) throws IOException {
    HFileContext meta = new HFileContextBuilder().withIncludesTags(withTag).withDataBlockEncoding(encoding).build();
    HFile.Writer writer = HFile.getWriterFactory(configuration, new CacheConfig(configuration)).withPath(fs, path).withFileContext(meta).create();
    long now = System.currentTimeMillis();
    try {
        // subtract 2 since iterateOnSplits doesn't include boundary keys
        for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, numRows - 2)) {
            Cell kv = new KeyValue(key, family, qualifier, now, key);
            if (withTag) {
                // add a tag.  Arbitrarily chose mob tag since we have a helper already.
                Tag tableNameTag = new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE, key);
                kv = MobUtils.createMobRefCell(kv, key, tableNameTag);
                // verify that the kv has the tag.
                Tag t = CellUtil.getTag(kv, TagType.MOB_TABLE_NAME_TAG_TYPE);
                if (t == null) {
                    throw new IllegalStateException("Tag didn't stick to KV " + kv.toString());
                }
            }
            writer.append(kv);
        }
    } finally {
        writer.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis()));
        writer.close();
    }
}

Example 38 with CacheConfig

use of org.apache.hadoop.hbase.io.hfile.CacheConfig in project hbase by apache.

the class LoadIncrementalHFiles method createTable.

/*
   * If the table is created for the first time, then "completebulkload" reads the files twice.
   * More modifications necessary if we want to avoid doing it.
   */
private void createTable(TableName tableName, String dirPath, Admin admin) throws Exception {
    final Path hfofDir = new Path(dirPath);
    final FileSystem fs = hfofDir.getFileSystem(getConf());
    // Add column families
    // Build a set of keys
    final HTableDescriptor htd = new HTableDescriptor(tableName);
    final TreeMap<byte[], Integer> map = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    visitBulkHFiles(fs, hfofDir, new BulkHFileVisitor<HColumnDescriptor>() {

        @Override
        public HColumnDescriptor bulkFamily(final byte[] familyName) {
            HColumnDescriptor hcd = new HColumnDescriptor(familyName);
            htd.addFamily(hcd);
            return hcd;
        }

        @Override
        public void bulkHFile(final HColumnDescriptor hcd, final FileStatus hfileStatus) throws IOException {
            Path hfile = hfileStatus.getPath();
            HFile.Reader reader = HFile.createReader(fs, hfile, new CacheConfig(getConf()), getConf());
            try {
                if (hcd.getCompressionType() != reader.getFileContext().getCompression()) {
                    hcd.setCompressionType(reader.getFileContext().getCompression());
                    LOG.info("Setting compression " + hcd.getCompressionType().name() + " for family " + hcd.toString());
                }
                reader.loadFileInfo();
                byte[] first = reader.getFirstRowKey();
                byte[] last = reader.getLastRowKey();
                LOG.info("Trying to figure out region boundaries hfile=" + hfile + " first=" + Bytes.toStringBinary(first) + " last=" + Bytes.toStringBinary(last));
                // To eventually infer start key-end key boundaries
                Integer value = map.containsKey(first) ? map.get(first) : 0;
                map.put(first, value + 1);
                value = map.containsKey(last) ? map.get(last) : 0;
                map.put(last, value - 1);
            } finally {
                reader.close();
            }
        }
    });
    byte[][] keys = LoadIncrementalHFiles.inferBoundaries(map);
    admin.createTable(htd, keys);
    LOG.info("Table " + tableName + " is available!!");
}

Example 39 with CacheConfig

use of org.apache.hadoop.hbase.io.hfile.CacheConfig in project hbase by apache.

the class HStore method createWriterInTmp.

/*
   * @param maxKeyCount
   * @param compression Compression algorithm to use
   * @param isCompaction whether we are creating a new file in a compaction
   * @param includesMVCCReadPoint - whether to include MVCC or not
   * @param includesTag - includesTag or not
   * @return Writer for a new StoreFile in the tmp dir.
   */
// TODO : allow the Writer factory to create Writers of ShipperListener type only in case of
// compaction
@Override
public StoreFileWriter createWriterInTmp(long maxKeyCount, Compression.Algorithm compression, boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag, boolean shouldDropBehind, final TimeRangeTracker trt) throws IOException {
    final CacheConfig writerCacheConf;
    if (isCompaction) {
        // Don't cache data on write on compactions.
        writerCacheConf = new CacheConfig(cacheConf);
        writerCacheConf.setCacheDataOnWrite(false);
    } else {
        writerCacheConf = cacheConf;
    }
    InetSocketAddress[] favoredNodes = null;
    if (region.getRegionServerServices() != null) {
        favoredNodes = region.getRegionServerServices().getFavoredNodesForRegion(region.getRegionInfo().getEncodedName());
    }
    HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag, cryptoContext);
    Path familyTempDir = new Path(fs.getTempDir(), family.getNameAsString());
    StoreFileWriter.Builder builder = new StoreFileWriter.Builder(conf, writerCacheConf, this.getFileSystem()).withOutputDir(familyTempDir).withComparator(comparator).withBloomType(family.getBloomFilterType()).withMaxKeyCount(maxKeyCount).withFavoredNodes(favoredNodes).withFileContext(hFileContext).withShouldDropCacheBehind(shouldDropBehind);
    if (trt != null) {
        builder.withTimeRangeTracker(trt);
    }
    return builder.build();
}

Example 40 with CacheConfig

use of org.apache.hadoop.hbase.io.hfile.CacheConfig in project hbase by apache.

the class TestBlockEvictionFromClient method testBlockEvictionAfterHBASE13082WithCompactionAndFlush.

@Test
public void testBlockEvictionAfterHBASE13082WithCompactionAndFlush() throws IOException, InterruptedException {
    // do flush and scan in parallel
    Table table = null;
    try {
        latch = new CountDownLatch(1);
        compactionLatch = new CountDownLatch(1);
        final TableName tableName = TableName.valueOf(name.getMethodName());
        // Create a table with block size as 1024
        table = TEST_UTIL.createTable(tableName, FAMILIES_1, 1, 1024, CustomInnerRegionObserverWrapper.class.getName());
        // get the block cache and region
        RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
        String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
        Region region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getFromOnlineRegions(regionName);
        Store store = region.getStores().iterator().next();
        CacheConfig cacheConf = store.getCacheConfig();
        cacheConf.setCacheDataOnWrite(true);
        cacheConf.setEvictOnClose(true);
        BlockCache cache = cacheConf.getBlockCache();
        // insert data. 2 Rows are added
        Put put = new Put(ROW);
        put.addColumn(FAMILY, QUALIFIER, data);
        table.put(put);
        put = new Put(ROW1);
        put.addColumn(FAMILY, QUALIFIER, data);
        table.put(put);
        assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
        // Should create one Hfile with 2 blocks
        region.flush(true);
        // read the data and expect same blocks, one new hit, no misses
        int refCount = 0;
        // Check how this miss is happening
        // insert a second column, read the row, no new blocks, 3 new hits
        byte[] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER);
        byte[] data2 = Bytes.add(data, data);
        put = new Put(ROW);
        put.addColumn(FAMILY, QUALIFIER2, data2);
        table.put(put);
        // flush, one new block
        System.out.println("Flushing cache");
        region.flush(true);
        Iterator<CachedBlock> iterator = cache.iterator();
        iterateBlockCache(cache, iterator);
        // Create three sets of scan
        ScanThread[] scanThreads = initiateScan(table, false);
        Thread.sleep(100);
        iterator = cache.iterator();
        boolean usedBlocksFound = false;
        while (iterator.hasNext()) {
            CachedBlock next = iterator.next();
            BlockCacheKey cacheKey = new BlockCacheKey(next.getFilename(), next.getOffset());
            if (cache instanceof BucketCache) {
                refCount = ((BucketCache) cache).getRefCount(cacheKey);
            } else if (cache instanceof CombinedBlockCache) {
                refCount = ((CombinedBlockCache) cache).getRefCount(cacheKey);
            } else {
                continue;
            }
            if (refCount != 0) {
                // Blocks will be with count 3
                assertEquals(NO_OF_THREADS, refCount);
                usedBlocksFound = true;
            }
        }
        // Make a put and do a flush
        QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER);
        data2 = Bytes.add(data, data);
        put = new Put(ROW1);
        put.addColumn(FAMILY, QUALIFIER2, data2);
        table.put(put);
        // flush, one new block
        System.out.println("Flushing cache");
        region.flush(true);
        assertTrue("Blocks with non zero ref count should be found ", usedBlocksFound);
        usedBlocksFound = false;
        System.out.println("Compacting");
        assertEquals(3, store.getStorefilesCount());
        store.triggerMajorCompaction();
        region.compact(true);
        // wait 10 seconds max
        waitForStoreFileCount(store, 1, 10000);
        assertEquals(1, store.getStorefilesCount());
        // Even after compaction is done we will have some blocks that cannot
        // be evicted this is because the scan is still referencing them
        iterator = cache.iterator();
        while (iterator.hasNext()) {
            CachedBlock next = iterator.next();
            BlockCacheKey cacheKey = new BlockCacheKey(next.getFilename(), next.getOffset());
            if (cache instanceof BucketCache) {
                refCount = ((BucketCache) cache).getRefCount(cacheKey);
            } else if (cache instanceof CombinedBlockCache) {
                refCount = ((CombinedBlockCache) cache).getRefCount(cacheKey);
            } else {
                continue;
            }
            if (refCount != 0) {
                // Blocks will be with count 3 as they are not yet cleared
                assertEquals(NO_OF_THREADS, refCount);
                usedBlocksFound = true;
            }
        }
        assertTrue("Blocks with non zero ref count should be found ", usedBlocksFound);
        // Should not throw exception
        compactionLatch.countDown();
        latch.countDown();
        for (ScanThread thread : scanThreads) {
            thread.join();
        }
        // by this time all blocks should have been evicted
        iterator = cache.iterator();
        // Since a flush and compaction happened after a scan started
        // we need to ensure that all the original blocks of the compacted file
        // is also removed.
        iterateBlockCache(cache, iterator);
        Result r = table.get(new Get(ROW));
        assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
        assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
        // The gets would be working on new blocks
        iterator = cache.iterator();
        iterateBlockCache(cache, iterator);
    } finally {
        if (table != null) {
            table.close();
        }
    }
}