Examples with HFileBlock org.apache.hadoop.hbase.io.hfile.HFileBlock used on opensource projects

Example 1 with HFileBlock

use of org.apache.hadoop.hbase.io.hfile.HFileBlock in project hbase by apache.

the class StoreFileReader method checkGeneralBloomFilter.

private boolean checkGeneralBloomFilter(byte[] key, Cell kvKey, BloomFilter bloomFilter) {
    // Empty file
    if (reader.getTrailer().getEntryCount() == 0) {
        return false;
    }
    HFileBlock bloomBlock = null;
    try {
        boolean shouldCheckBloom;
        ByteBuff bloom;
        if (bloomFilter.supportsAutoLoading()) {
            bloom = null;
            shouldCheckBloom = true;
        } else {
            bloomBlock = reader.getMetaBlock(HFile.BLOOM_FILTER_DATA_KEY, true);
            bloom = bloomBlock.getBufferWithoutHeader();
            shouldCheckBloom = bloom != null;
        }
        if (shouldCheckBloom) {
            boolean exists;
            // Whether the primary Bloom key is greater than the last Bloom key
            // from the file info. For row-column Bloom filters this is not yet
            // a sufficient condition to return false.
            boolean keyIsAfterLast = (lastBloomKey != null);
            // of the hbase:meta cells.  We can safely use Bytes.BYTES_RAWCOMPARATOR for ROW Bloom
            if (keyIsAfterLast) {
                if (bloomFilterType == BloomType.ROW) {
                    keyIsAfterLast = (Bytes.BYTES_RAWCOMPARATOR.compare(key, lastBloomKey) > 0);
                } else {
                    keyIsAfterLast = (CellComparator.COMPARATOR.compare(kvKey, lastBloomKeyOnlyKV)) > 0;
                }
            }
            if (bloomFilterType == BloomType.ROWCOL) {
                // Since a Row Delete is essentially a DeleteFamily applied to all
                // columns, a file might be skipped if using row+col Bloom filter.
                // In order to ensure this file is included an additional check is
                // required looking only for a row bloom.
                Cell rowBloomKey = CellUtil.createFirstOnRow(kvKey);
                // of the hbase:meta cells.  We can safely use Bytes.BYTES_RAWCOMPARATOR for ROW Bloom
                if (keyIsAfterLast && (CellComparator.COMPARATOR.compare(rowBloomKey, lastBloomKeyOnlyKV)) > 0) {
                    exists = false;
                } else {
                    exists = bloomFilter.contains(kvKey, bloom, BloomType.ROWCOL) || bloomFilter.contains(rowBloomKey, bloom, BloomType.ROWCOL);
                }
            } else {
                exists = !keyIsAfterLast && bloomFilter.contains(key, 0, key.length, bloom);
            }
            return exists;
        }
    } catch (IOException e) {
        LOG.error("Error reading bloom filter data -- proceeding without", e);
        setGeneralBloomFilterFaulty();
    } catch (IllegalArgumentException e) {
        LOG.error("Bad bloom filter data -- proceeding without", e);
        setGeneralBloomFilterFaulty();
    } finally {
        // Return the bloom block so that its ref count can be decremented.
        reader.returnBlock(bloomBlock);
    }
    return true;
}

Example 2 with HFileBlock

use of org.apache.hadoop.hbase.io.hfile.HFileBlock in project hbase by apache.

the class TestCacheOnWriteInSchema method readStoreFile.

private void readStoreFile(Path path) throws IOException {
    CacheConfig cacheConf = store.getCacheConfig();
    BlockCache cache = cacheConf.getBlockCache();
    StoreFile sf = new StoreFile(fs, path, conf, cacheConf, BloomType.ROWCOL);
    HFile.Reader reader = sf.createReader().getHFileReader();
    try {
        // Open a scanner with (on read) caching disabled
        HFileScanner scanner = reader.getScanner(false, false);
        assertTrue(testDescription, scanner.seekTo());
        // Cribbed from io.hfile.TestCacheOnWrite
        long offset = 0;
        while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
            // Flags: don't cache the block, use pread, this is not a compaction.
            // Also, pass null for expected block type to avoid checking it.
            HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, DataBlockEncoding.NONE);
            BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(), offset);
            boolean isCached = cache.getBlock(blockCacheKey, true, false, true) != null;
            boolean shouldBeCached = cowType.shouldBeCached(block.getBlockType());
            if (shouldBeCached != isCached) {
                throw new AssertionError("shouldBeCached: " + shouldBeCached + "\n" + "isCached: " + isCached + "\n" + "Test description: " + testDescription + "\n" + "block: " + block + "\n" + "blockCacheKey: " + blockCacheKey);
            }
            offset += block.getOnDiskSizeWithHeader();
        }
    } finally {
        reader.close();
    }
}