Examples with ByteBuff org.apache.hadoop.hbase.nio.ByteBuff used on opensource projects

Example 11 with ByteBuff

use of org.apache.hadoop.hbase.nio.ByteBuff in project hbase by apache.

the class TestHFileBlockIndex method testHFileWriterAndReader.

/**
   * Testing block index through the HFile writer/reader APIs. Allows to test
   * setting index block size through configuration, intermediate-level index
   * blocks, and caching index blocks on write.
   *
   * @throws IOException
   */
@Test
public void testHFileWriterAndReader() throws IOException {
    Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), "hfile_for_block_index");
    CacheConfig cacheConf = new CacheConfig(conf);
    BlockCache blockCache = cacheConf.getBlockCache();
    for (int testI = 0; testI < INDEX_CHUNK_SIZES.length; ++testI) {
        int indexBlockSize = INDEX_CHUNK_SIZES[testI];
        int expectedNumLevels = EXPECTED_NUM_LEVELS[testI];
        LOG.info("Index block size: " + indexBlockSize + ", compression: " + compr);
        // Evict all blocks that were cached-on-write by the previous invocation.
        blockCache.evictBlocksByHfileName(hfilePath.getName());
        conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, indexBlockSize);
        Set<String> keyStrSet = new HashSet<>();
        byte[][] keys = new byte[NUM_KV][];
        byte[][] values = new byte[NUM_KV][];
        // Write the HFile
        {
            HFileContext meta = new HFileContextBuilder().withBlockSize(SMALL_BLOCK_SIZE).withCompression(compr).build();
            HFile.Writer writer = HFile.getWriterFactory(conf, cacheConf).withPath(fs, hfilePath).withFileContext(meta).create();
            Random rand = new Random(19231737);
            byte[] family = Bytes.toBytes("f");
            byte[] qualifier = Bytes.toBytes("q");
            for (int i = 0; i < NUM_KV; ++i) {
                byte[] row = RandomKeyValueUtil.randomOrderedKey(rand, i);
                // Key will be interpreted by KeyValue.KEY_COMPARATOR
                KeyValue kv = new KeyValue(row, family, qualifier, EnvironmentEdgeManager.currentTime(), RandomKeyValueUtil.randomValue(rand));
                byte[] k = kv.getKey();
                writer.append(kv);
                keys[i] = k;
                values[i] = CellUtil.cloneValue(kv);
                keyStrSet.add(Bytes.toStringBinary(k));
                if (i > 0) {
                    assertTrue((CellComparator.COMPARATOR.compare(kv, keys[i - 1], 0, keys[i - 1].length)) > 0);
                }
            }
            writer.close();
        }
        // Read the HFile
        HFile.Reader reader = HFile.createReader(fs, hfilePath, cacheConf, conf);
        assertEquals(expectedNumLevels, reader.getTrailer().getNumDataIndexLevels());
        assertTrue(Bytes.equals(keys[0], ((KeyValue) reader.getFirstKey()).getKey()));
        assertTrue(Bytes.equals(keys[NUM_KV - 1], ((KeyValue) reader.getLastKey()).getKey()));
        LOG.info("Last key: " + Bytes.toStringBinary(keys[NUM_KV - 1]));
        for (boolean pread : new boolean[] { false, true }) {
            HFileScanner scanner = reader.getScanner(true, pread);
            for (int i = 0; i < NUM_KV; ++i) {
                checkSeekTo(keys, scanner, i);
                checkKeyValue("i=" + i, keys[i], values[i], ByteBuffer.wrap(((KeyValue) scanner.getKey()).getKey()), scanner.getValue());
            }
            assertTrue(scanner.seekTo());
            for (int i = NUM_KV - 1; i >= 0; --i) {
                checkSeekTo(keys, scanner, i);
                checkKeyValue("i=" + i, keys[i], values[i], ByteBuffer.wrap(((KeyValue) scanner.getKey()).getKey()), scanner.getValue());
            }
        }
        // Manually compute the mid-key and validate it.
        HFile.Reader reader2 = reader;
        HFileBlock.FSReader fsReader = reader2.getUncachedBlockReader();
        HFileBlock.BlockIterator iter = fsReader.blockRange(0, reader.getTrailer().getLoadOnOpenDataOffset());
        HFileBlock block;
        List<byte[]> blockKeys = new ArrayList<>();
        while ((block = iter.nextBlock()) != null) {
            if (block.getBlockType() != BlockType.LEAF_INDEX)
                return;
            ByteBuff b = block.getBufferReadOnly();
            int n = b.getIntAfterPosition(0);
            // One int for the number of items, and n + 1 for the secondary index.
            int entriesOffset = Bytes.SIZEOF_INT * (n + 2);
            // Get all the keys from the leaf index block. S
            for (int i = 0; i < n; ++i) {
                int keyRelOffset = b.getIntAfterPosition(Bytes.SIZEOF_INT * (i + 1));
                int nextKeyRelOffset = b.getIntAfterPosition(Bytes.SIZEOF_INT * (i + 2));
                int keyLen = nextKeyRelOffset - keyRelOffset;
                int keyOffset = b.arrayOffset() + entriesOffset + keyRelOffset + HFileBlockIndex.SECONDARY_INDEX_ENTRY_OVERHEAD;
                byte[] blockKey = Arrays.copyOfRange(b.array(), keyOffset, keyOffset + keyLen);
                String blockKeyStr = Bytes.toString(blockKey);
                blockKeys.add(blockKey);
                // If the first key of the block is not among the keys written, we
                // are not parsing the non-root index block format correctly.
                assertTrue("Invalid block key from leaf-level block: " + blockKeyStr, keyStrSet.contains(blockKeyStr));
            }
        }
        // Validate the mid-key.
        assertEquals(Bytes.toStringBinary(blockKeys.get((blockKeys.size() - 1) / 2)), reader.midkey());
        assertEquals(UNCOMPRESSED_INDEX_SIZES[testI], reader.getTrailer().getUncompressedDataIndexSize());
        reader.close();
        reader2.close();
    }
}

Example 12 with ByteBuff

use of org.apache.hadoop.hbase.nio.ByteBuff in project hbase by apache.

the class TestByteBufferArray method testAsSubBufferWhenEndOffsetLandInLastBuffer.

@Test
public void testAsSubBufferWhenEndOffsetLandInLastBuffer() throws Exception {
    int capacity = 4 * 1024 * 1024;
    ByteBufferAllocator allocator = new ByteBufferAllocator() {

        @Override
        public ByteBuffer allocate(long size, boolean directByteBuffer) throws IOException {
            if (directByteBuffer) {
                return ByteBuffer.allocateDirect((int) size);
            } else {
                return ByteBuffer.allocate((int) size);
            }
        }
    };
    ByteBufferArray array = new ByteBufferArray(capacity, false, allocator);
    ByteBuff subBuf = array.asSubByteBuff(0, capacity);
    // Position to the last byte
    subBuf.position(capacity - 1);
    assertTrue(subBuf.hasRemaining());
    // Read last byte
    subBuf.get();
    assertFalse(subBuf.hasRemaining());
}

Example 13 with ByteBuff

use of org.apache.hadoop.hbase.nio.ByteBuff in project hbase by apache.

the class StoreFileReader method checkGeneralBloomFilter.

private boolean checkGeneralBloomFilter(byte[] key, Cell kvKey, BloomFilter bloomFilter) {
    // Empty file
    if (reader.getTrailer().getEntryCount() == 0) {
        return false;
    }
    HFileBlock bloomBlock = null;
    try {
        boolean shouldCheckBloom;
        ByteBuff bloom;
        if (bloomFilter.supportsAutoLoading()) {
            bloom = null;
            shouldCheckBloom = true;
        } else {
            bloomBlock = reader.getMetaBlock(HFile.BLOOM_FILTER_DATA_KEY, true);
            bloom = bloomBlock.getBufferWithoutHeader();
            shouldCheckBloom = bloom != null;
        }
        if (shouldCheckBloom) {
            boolean exists;
            // Whether the primary Bloom key is greater than the last Bloom key
            // from the file info. For row-column Bloom filters this is not yet
            // a sufficient condition to return false.
            boolean keyIsAfterLast = (lastBloomKey != null);
            // of the hbase:meta cells.  We can safely use Bytes.BYTES_RAWCOMPARATOR for ROW Bloom
            if (keyIsAfterLast) {
                if (bloomFilterType == BloomType.ROW) {
                    keyIsAfterLast = (Bytes.BYTES_RAWCOMPARATOR.compare(key, lastBloomKey) > 0);
                } else {
                    keyIsAfterLast = (CellComparator.COMPARATOR.compare(kvKey, lastBloomKeyOnlyKV)) > 0;
                }
            }
            if (bloomFilterType == BloomType.ROWCOL) {
                // Since a Row Delete is essentially a DeleteFamily applied to all
                // columns, a file might be skipped if using row+col Bloom filter.
                // In order to ensure this file is included an additional check is
                // required looking only for a row bloom.
                Cell rowBloomKey = CellUtil.createFirstOnRow(kvKey);
                // of the hbase:meta cells.  We can safely use Bytes.BYTES_RAWCOMPARATOR for ROW Bloom
                if (keyIsAfterLast && (CellComparator.COMPARATOR.compare(rowBloomKey, lastBloomKeyOnlyKV)) > 0) {
                    exists = false;
                } else {
                    exists = bloomFilter.contains(kvKey, bloom, BloomType.ROWCOL) || bloomFilter.contains(rowBloomKey, bloom, BloomType.ROWCOL);
                }
            } else {
                exists = !keyIsAfterLast && bloomFilter.contains(key, 0, key.length, bloom);
            }
            return exists;
        }
    } catch (IOException e) {
        LOG.error("Error reading bloom filter data -- proceeding without", e);
        setGeneralBloomFilterFaulty();
    } catch (IllegalArgumentException e) {
        LOG.error("Bad bloom filter data -- proceeding without", e);
        setGeneralBloomFilterFaulty();
    } finally {
        // Return the bloom block so that its ref count can be decremented.
        reader.returnBlock(bloomBlock);
    }
    return true;
}

Example 14 with ByteBuff

use of org.apache.hadoop.hbase.nio.ByteBuff in project hbase by apache.

the class RowIndexSeekerV1 method setCurrentBuffer.

@Override
public void setCurrentBuffer(ByteBuff buffer) {
    int onDiskSize = buffer.getInt(buffer.limit() - Bytes.SIZEOF_INT);
    // Data part
    ByteBuff dup = buffer.duplicate();
    dup.position(buffer.position());
    dup.limit(buffer.position() + onDiskSize);
    currentBuffer = dup.slice();
    current.currentBuffer = currentBuffer;
    buffer.skip(onDiskSize);
    // Row offset
    rowNumber = buffer.getInt();
    int totalRowOffsetsLength = Bytes.SIZEOF_INT * rowNumber;
    ByteBuff rowDup = buffer.duplicate();
    rowDup.position(buffer.position());
    rowDup.limit(buffer.position() + totalRowOffsetsLength);
    rowOffsets = rowDup.slice();
    decodeFirst();
}

Example 15 with ByteBuff

use of org.apache.hadoop.hbase.nio.ByteBuff in project hbase by apache.

the class RpcServer method allocateByteBuffToReadInto.

/**
   * This is extracted to a static method for better unit testing. We try to get buffer(s) from pool
   * as much as possible.
   *
   * @param pool The ByteBufferPool to use
   * @param minSizeForPoolUse Only for buffer size above this, we will try to use pool. Any buffer
   *           need of size below this, create on heap ByteBuffer.
   * @param reqLen Bytes count in request
   */
@VisibleForTesting
static Pair<ByteBuff, CallCleanup> allocateByteBuffToReadInto(ByteBufferPool pool, int minSizeForPoolUse, int reqLen) {
    ByteBuff resultBuf;
    List<ByteBuffer> bbs = new ArrayList<>((reqLen / pool.getBufferSize()) + 1);
    int remain = reqLen;
    ByteBuffer buf = null;
    while (remain >= minSizeForPoolUse && (buf = pool.getBuffer()) != null) {
        bbs.add(buf);
        remain -= pool.getBufferSize();
    }
    ByteBuffer[] bufsFromPool = null;
    if (bbs.size() > 0) {
        bufsFromPool = new ByteBuffer[bbs.size()];
        bbs.toArray(bufsFromPool);
    }
    if (remain > 0) {
        bbs.add(ByteBuffer.allocate(remain));
    }
    if (bbs.size() > 1) {
        ByteBuffer[] items = new ByteBuffer[bbs.size()];
        bbs.toArray(items);
        resultBuf = new MultiByteBuff(items);
    } else {
        // We are backed by single BB
        resultBuf = new SingleByteBuff(bbs.get(0));
    }
    resultBuf.limit(reqLen);
    if (bufsFromPool != null) {
        final ByteBuffer[] bufsFromPoolFinal = bufsFromPool;
        return new Pair<>(resultBuf, () -> {
            // Return back all the BBs to pool
            for (int i = 0; i < bufsFromPoolFinal.length; i++) {
                pool.putbackBuffer(bufsFromPoolFinal[i]);
            }
        });
    }
    return new Pair<>(resultBuf, null);
}