Examples with KeyValue org.apache.hadoop.hbase.KeyValue used on opensource projects

Example 46 with KeyValue

use of org.apache.hadoop.hbase.KeyValue in project hbase by apache.

the class TestResultSizeEstimation method testResultSizeEstimation.

@Test
public void testResultSizeEstimation() throws Exception {
    byte[] ROW1 = Bytes.toBytes("testRow1");
    byte[] ROW2 = Bytes.toBytes("testRow2");
    byte[] FAMILY = Bytes.toBytes("testFamily");
    byte[] QUALIFIER = Bytes.toBytes("testQualifier");
    byte[] VALUE = Bytes.toBytes("testValue");
    final TableName tableName = TableName.valueOf(name.getMethodName());
    byte[][] FAMILIES = new byte[][] { FAMILY };
    Table table = TEST_UTIL.createTable(tableName, FAMILIES);
    Put p = new Put(ROW1);
    p.add(new KeyValue(ROW1, FAMILY, QUALIFIER, Long.MAX_VALUE, VALUE));
    table.put(p);
    p = new Put(ROW2);
    p.add(new KeyValue(ROW2, FAMILY, QUALIFIER, Long.MAX_VALUE, VALUE));
    table.put(p);
    Scan s = new Scan();
    s.setMaxResultSize(SCANNER_DATA_LIMIT);
    ResultScanner rs = table.getScanner(s);
    int count = 0;
    while (rs.next() != null) {
        count++;
    }
    assertEquals("Result size estimation did not work properly", 2, count);
    rs.close();
    table.close();
}

Example 47 with KeyValue

use of org.apache.hadoop.hbase.KeyValue in project hbase by apache.

the class TestCacheOnWrite method testNotCachingDataBlocksDuringCompactionInternals.

private void testNotCachingDataBlocksDuringCompactionInternals(boolean useTags) throws IOException, InterruptedException {
    // TODO: need to change this test if we add a cache size threshold for
    // compactions, or if we implement some other kind of intelligent logic for
    // deciding what blocks to cache-on-write on compaction.
    final String table = "CompactionCacheOnWrite";
    final String cf = "myCF";
    final byte[] cfBytes = Bytes.toBytes(cf);
    final int maxVersions = 3;
    Region region = TEST_UTIL.createTestRegion(table, new HColumnDescriptor(cf).setCompressionType(compress).setBloomFilterType(BLOOM_TYPE).setMaxVersions(maxVersions).setDataBlockEncoding(NoOpDataBlockEncoder.INSTANCE.getDataBlockEncoding()));
    int rowIdx = 0;
    long ts = EnvironmentEdgeManager.currentTime();
    for (int iFile = 0; iFile < 5; ++iFile) {
        for (int iRow = 0; iRow < 500; ++iRow) {
            String rowStr = "" + (rowIdx * rowIdx * rowIdx) + "row" + iFile + "_" + iRow;
            Put p = new Put(Bytes.toBytes(rowStr));
            ++rowIdx;
            for (int iCol = 0; iCol < 10; ++iCol) {
                String qualStr = "col" + iCol;
                String valueStr = "value_" + rowStr + "_" + qualStr;
                for (int iTS = 0; iTS < 5; ++iTS) {
                    if (useTags) {
                        Tag t = new ArrayBackedTag((byte) 1, "visibility");
                        Tag[] tags = new Tag[1];
                        tags[0] = t;
                        KeyValue kv = new KeyValue(Bytes.toBytes(rowStr), cfBytes, Bytes.toBytes(qualStr), HConstants.LATEST_TIMESTAMP, Bytes.toBytes(valueStr), tags);
                        p.add(kv);
                    } else {
                        p.addColumn(cfBytes, Bytes.toBytes(qualStr), ts++, Bytes.toBytes(valueStr));
                    }
                }
            }
            p.setDurability(Durability.ASYNC_WAL);
            region.put(p);
        }
        region.flush(true);
    }
    clearBlockCache(blockCache);
    assertEquals(0, blockCache.getBlockCount());
    region.compact(false);
    LOG.debug("compactStores() returned");
    for (CachedBlock block : blockCache) {
        assertNotEquals(BlockType.ENCODED_DATA, block.getBlockType());
        assertNotEquals(BlockType.DATA, block.getBlockType());
    }
    ((HRegion) region).close();
}

Example 48 with KeyValue

use of org.apache.hadoop.hbase.KeyValue in project hbase by apache.

the class TestHFileBlockIndex method testHFileWriterAndReader.

/**
   * Testing block index through the HFile writer/reader APIs. Allows to test
   * setting index block size through configuration, intermediate-level index
   * blocks, and caching index blocks on write.
   *
   * @throws IOException
   */
@Test
public void testHFileWriterAndReader() throws IOException {
    Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), "hfile_for_block_index");
    CacheConfig cacheConf = new CacheConfig(conf);
    BlockCache blockCache = cacheConf.getBlockCache();
    for (int testI = 0; testI < INDEX_CHUNK_SIZES.length; ++testI) {
        int indexBlockSize = INDEX_CHUNK_SIZES[testI];
        int expectedNumLevels = EXPECTED_NUM_LEVELS[testI];
        LOG.info("Index block size: " + indexBlockSize + ", compression: " + compr);
        // Evict all blocks that were cached-on-write by the previous invocation.
        blockCache.evictBlocksByHfileName(hfilePath.getName());
        conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, indexBlockSize);
        Set<String> keyStrSet = new HashSet<>();
        byte[][] keys = new byte[NUM_KV][];
        byte[][] values = new byte[NUM_KV][];
        // Write the HFile
        {
            HFileContext meta = new HFileContextBuilder().withBlockSize(SMALL_BLOCK_SIZE).withCompression(compr).build();
            HFile.Writer writer = HFile.getWriterFactory(conf, cacheConf).withPath(fs, hfilePath).withFileContext(meta).create();
            Random rand = new Random(19231737);
            byte[] family = Bytes.toBytes("f");
            byte[] qualifier = Bytes.toBytes("q");
            for (int i = 0; i < NUM_KV; ++i) {
                byte[] row = RandomKeyValueUtil.randomOrderedKey(rand, i);
                // Key will be interpreted by KeyValue.KEY_COMPARATOR
                KeyValue kv = new KeyValue(row, family, qualifier, EnvironmentEdgeManager.currentTime(), RandomKeyValueUtil.randomValue(rand));
                byte[] k = kv.getKey();
                writer.append(kv);
                keys[i] = k;
                values[i] = CellUtil.cloneValue(kv);
                keyStrSet.add(Bytes.toStringBinary(k));
                if (i > 0) {
                    assertTrue((CellComparator.COMPARATOR.compare(kv, keys[i - 1], 0, keys[i - 1].length)) > 0);
                }
            }
            writer.close();
        }
        // Read the HFile
        HFile.Reader reader = HFile.createReader(fs, hfilePath, cacheConf, conf);
        assertEquals(expectedNumLevels, reader.getTrailer().getNumDataIndexLevels());
        assertTrue(Bytes.equals(keys[0], ((KeyValue) reader.getFirstKey()).getKey()));
        assertTrue(Bytes.equals(keys[NUM_KV - 1], ((KeyValue) reader.getLastKey()).getKey()));
        LOG.info("Last key: " + Bytes.toStringBinary(keys[NUM_KV - 1]));
        for (boolean pread : new boolean[] { false, true }) {
            HFileScanner scanner = reader.getScanner(true, pread);
            for (int i = 0; i < NUM_KV; ++i) {
                checkSeekTo(keys, scanner, i);
                checkKeyValue("i=" + i, keys[i], values[i], ByteBuffer.wrap(((KeyValue) scanner.getKey()).getKey()), scanner.getValue());
            }
            assertTrue(scanner.seekTo());
            for (int i = NUM_KV - 1; i >= 0; --i) {
                checkSeekTo(keys, scanner, i);
                checkKeyValue("i=" + i, keys[i], values[i], ByteBuffer.wrap(((KeyValue) scanner.getKey()).getKey()), scanner.getValue());
            }
        }
        // Manually compute the mid-key and validate it.
        HFile.Reader reader2 = reader;
        HFileBlock.FSReader fsReader = reader2.getUncachedBlockReader();
        HFileBlock.BlockIterator iter = fsReader.blockRange(0, reader.getTrailer().getLoadOnOpenDataOffset());
        HFileBlock block;
        List<byte[]> blockKeys = new ArrayList<>();
        while ((block = iter.nextBlock()) != null) {
            if (block.getBlockType() != BlockType.LEAF_INDEX)
                return;
            ByteBuff b = block.getBufferReadOnly();
            int n = b.getIntAfterPosition(0);
            // One int for the number of items, and n + 1 for the secondary index.
            int entriesOffset = Bytes.SIZEOF_INT * (n + 2);
            // Get all the keys from the leaf index block. S
            for (int i = 0; i < n; ++i) {
                int keyRelOffset = b.getIntAfterPosition(Bytes.SIZEOF_INT * (i + 1));
                int nextKeyRelOffset = b.getIntAfterPosition(Bytes.SIZEOF_INT * (i + 2));
                int keyLen = nextKeyRelOffset - keyRelOffset;
                int keyOffset = b.arrayOffset() + entriesOffset + keyRelOffset + HFileBlockIndex.SECONDARY_INDEX_ENTRY_OVERHEAD;
                byte[] blockKey = Arrays.copyOfRange(b.array(), keyOffset, keyOffset + keyLen);
                String blockKeyStr = Bytes.toString(blockKey);
                blockKeys.add(blockKey);
                // If the first key of the block is not among the keys written, we
                // are not parsing the non-root index block format correctly.
                assertTrue("Invalid block key from leaf-level block: " + blockKeyStr, keyStrSet.contains(blockKeyStr));
            }
        }
        // Validate the mid-key.
        assertEquals(Bytes.toStringBinary(blockKeys.get((blockKeys.size() - 1) / 2)), reader.midkey());
        assertEquals(UNCOMPRESSED_INDEX_SIZES[testI], reader.getTrailer().getUncompressedDataIndexSize());
        reader.close();
        reader2.close();
    }
}

Example 49 with KeyValue

use of org.apache.hadoop.hbase.KeyValue in project hbase by apache.

the class TestHFileBlockIndex method testMidKeyOnLeafIndexBlockBoundary.

/**
  * to check if looks good when midKey on a leaf index block boundary
  * @throws IOException
  */
@Test
public void testMidKeyOnLeafIndexBlockBoundary() throws IOException {
    Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), "hfile_for_midkey");
    int maxChunkSize = 512;
    conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, maxChunkSize);
    // should open hfile.block.index.cacheonwrite
    conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, true);
    CacheConfig cacheConf = new CacheConfig(conf);
    BlockCache blockCache = cacheConf.getBlockCache();
    // Evict all blocks that were cached-on-write by the previous invocation.
    blockCache.evictBlocksByHfileName(hfilePath.getName());
    // Write the HFile
    {
        HFileContext meta = new HFileContextBuilder().withBlockSize(SMALL_BLOCK_SIZE).withCompression(Algorithm.NONE).withDataBlockEncoding(DataBlockEncoding.NONE).build();
        HFile.Writer writer = HFile.getWriterFactory(conf, cacheConf).withPath(fs, hfilePath).withFileContext(meta).create();
        Random rand = new Random(19231737);
        byte[] family = Bytes.toBytes("f");
        byte[] qualifier = Bytes.toBytes("q");
        int kvNumberToBeWritten = 16;
        // midkey is just on the boundary of the first leaf-index block
        for (int i = 0; i < kvNumberToBeWritten; ++i) {
            byte[] row = RandomKeyValueUtil.randomOrderedFixedLengthKey(rand, i, 30);
            // Key will be interpreted by KeyValue.KEY_COMPARATOR
            KeyValue kv = new KeyValue(row, family, qualifier, EnvironmentEdgeManager.currentTime(), RandomKeyValueUtil.randomFixedLengthValue(rand, SMALL_BLOCK_SIZE));
            writer.append(kv);
        }
        writer.close();
    }
    // close hfile.block.index.cacheonwrite
    conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, false);
    // Read the HFile
    HFile.Reader reader = HFile.createReader(fs, hfilePath, cacheConf, conf);
    boolean hasArrayIndexOutOfBoundsException = false;
    try {
        // get the mid-key.
        reader.midkey();
    } catch (ArrayIndexOutOfBoundsException e) {
        hasArrayIndexOutOfBoundsException = true;
    } finally {
        reader.close();
    }
    // to check if ArrayIndexOutOfBoundsException occured
    assertFalse(hasArrayIndexOutOfBoundsException);
}

Example 50 with KeyValue

use of org.apache.hadoop.hbase.KeyValue in project hbase by apache.

the class TestEncodedSeekers method doPuts.

private void doPuts(Region region) throws IOException {
    LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(MIN_VALUE_SIZE, MAX_VALUE_SIZE);
    for (int i = 0; i < NUM_ROWS; ++i) {
        byte[] key = LoadTestKVGenerator.md5PrefixedKey(i).getBytes();
        for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
            Put put = new Put(key);
            put.setDurability(Durability.ASYNC_WAL);
            byte[] col = Bytes.toBytes(String.valueOf(j));
            byte[] value = dataGenerator.generateRandomSizeValue(key, col);
            if (includeTags) {
                Tag[] tag = new Tag[1];
                tag[0] = new ArrayBackedTag((byte) 1, "Visibility");
                KeyValue kv = new KeyValue(key, CF_BYTES, col, HConstants.LATEST_TIMESTAMP, value, tag);
                put.add(kv);
            } else {
                put.addColumn(CF_BYTES, col, value);
            }
            if (VERBOSE) {
                KeyValue kvPut = new KeyValue(key, CF_BYTES, col, value);
                System.err.println(Strings.padFront(i + "", ' ', 4) + " " + kvPut);
            }
            region.put(put);
        }
        if (i % NUM_ROWS_PER_FLUSH == 0) {
            region.flush(true);
        }
    }
}