Examples with KeyValue org.apache.hadoop.hbase.KeyValue used on opensource projects

Example 51 with KeyValue

use of org.apache.hadoop.hbase.KeyValue in project hbase by apache.

the class TestHalfStoreFileReader method testHalfScanner.

// Tests the scanner on an HFile that is backed by HalfStoreFiles
@Test
public void testHalfScanner() throws IOException {
    String root_dir = TEST_UTIL.getDataTestDir().toString();
    Path p = new Path(root_dir, "test");
    Configuration conf = TEST_UTIL.getConfiguration();
    FileSystem fs = FileSystem.get(conf);
    CacheConfig cacheConf = new CacheConfig(conf);
    HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build();
    HFile.Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(fs, p).withFileContext(meta).create();
    // write some things.
    List<KeyValue> items = genSomeKeys();
    for (KeyValue kv : items) {
        w.append(kv);
    }
    w.close();
    HFile.Reader r = HFile.createReader(fs, p, cacheConf, conf);
    r.loadFileInfo();
    Cell midKV = r.midkey();
    byte[] midkey = CellUtil.cloneRow(midKV);
    Reference bottom = new Reference(midkey, Reference.Range.bottom);
    Reference top = new Reference(midkey, Reference.Range.top);
    // Ugly code to get the item before the midkey
    KeyValue beforeMidKey = null;
    for (KeyValue item : items) {
        if (CellComparator.COMPARATOR.compare(item, midKV) >= 0) {
            break;
        }
        beforeMidKey = item;
    }
    System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));
    System.out.println("beforeMidKey: " + beforeMidKey);
    // Seek on the splitKey, should be in top, not in bottom
    Cell foundKeyValue = doTestOfSeekBefore(p, fs, bottom, midKV, cacheConf);
    assertEquals(beforeMidKey, foundKeyValue);
    // Seek tot the last thing should be the penultimate on the top, the one before the midkey on the bottom.
    foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(items.size() - 1), cacheConf);
    assertEquals(items.get(items.size() - 2), foundKeyValue);
    foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(items.size() - 1), cacheConf);
    assertEquals(beforeMidKey, foundKeyValue);
    // Try and seek before something that is in the bottom.
    foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(0), cacheConf);
    assertNull(foundKeyValue);
    // Try and seek before the first thing.
    foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(0), cacheConf);
    assertNull(foundKeyValue);
    // Try and seek before the second thing in the top and bottom.
    foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(1), cacheConf);
    assertNull(foundKeyValue);
    foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(1), cacheConf);
    assertEquals(items.get(0), foundKeyValue);
    // Try to seek before the splitKey in the top file
    foundKeyValue = doTestOfSeekBefore(p, fs, top, midKV, cacheConf);
    assertNull(foundKeyValue);
}

Example 52 with KeyValue

use of org.apache.hadoop.hbase.KeyValue in project hbase by apache.

the class TestHeapSize method testSizes.

/**
   * Testing the classes that implements HeapSize and are a part of 0.20.
   * Some are not tested here for example BlockIndex which is tested in
   * TestHFile since it is a non public class
   * @throws IOException
   */
@Test
public void testSizes() throws IOException {
    Class<?> cl;
    long expected;
    long actual;
    //KeyValue
    cl = KeyValue.class;
    expected = ClassSize.estimateBase(cl, false);
    KeyValue kv = new KeyValue();
    actual = kv.heapSize();
    if (expected != actual) {
        ClassSize.estimateBase(cl, true);
        assertEquals(expected, actual);
    }
    //LruBlockCache Overhead
    cl = LruBlockCache.class;
    actual = LruBlockCache.CACHE_FIXED_OVERHEAD;
    expected = ClassSize.estimateBase(cl, false);
    if (expected != actual) {
        ClassSize.estimateBase(cl, true);
        assertEquals(expected, actual);
    }
    // CachedBlock Fixed Overhead
    // We really need "deep" sizing but ClassSize does not do this.
    // Perhaps we should do all these more in this style....
    cl = LruCachedBlock.class;
    actual = LruCachedBlock.PER_BLOCK_OVERHEAD;
    expected = ClassSize.estimateBase(cl, false);
    expected += ClassSize.estimateBase(String.class, false);
    expected += ClassSize.estimateBase(ByteBuffer.class, false);
    if (expected != actual) {
        ClassSize.estimateBase(cl, true);
        ClassSize.estimateBase(String.class, true);
        ClassSize.estimateBase(ByteBuffer.class, true);
        assertEquals(expected, actual);
    }
    // DefaultMemStore Overhead
    cl = DefaultMemStore.class;
    actual = DefaultMemStore.FIXED_OVERHEAD;
    expected = ClassSize.estimateBase(cl, false);
    if (expected != actual) {
        ClassSize.estimateBase(cl, true);
        assertEquals(expected, actual);
    }
    // DefaultMemStore Deep Overhead
    actual = DefaultMemStore.DEEP_OVERHEAD;
    expected = ClassSize.estimateBase(cl, false);
    if (expected != actual) {
        ClassSize.estimateBase(cl, true);
        assertEquals(expected, actual);
    }
    // CompactingMemStore Deep Overhead
    cl = CompactingMemStore.class;
    actual = CompactingMemStore.DEEP_OVERHEAD;
    expected = ClassSize.estimateBase(cl, false);
    expected += ClassSize.estimateBase(AtomicBoolean.class, false);
    expected += ClassSize.estimateBase(AtomicBoolean.class, false);
    expected += ClassSize.estimateBase(CompactionPipeline.class, false);
    //inside CompactionPipeline
    expected += ClassSize.estimateBase(LinkedList.class, false);
    //inside CompactionPipeline
    expected += ClassSize.estimateBase(LinkedList.class, false);
    expected += ClassSize.estimateBase(MemStoreCompactor.class, false);
    // inside MemStoreCompactor
    expected += ClassSize.estimateBase(AtomicBoolean.class, false);
    if (expected != actual) {
        ClassSize.estimateBase(cl, true);
        ClassSize.estimateBase(AtomicBoolean.class, true);
        ClassSize.estimateBase(AtomicBoolean.class, true);
        ClassSize.estimateBase(CompactionPipeline.class, true);
        ClassSize.estimateBase(LinkedList.class, true);
        ClassSize.estimateBase(LinkedList.class, true);
        ClassSize.estimateBase(MemStoreCompactor.class, true);
        ClassSize.estimateBase(AtomicBoolean.class, true);
        assertEquals(expected, actual);
    }
    // Segment Deep overhead
    cl = Segment.class;
    actual = Segment.DEEP_OVERHEAD;
    expected = ClassSize.estimateBase(cl, false);
    expected += ClassSize.estimateBase(AtomicLong.class, false);
    expected += ClassSize.estimateBase(AtomicReference.class, false);
    expected += ClassSize.estimateBase(CellSet.class, false);
    expected += ClassSize.estimateBase(TimeRangeTracker.class, false);
    if (expected != actual) {
        ClassSize.estimateBase(cl, true);
        ClassSize.estimateBase(AtomicLong.class, true);
        ClassSize.estimateBase(AtomicReference.class, true);
        ClassSize.estimateBase(CellSet.class, true);
        ClassSize.estimateBase(TimeRangeTracker.class, true);
        assertEquals(expected, actual);
    }
    // MutableSegment Deep overhead
    cl = MutableSegment.class;
    actual = MutableSegment.DEEP_OVERHEAD;
    expected = ClassSize.estimateBase(cl, false);
    expected += ClassSize.estimateBase(AtomicLong.class, false);
    expected += ClassSize.estimateBase(AtomicReference.class, false);
    expected += ClassSize.estimateBase(CellSet.class, false);
    expected += ClassSize.estimateBase(TimeRangeTracker.class, false);
    expected += ClassSize.estimateBase(ConcurrentSkipListMap.class, false);
    if (expected != actual) {
        ClassSize.estimateBase(cl, true);
        ClassSize.estimateBase(AtomicLong.class, true);
        ClassSize.estimateBase(AtomicReference.class, true);
        ClassSize.estimateBase(CellSet.class, true);
        ClassSize.estimateBase(TimeRangeTracker.class, true);
        ClassSize.estimateBase(ConcurrentSkipListMap.class, true);
        assertEquals(expected, actual);
    }
    // ImmutableSegment Deep overhead
    cl = ImmutableSegment.class;
    actual = ImmutableSegment.DEEP_OVERHEAD_CSLM;
    expected = ClassSize.estimateBase(cl, false);
    expected += ClassSize.estimateBase(AtomicLong.class, false);
    expected += ClassSize.estimateBase(AtomicReference.class, false);
    expected += ClassSize.estimateBase(CellSet.class, false);
    expected += ClassSize.estimateBase(TimeRangeTracker.class, false);
    expected += ClassSize.estimateBase(TimeRange.class, false);
    expected += ClassSize.estimateBase(ConcurrentSkipListMap.class, false);
    if (expected != actual) {
        ClassSize.estimateBase(cl, true);
        ClassSize.estimateBase(AtomicLong.class, true);
        ClassSize.estimateBase(AtomicReference.class, true);
        ClassSize.estimateBase(CellSet.class, true);
        ClassSize.estimateBase(TimeRangeTracker.class, true);
        ClassSize.estimateBase(TimeRange.class, true);
        ClassSize.estimateBase(ConcurrentSkipListMap.class, true);
        assertEquals(expected, actual);
    }
    actual = ImmutableSegment.DEEP_OVERHEAD_CAM;
    expected = ClassSize.estimateBase(cl, false);
    expected += ClassSize.estimateBase(AtomicLong.class, false);
    expected += ClassSize.estimateBase(AtomicReference.class, false);
    expected += ClassSize.estimateBase(CellSet.class, false);
    expected += ClassSize.estimateBase(TimeRangeTracker.class, false);
    expected += ClassSize.estimateBase(TimeRange.class, false);
    expected += ClassSize.estimateBase(CellArrayMap.class, false);
    if (expected != actual) {
        ClassSize.estimateBase(cl, true);
        ClassSize.estimateBase(AtomicLong.class, true);
        ClassSize.estimateBase(AtomicReference.class, true);
        ClassSize.estimateBase(CellSet.class, true);
        ClassSize.estimateBase(TimeRangeTracker.class, true);
        ClassSize.estimateBase(TimeRange.class, true);
        ClassSize.estimateBase(CellArrayMap.class, true);
        assertEquals(expected, actual);
    }
    // Store Overhead
    cl = HStore.class;
    actual = HStore.FIXED_OVERHEAD;
    expected = ClassSize.estimateBase(cl, false);
    if (expected != actual) {
        ClassSize.estimateBase(cl, true);
        assertEquals(expected, actual);
    }
    // Region Overhead
    cl = HRegion.class;
    actual = HRegion.FIXED_OVERHEAD;
    expected = ClassSize.estimateBase(cl, false);
    if (expected != actual) {
        ClassSize.estimateBase(cl, true);
        assertEquals(expected, actual);
    }
    // Block cache key overhead. Only tests fixed overhead as estimating heap
    // size of strings is hard.
    cl = BlockCacheKey.class;
    actual = BlockCacheKey.FIXED_OVERHEAD;
    expected = ClassSize.estimateBase(cl, false);
    if (expected != actual) {
        ClassSize.estimateBase(cl, true);
        assertEquals(expected, actual);
    }
// Currently NOT testing Deep Overheads of many of these classes.
// Deep overheads cover a vast majority of stuff, but will not be 100%
// accurate because it's unclear when we're referencing stuff that's already
// accounted for.  But we have satisfied our two core requirements.
// Sizing is quite accurate now, and our tests will throw errors if
// any of these classes are modified without updating overhead sizes.
}

Example 53 with KeyValue

use of org.apache.hadoop.hbase.KeyValue in project hbase by apache.

the class TestBufferedDataBlockEncoder method testCommonPrefixComparators.

@Test
public void testCommonPrefixComparators() {
    KeyValue kv1 = new KeyValue(row1, fam1, qual1, 1l, Type.Put);
    KeyValue kv2 = new KeyValue(row1, fam_1_2, qual1, 1l, Type.Maximum);
    assertTrue((BufferedDataBlockEncoder.compareCommonFamilyPrefix(kv1, kv2, 4) < 0));
    kv1 = new KeyValue(row1, fam1, qual1, 1l, Type.Put);
    kv2 = new KeyValue(row_1_0, fam_1_2, qual1, 1l, Type.Maximum);
    assertTrue((BufferedDataBlockEncoder.compareCommonRowPrefix(kv1, kv2, 4) < 0));
    kv1 = new KeyValue(row1, fam1, qual2, 1l, Type.Put);
    kv2 = new KeyValue(row1, fam1, qual1, 1l, Type.Maximum);
    assertTrue((BufferedDataBlockEncoder.compareCommonQualifierPrefix(kv1, kv2, 4) > 0));
}

Example 54 with KeyValue

use of org.apache.hadoop.hbase.KeyValue in project hbase by apache.

the class TestDataBlockEncoders method testSeekingOnSample.

/**
   * Test seeking while file is encoded.
   */
@Test
public void testSeekingOnSample() throws IOException {
    List<KeyValue> sampleKv = generator.generateTestKeyValues(NUMBER_OF_KV, includesTags);
    // create all seekers
    List<DataBlockEncoder.EncodedSeeker> encodedSeekers = new ArrayList<>();
    for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
        LOG.info("Encoding: " + encoding);
        // TODO remove this once support is added. HBASE-12298
        if (this.useOffheapData && encoding == DataBlockEncoding.PREFIX_TREE)
            continue;
        DataBlockEncoder encoder = encoding.getEncoder();
        if (encoder == null) {
            continue;
        }
        LOG.info("Encoder: " + encoder);
        ByteBuffer encodedBuffer = encodeKeyValues(encoding, sampleKv, getEncodingContext(Compression.Algorithm.NONE, encoding), this.useOffheapData);
        HFileContext meta = new HFileContextBuilder().withHBaseCheckSum(false).withIncludesMvcc(includesMemstoreTS).withIncludesTags(includesTags).withCompression(Compression.Algorithm.NONE).build();
        DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparator.COMPARATOR, encoder.newDataBlockDecodingContext(meta));
        seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
        encodedSeekers.add(seeker);
    }
    LOG.info("Testing it!");
    // try a few random seeks
    for (boolean seekBefore : new boolean[] { false, true }) {
        for (int i = 0; i < NUM_RANDOM_SEEKS; ++i) {
            int keyValueId;
            if (!seekBefore) {
                keyValueId = randomizer.nextInt(sampleKv.size());
            } else {
                keyValueId = randomizer.nextInt(sampleKv.size() - 1) + 1;
            }
            KeyValue keyValue = sampleKv.get(keyValueId);
            checkSeekingConsistency(encodedSeekers, seekBefore, keyValue);
        }
    }
    // check edge cases
    LOG.info("Checking edge cases");
    checkSeekingConsistency(encodedSeekers, false, sampleKv.get(0));
    for (boolean seekBefore : new boolean[] { false, true }) {
        checkSeekingConsistency(encodedSeekers, seekBefore, sampleKv.get(sampleKv.size() - 1));
        KeyValue midKv = sampleKv.get(sampleKv.size() / 2);
        Cell lastMidKv = CellUtil.createLastOnRowCol(midKv);
        checkSeekingConsistency(encodedSeekers, seekBefore, lastMidKv);
    }
    LOG.info("Done");
}

Example 55 with KeyValue

use of org.apache.hadoop.hbase.KeyValue in project hbase by apache.

the class TestDataBlockEncoders method encodeKeyValues.

static ByteBuffer encodeKeyValues(DataBlockEncoding encoding, List<KeyValue> kvs, HFileBlockEncodingContext encodingContext, boolean useOffheapData) throws IOException {
    DataBlockEncoder encoder = encoding.getEncoder();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    baos.write(HFILEBLOCK_DUMMY_HEADER);
    DataOutputStream dos = new DataOutputStream(baos);
    encoder.startBlockEncoding(encodingContext, dos);
    for (KeyValue kv : kvs) {
        encoder.encode(kv, encodingContext, dos);
    }
    encoder.endBlockEncoding(encodingContext, dos, baos.getBuffer());
    byte[] encodedData = new byte[baos.size() - ENCODED_DATA_OFFSET];
    System.arraycopy(baos.toByteArray(), ENCODED_DATA_OFFSET, encodedData, 0, encodedData.length);
    if (useOffheapData) {
        ByteBuffer bb = ByteBuffer.allocateDirect(encodedData.length);
        bb.put(encodedData);
        bb.rewind();
        return bb;
    }
    return ByteBuffer.wrap(encodedData);
}