Examples with ByteBufferKeyValue org.apache.hadoop.hbase.ByteBufferKeyValue used on opensource projects

Example 1 with ByteBufferKeyValue

use of org.apache.hadoop.hbase.ByteBufferKeyValue in project hbase by apache.

the class TestProtobufUtil method testToCell.

@Test
public void testToCell() throws Exception {
    KeyValue kv1 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
    KeyValue kv2 = new KeyValue(Bytes.toBytes("bbb"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
    KeyValue kv3 = new KeyValue(Bytes.toBytes("ccc"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
    byte[] arr = new byte[kv1.getLength() + kv2.getLength() + kv3.getLength()];
    System.arraycopy(kv1.getBuffer(), kv1.getOffset(), arr, 0, kv1.getLength());
    System.arraycopy(kv2.getBuffer(), kv2.getOffset(), arr, kv1.getLength(), kv2.getLength());
    System.arraycopy(kv3.getBuffer(), kv3.getOffset(), arr, kv1.getLength() + kv2.getLength(), kv3.getLength());
    ByteBuffer dbb = ByteBuffer.allocateDirect(arr.length);
    dbb.put(arr);
    ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(dbb, kv1.getLength(), kv2.getLength());
    CellProtos.Cell cell = org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.toCell(offheapKV);
    Cell newOffheapKV = org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.toCell(cell);
    assertTrue(CellComparator.COMPARATOR.compare(offheapKV, newOffheapKV) == 0);
}

Example 2 with ByteBufferKeyValue

use of org.apache.hadoop.hbase.ByteBufferKeyValue in project hbase by apache.

the class TestKeyOnlyFilter method testKeyOnly.

@Test
public void testKeyOnly() throws Exception {
    byte[] r = Bytes.toBytes("row1");
    byte[] f = Bytes.toBytes("cf1");
    byte[] q = Bytes.toBytes("qual1");
    byte[] v = Bytes.toBytes("val1");
    byte[] tags = Bytes.toBytes("tag1");
    KeyValue kv = new KeyValue(r, f, q, 0, q.length, 1234L, Type.Put, v, 0, v.length, tags);
    ByteBuffer buffer = ByteBuffer.wrap(kv.getBuffer());
    ByteBufferKeyValue bbCell = new ByteBufferKeyValue(buffer, 0, buffer.remaining());
    // KV format: <keylen:4><valuelen:4><key:keylen><value:valuelen>
    // Rebuild as: <keylen:4><0:4><key:keylen>
    int dataLen = lenAsVal ? Bytes.SIZEOF_INT : 0;
    int keyOffset = (2 * Bytes.SIZEOF_INT);
    int keyLen = KeyValueUtil.keyLength(kv);
    byte[] newBuffer = new byte[keyLen + keyOffset + dataLen];
    Bytes.putInt(newBuffer, 0, keyLen);
    Bytes.putInt(newBuffer, Bytes.SIZEOF_INT, dataLen);
    KeyValueUtil.appendKeyTo(kv, newBuffer, keyOffset);
    if (lenAsVal) {
        Bytes.putInt(newBuffer, newBuffer.length - dataLen, kv.getValueLength());
    }
    KeyValue KeyOnlyKeyValue = new KeyValue(newBuffer);
    KeyOnlyCell keyOnlyCell = new KeyOnlyCell(kv, lenAsVal);
    KeyOnlyByteBufferExtendedCell keyOnlyByteBufferedCell = new KeyOnlyByteBufferExtendedCell(bbCell, lenAsVal);
    assertTrue(CellUtil.matchingRows(KeyOnlyKeyValue, keyOnlyCell));
    assertTrue(CellUtil.matchingRows(KeyOnlyKeyValue, keyOnlyByteBufferedCell));
    assertTrue(CellUtil.matchingFamily(KeyOnlyKeyValue, keyOnlyCell));
    assertTrue(CellUtil.matchingFamily(KeyOnlyKeyValue, keyOnlyByteBufferedCell));
    assertTrue(CellUtil.matchingQualifier(KeyOnlyKeyValue, keyOnlyCell));
    assertTrue(CellUtil.matchingQualifier(KeyOnlyKeyValue, keyOnlyByteBufferedCell));
    assertTrue(CellUtil.matchingValue(KeyOnlyKeyValue, keyOnlyCell));
    assertTrue(KeyOnlyKeyValue.getValueLength() == keyOnlyByteBufferedCell.getValueLength());
    assertEquals(8 + keyLen + (lenAsVal ? 4 : 0), KeyOnlyKeyValue.getSerializedSize());
    assertEquals(8 + keyLen + (lenAsVal ? 4 : 0), keyOnlyCell.getSerializedSize());
    if (keyOnlyByteBufferedCell.getValueLength() > 0) {
        assertTrue(CellUtil.matchingValue(KeyOnlyKeyValue, keyOnlyByteBufferedCell));
    }
    assertTrue(KeyOnlyKeyValue.getTimestamp() == keyOnlyCell.getTimestamp());
    assertTrue(KeyOnlyKeyValue.getTimestamp() == keyOnlyByteBufferedCell.getTimestamp());
    assertTrue(KeyOnlyKeyValue.getTypeByte() == keyOnlyCell.getTypeByte());
    assertTrue(KeyOnlyKeyValue.getTypeByte() == keyOnlyByteBufferedCell.getTypeByte());
    assertTrue(KeyOnlyKeyValue.getTagsLength() == keyOnlyCell.getTagsLength());
    assertTrue(KeyOnlyKeyValue.getTagsLength() == keyOnlyByteBufferedCell.getTagsLength());
}

Example 3 with ByteBufferKeyValue

use of org.apache.hadoop.hbase.ByteBufferKeyValue in project hbase by apache.

the class TestProtobufUtil method testToCell.

@Test
public void testToCell() {
    KeyValue kv1 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
    KeyValue kv2 = new KeyValue(Bytes.toBytes("bbb"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
    KeyValue kv3 = new KeyValue(Bytes.toBytes("ccc"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
    byte[] arr = new byte[kv1.getLength() + kv2.getLength() + kv3.getLength()];
    System.arraycopy(kv1.getBuffer(), kv1.getOffset(), arr, 0, kv1.getLength());
    System.arraycopy(kv2.getBuffer(), kv2.getOffset(), arr, kv1.getLength(), kv2.getLength());
    System.arraycopy(kv3.getBuffer(), kv3.getOffset(), arr, kv1.getLength() + kv2.getLength(), kv3.getLength());
    ByteBuffer dbb = ByteBuffer.allocateDirect(arr.length);
    dbb.put(arr);
    ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(dbb, kv1.getLength(), kv2.getLength());
    CellProtos.Cell cell = ProtobufUtil.toCell(offheapKV, false);
    Cell newOffheapKV = ProtobufUtil.toCell(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY), cell, false);
    assertTrue(CellComparatorImpl.COMPARATOR.compare(offheapKV, newOffheapKV) == 0);
}

Example 4 with ByteBufferKeyValue

use of org.apache.hadoop.hbase.ByteBufferKeyValue in project hbase by apache.

the class CellChunkImmutableSegment method initializeCellSet.

// ///////////////////  PRIVATE METHODS  /////////////////////
/*------------------------------------------------------------------------*/
// Create CellSet based on CellChunkMap from compacting iterator
private void initializeCellSet(int numOfCells, MemStoreSegmentsIterator iterator, MemStoreCompactionStrategy.Action action) {
    int numOfCellsAfterCompaction = 0;
    int currentChunkIdx = 0;
    int offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER;
    int numUniqueKeys = 0;
    Cell prev = null;
    Chunk[] chunks = allocIndexChunks(numOfCells);
    while (iterator.hasNext()) {
        // the iterator hides the elimination logic for compaction
        boolean alreadyCopied = false;
        Cell c = iterator.next();
        numOfCellsAfterCompaction++;
        assert (c instanceof ExtendedCell);
        if (((ExtendedCell) c).getChunkId() == ExtendedCell.CELL_NOT_BASED_ON_CHUNK) {
            // CellChunkMap assumes all cells are allocated on MSLAB.
            // Therefore, cells which are not allocated on MSLAB initially,
            // are copied into MSLAB here.
            // no memstore sizing object to update
            c = copyCellIntoMSLAB(c, null);
            alreadyCopied = true;
        }
        if (offsetInCurentChunk + ClassSize.CELL_CHUNK_MAP_ENTRY > chunks[currentChunkIdx].size) {
            // continue to the next index chunk
            currentChunkIdx++;
            offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER;
        }
        if (action == MemStoreCompactionStrategy.Action.COMPACT && !alreadyCopied) {
            // for compaction copy cell to the new segment (MSLAB copy)
            c = maybeCloneWithAllocator(c, false);
        }
        // add the Cell reference to the index chunk
        offsetInCurentChunk = createCellReference((ByteBufferKeyValue) c, chunks[currentChunkIdx].getData(), offsetInCurentChunk);
        // the sizes still need to be updated in the new segment
        // second parameter true, because in compaction/merge the addition of the cell to new segment
        // is always successful
        // updates the size per cell
        updateMetaInfo(c, true, null);
        if (action == MemStoreCompactionStrategy.Action.MERGE_COUNT_UNIQUE_KEYS) {
            // counting number of unique keys
            if (prev != null) {
                if (!CellUtil.matchingRowColumnBytes(prev, c)) {
                    numUniqueKeys++;
                }
            } else {
                numUniqueKeys++;
            }
        }
        prev = c;
    }
    if (action == MemStoreCompactionStrategy.Action.COMPACT) {
        numUniqueKeys = numOfCells;
    } else if (action != MemStoreCompactionStrategy.Action.MERGE_COUNT_UNIQUE_KEYS) {
        numUniqueKeys = CellSet.UNKNOWN_NUM_UNIQUES;
    }
    // build the immutable CellSet
    CellChunkMap ccm = new CellChunkMap(getComparator(), chunks, 0, numOfCellsAfterCompaction, false);
    // update the CellSet of this Segment
    this.setCellSet(null, new CellSet(ccm, numUniqueKeys));
}

Example 5 with ByteBufferKeyValue

use of org.apache.hadoop.hbase.ByteBufferKeyValue in project hbase by apache.

the class TestComparators method testCellFieldsCompare.

@Test
public void testCellFieldsCompare() throws Exception {
    byte[] r0 = Bytes.toBytes("row0");
    byte[] r1 = Bytes.toBytes("row1");
    byte[] r2 = Bytes.toBytes("row2");
    byte[] f = Bytes.toBytes("cf1");
    byte[] q1 = Bytes.toBytes("qual1");
    byte[] q2 = Bytes.toBytes("qual2");
    byte[] q3 = Bytes.toBytes("r");
    long l1 = 1234L;
    byte[] v1 = Bytes.toBytes(l1);
    long l2 = 2000L;
    byte[] v2 = Bytes.toBytes(l2);
    // Row compare
    KeyValue kv = new KeyValue(r1, f, q1, v1);
    ByteBuffer buffer = ByteBuffer.wrap(kv.getBuffer());
    Cell bbCell = new ByteBufferKeyValue(buffer, 0, buffer.remaining());
    ByteArrayComparable comparable = new BinaryComparator(r1);
    assertEquals(0, PrivateCellUtil.compareRow(bbCell, comparable));
    assertEquals(0, PrivateCellUtil.compareRow(kv, comparable));
    kv = new KeyValue(r0, f, q1, v1);
    buffer = ByteBuffer.wrap(kv.getBuffer());
    bbCell = new ByteBufferKeyValue(buffer, 0, buffer.remaining());
    assertTrue(PrivateCellUtil.compareRow(bbCell, comparable) > 0);
    assertTrue(PrivateCellUtil.compareRow(kv, comparable) > 0);
    kv = new KeyValue(r2, f, q1, v1);
    buffer = ByteBuffer.wrap(kv.getBuffer());
    bbCell = new ByteBufferKeyValue(buffer, 0, buffer.remaining());
    assertTrue(PrivateCellUtil.compareRow(bbCell, comparable) < 0);
    assertTrue(PrivateCellUtil.compareRow(kv, comparable) < 0);
    // Qualifier compare
    comparable = new BinaryPrefixComparator(Bytes.toBytes("qual"));
    assertEquals(0, PrivateCellUtil.compareQualifier(bbCell, comparable));
    assertEquals(0, PrivateCellUtil.compareQualifier(kv, comparable));
    kv = new KeyValue(r2, f, q2, v1);
    buffer = ByteBuffer.wrap(kv.getBuffer());
    bbCell = new ByteBufferKeyValue(buffer, 0, buffer.remaining());
    assertEquals(0, PrivateCellUtil.compareQualifier(bbCell, comparable));
    assertEquals(0, PrivateCellUtil.compareQualifier(kv, comparable));
    kv = new KeyValue(r2, f, q3, v1);
    buffer = ByteBuffer.wrap(kv.getBuffer());
    bbCell = new ByteBufferKeyValue(buffer, 0, buffer.remaining());
    assertTrue(PrivateCellUtil.compareQualifier(bbCell, comparable) < 0);
    assertTrue(PrivateCellUtil.compareQualifier(kv, comparable) < 0);
    // Value compare
    comparable = new LongComparator(l1);
    assertEquals(0, PrivateCellUtil.compareValue(bbCell, comparable));
    assertEquals(0, PrivateCellUtil.compareValue(kv, comparable));
    kv = new KeyValue(r1, f, q1, v2);
    buffer = ByteBuffer.wrap(kv.getBuffer());
    bbCell = new ByteBufferKeyValue(buffer, 0, buffer.remaining());
    assertTrue(PrivateCellUtil.compareValue(bbCell, comparable) < 0);
    assertTrue(PrivateCellUtil.compareValue(kv, comparable) < 0);
    // Family compare
    comparable = new SubstringComparator("cf");
    assertEquals(0, PrivateCellUtil.compareFamily(bbCell, comparable));
    assertEquals(0, PrivateCellUtil.compareFamily(kv, comparable));
    // Qualifier starts with
    kv = new KeyValue(r1, f, q1, v2);
    assertTrue(PrivateCellUtil.qualifierStartsWith(kv, Bytes.toBytes("q")));
    assertTrue(PrivateCellUtil.qualifierStartsWith(kv, q1));
    assertFalse(PrivateCellUtil.qualifierStartsWith(kv, q2));
    assertFalse(PrivateCellUtil.qualifierStartsWith(kv, Bytes.toBytes("longerthanthequalifier")));
    // Binary component comparisons
    byte[] val = Bytes.toBytes("abcd");
    kv = new KeyValue(r0, f, q1, val);
    buffer = ByteBuffer.wrap(kv.getBuffer());
    bbCell = new ByteBufferKeyValue(buffer, 0, buffer.remaining());
    // equality check
    // row comparison
    // row is "row0"(set by variable r0)
    // and we are checking for equality to 'o' at position 1
    // 'r' is at position 0.
    byte[] component = Bytes.toBytes("o");
    comparable = new BinaryComponentComparator(component, 1);
    assertEquals(0, PrivateCellUtil.compareRow(bbCell, comparable));
    assertEquals(0, PrivateCellUtil.compareRow(kv, comparable));
    // value comparison
    // value is "abcd"(set by variable val).
    // and we are checking for equality to 'c' at position 2.
    // 'a' is at position 0.
    component = Bytes.toBytes("c");
    comparable = new BinaryComponentComparator(component, 2);
    assertEquals(0, PrivateCellUtil.compareValue(bbCell, comparable));
    assertEquals(0, PrivateCellUtil.compareValue(kv, comparable));
    // greater than
    component = Bytes.toBytes("z");
    // checking for greater than at position 1.
    // for both row("row0") and value("abcd")
    // 'z' > 'r'
    comparable = new BinaryComponentComparator(component, 1);
    // row comparison
    assertTrue(PrivateCellUtil.compareRow(bbCell, comparable) > 0);
    assertTrue(PrivateCellUtil.compareRow(kv, comparable) > 0);
    // value comparison
    // 'z' > 'a'
    assertTrue(PrivateCellUtil.compareValue(bbCell, comparable) > 0);
    assertTrue(PrivateCellUtil.compareValue(kv, comparable) > 0);
    // less than
    component = Bytes.toBytes("a");
    // checking for less than at position 1 for row ("row0")
    comparable = new BinaryComponentComparator(component, 1);
    // row comparison
    // 'a' < 'r'
    assertTrue(PrivateCellUtil.compareRow(bbCell, comparable) < 0);
    assertTrue(PrivateCellUtil.compareRow(kv, comparable) < 0);
    // value comparison
    // checking for less than at position 2 for value("abcd")
    // 'a' < 'c'
    comparable = new BinaryComponentComparator(component, 2);
    assertTrue(PrivateCellUtil.compareValue(bbCell, comparable) < 0);
    assertTrue(PrivateCellUtil.compareValue(kv, comparable) < 0);
}