Examples with ByteBufferKeyValue org.apache.hadoop.hbase.ByteBufferKeyValue used on opensource projects

Example 16 with ByteBufferKeyValue

use of org.apache.hadoop.hbase.ByteBufferKeyValue in project hbase by apache.

the class TestWALReaderOnSecureWAL method writeWAL.

private Path writeWAL(final WALFactory wals, final String tblName, boolean offheap) throws IOException {
    Configuration conf = TEST_UTIL.getConfiguration();
    String clsName = conf.get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());
    conf.setClass(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, SecureWALCellCodec.class, WALCellCodec.class);
    try {
        TableName tableName = TableName.valueOf(tblName);
        NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
        scopes.put(tableName.getName(), 0);
        RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();
        final int total = 10;
        final byte[] row = Bytes.toBytes("row");
        final byte[] family = Bytes.toBytes("family");
        final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);
        // Write the WAL
        WAL wal = wals.getWAL(regionInfo);
        for (int i = 0; i < total; i++) {
            WALEdit kvs = new WALEdit();
            KeyValue kv = new KeyValue(row, family, Bytes.toBytes(i), value);
            if (offheap) {
                ByteBuffer bb = ByteBuffer.allocateDirect(kv.getBuffer().length);
                bb.put(kv.getBuffer());
                ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(bb, 0, kv.getLength());
                kvs.add(offheapKV);
            } else {
                kvs.add(kv);
            }
            wal.appendData(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName, EnvironmentEdgeManager.currentTime(), mvcc, scopes), kvs);
        }
        wal.sync();
        final Path walPath = AbstractFSWALProvider.getCurrentFileName(wal);
        wal.shutdown();
        return walPath;
    } finally {
        // restore the cell codec class
        conf.set(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, clsName);
    }
}

Example 17 with ByteBufferKeyValue

use of org.apache.hadoop.hbase.ByteBufferKeyValue in project hbase by apache.

the class TestWALCellCodecWithCompression method createOffheapKV.

private ByteBufferKeyValue createOffheapKV(int noOfTags) {
    byte[] row = Bytes.toBytes("myRow");
    byte[] cf = Bytes.toBytes("myCF");
    byte[] q = Bytes.toBytes("myQualifier");
    byte[] value = Bytes.toBytes("myValue");
    List<Tag> tags = new ArrayList<>(noOfTags);
    for (int i = 1; i <= noOfTags; i++) {
        tags.add(new ArrayBackedTag((byte) i, Bytes.toBytes("tagValue" + i)));
    }
    KeyValue kv = new KeyValue(row, cf, q, HConstants.LATEST_TIMESTAMP, value, tags);
    ByteBuffer dbb = ByteBuffer.allocateDirect(kv.getBuffer().length);
    dbb.put(kv.getBuffer());
    return new ByteBufferKeyValue(dbb, 0, kv.getBuffer().length);
}

Example 18 with ByteBufferKeyValue

use of org.apache.hadoop.hbase.ByteBufferKeyValue in project hbase by apache.

the class RedundantKVGenerator method generateTestExtendedOffheapKeyValues.

/**
   * Generate test data useful to test encoders.
   * @param howMany How many Key values should be generated.
   * @return sorted list of key values
   */
public List<Cell> generateTestExtendedOffheapKeyValues(int howMany, boolean useTags) {
    List<Cell> result = new ArrayList<>();
    List<byte[]> rows = generateRows();
    Map<Integer, List<byte[]>> rowsToQualifier = new HashMap<>();
    if (family == null) {
        family = new byte[columnFamilyLength];
        randomizer.nextBytes(family);
    }
    long baseTimestamp = Math.abs(randomizer.nextInt()) / baseTimestampDivide;
    byte[] value = new byte[valueLength];
    for (int i = 0; i < howMany; ++i) {
        long timestamp = baseTimestamp;
        if (timestampDiffSize > 0) {
            timestamp += randomizer.nextInt(timestampDiffSize);
        }
        Integer rowId = randomizer.nextInt(rows.size());
        byte[] row = rows.get(rowId);
        // generate qualifier, sometimes it is same, sometimes similar,
        // occasionally completely different
        byte[] qualifier;
        float qualifierChance = randomizer.nextFloat();
        if (!rowsToQualifier.containsKey(rowId) || qualifierChance > chanceForSameQualifier + chanceForSimilarQualifier) {
            int qualifierLength = averageQualifierLength;
            qualifierLength += randomizer.nextInt(2 * qualifierLengthVariance + 1) - qualifierLengthVariance;
            qualifier = new byte[qualifierLength];
            randomizer.nextBytes(qualifier);
            // add it to map
            if (!rowsToQualifier.containsKey(rowId)) {
                rowsToQualifier.put(rowId, new ArrayList<>());
            }
            rowsToQualifier.get(rowId).add(qualifier);
        } else if (qualifierChance > chanceForSameQualifier) {
            // similar qualifier
            List<byte[]> previousQualifiers = rowsToQualifier.get(rowId);
            byte[] originalQualifier = previousQualifiers.get(randomizer.nextInt(previousQualifiers.size()));
            qualifier = new byte[originalQualifier.length];
            int commonPrefix = randomizer.nextInt(qualifier.length);
            System.arraycopy(originalQualifier, 0, qualifier, 0, commonPrefix);
            for (int j = commonPrefix; j < qualifier.length; ++j) {
                qualifier[j] = (byte) (randomizer.nextInt() & 0xff);
            }
            rowsToQualifier.get(rowId).add(qualifier);
        } else {
            // same qualifier
            List<byte[]> previousQualifiers = rowsToQualifier.get(rowId);
            qualifier = previousQualifiers.get(randomizer.nextInt(previousQualifiers.size()));
        }
        if (randomizer.nextFloat() < chanceForZeroValue) {
            for (int j = 0; j < value.length; ++j) {
                value[j] = (byte) 0;
            }
        } else {
            randomizer.nextBytes(value);
        }
        if (useTags) {
            KeyValue keyValue = new KeyValue(row, family, qualifier, timestamp, value, new Tag[] { new ArrayBackedTag((byte) 1, "value1") });
            ByteBuffer offheapKVBB = ByteBuffer.allocateDirect(keyValue.getLength());
            ByteBufferUtils.copyFromArrayToBuffer(offheapKVBB, keyValue.getBuffer(), keyValue.getOffset(), keyValue.getLength());
            ByteBufferKeyValue offheapKV = new ExtendedOffheapKeyValue(offheapKVBB, 0, keyValue.getLength(), 0);
            result.add(offheapKV);
        } else {
            KeyValue keyValue = new KeyValue(row, family, qualifier, timestamp, value);
            ByteBuffer offheapKVBB = ByteBuffer.allocateDirect(keyValue.getLength());
            ByteBufferUtils.copyFromArrayToBuffer(offheapKVBB, keyValue.getBuffer(), keyValue.getOffset(), keyValue.getLength());
            ByteBufferKeyValue offheapKV = new ExtendedOffheapKeyValue(offheapKVBB, 0, keyValue.getLength(), 0);
            result.add(offheapKV);
        }
    }
    Collections.sort(result, CellComparator.COMPARATOR);
    return result;
}

Example 19 with ByteBufferKeyValue

use of org.apache.hadoop.hbase.ByteBufferKeyValue in project hbase by apache.

the class CellChunkImmutableSegment method reinitializeCellSet.

/*------------------------------------------------------------------------*/
// Create CellSet based on CellChunkMap from current ConcurrentSkipListMap based CellSet
// (without compacting iterator)
// This is a service for not-flat immutable segments
private void reinitializeCellSet(int numOfCells, KeyValueScanner segmentScanner, CellSet oldCellSet, MemStoreSizing memstoreSizing, MemStoreCompactionStrategy.Action action) {
    Cell curCell;
    Chunk[] chunks = allocIndexChunks(numOfCells);
    int currentChunkIdx = 0;
    int offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER;
    int numUniqueKeys = 0;
    Cell prev = null;
    try {
        while ((curCell = segmentScanner.next()) != null) {
            assert (curCell instanceof ExtendedCell);
            if (((ExtendedCell) curCell).getChunkId() == ExtendedCell.CELL_NOT_BASED_ON_CHUNK) {
                // CellChunkMap assumes all cells are allocated on MSLAB.
                // Therefore, cells which are not allocated on MSLAB initially,
                // are copied into MSLAB here.
                curCell = copyCellIntoMSLAB(curCell, memstoreSizing);
            }
            if (offsetInCurentChunk + ClassSize.CELL_CHUNK_MAP_ENTRY > chunks[currentChunkIdx].size) {
                // continue to the next metadata chunk
                currentChunkIdx++;
                offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER;
            }
            offsetInCurentChunk = createCellReference((ByteBufferKeyValue) curCell, chunks[currentChunkIdx].getData(), offsetInCurentChunk);
            if (action == MemStoreCompactionStrategy.Action.FLATTEN_COUNT_UNIQUE_KEYS) {
                // counting number of unique keys
                if (prev != null) {
                    if (!CellUtil.matchingRowColumn(prev, curCell)) {
                        numUniqueKeys++;
                    }
                } else {
                    numUniqueKeys++;
                }
            }
            prev = curCell;
        }
        if (action != MemStoreCompactionStrategy.Action.FLATTEN_COUNT_UNIQUE_KEYS) {
            numUniqueKeys = CellSet.UNKNOWN_NUM_UNIQUES;
        }
    } catch (IOException ie) {
        throw new IllegalStateException(ie);
    } finally {
        segmentScanner.close();
    }
    CellChunkMap ccm = new CellChunkMap(getComparator(), chunks, 0, numOfCells, false);
    // update the CellSet of this Segment
    this.setCellSet(oldCellSet, new CellSet(ccm, numUniqueKeys));
}

Example 20 with ByteBufferKeyValue

use of org.apache.hadoop.hbase.ByteBufferKeyValue in project hbase by apache.

the class TestSingleColumnValueFilter method regexFilterTests.

private void regexFilterTests(Filter filter) throws Exception {
    KeyValue cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_1);
    assertTrue("regexTrue", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE);
    byte[] buffer = cell.getBuffer();
    Cell c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length);
    assertTrue("regexTrue", filter.filterCell(c) == Filter.ReturnCode.INCLUDE);
    cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_2);
    assertTrue("regexFalse", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE);
    buffer = cell.getBuffer();
    c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length);
    assertTrue("regexFalse", filter.filterCell(c) == Filter.ReturnCode.INCLUDE);
    assertFalse("regexFilterAllRemaining", filter.filterAllRemaining());
    assertFalse("regexFilterNotNull", filter.filterRow());
}