Examples with HFileContextBuilder org.apache.hadoop.hbase.io.hfile.HFileContextBuilder used on opensource projects

Example 21 with HFileContextBuilder

use of org.apache.hadoop.hbase.io.hfile.HFileContextBuilder in project hbase by apache.

the class TestRAMCache method testAtomicRAMCache.

@Test
public void testAtomicRAMCache() throws Exception {
    int size = 100;
    int length = HConstants.HFILEBLOCK_HEADER_SIZE + size;
    byte[] byteArr = new byte[length];
    RAMCache cache = new RAMCache();
    BlockCacheKey key = new BlockCacheKey("file-1", 1);
    MockHFileBlock blk = new MockHFileBlock(BlockType.DATA, size, size, -1, ByteBuffer.wrap(byteArr, 0, size), HFileBlock.FILL_HEADER, -1, 52, -1, new HFileContextBuilder().build(), ByteBuffAllocator.HEAP);
    RAMQueueEntry re = new RAMQueueEntry(key, blk, 1, false);
    Assert.assertNull(cache.putIfAbsent(key, re));
    Assert.assertEquals(cache.putIfAbsent(key, re), re);
    CountDownLatch latch = new CountDownLatch(1);
    blk.setLatch(latch);
    AtomicBoolean error = new AtomicBoolean(false);
    Thread t1 = new Thread(() -> {
        try {
            cache.get(key);
        } catch (Exception e) {
            error.set(true);
        }
    });
    t1.start();
    Thread.sleep(200);
    AtomicBoolean removed = new AtomicBoolean(false);
    Thread t2 = new Thread(() -> {
        cache.remove(key);
        removed.set(true);
    });
    t2.start();
    Thread.sleep(200);
    Assert.assertFalse(removed.get());
    latch.countDown();
    Thread.sleep(200);
    Assert.assertTrue(removed.get());
    Assert.assertFalse(error.get());
}

Example 22 with HFileContextBuilder

use of org.apache.hadoop.hbase.io.hfile.HFileContextBuilder in project hbase by apache.

the class TestBulkLoadReplication method createHFileForFamilies.

private String createHFileForFamilies(byte[] row, byte[] value, Configuration clusterConfig) throws IOException {
    CellBuilder cellBuilder = CellBuilderFactory.create(CellBuilderType.DEEP_COPY);
    cellBuilder.setRow(row).setFamily(TestReplicationBase.famName).setQualifier(Bytes.toBytes("1")).setValue(value).setType(Cell.Type.Put);
    HFile.WriterFactory hFileFactory = HFile.getWriterFactoryNoCache(clusterConfig);
    // TODO We need a way to do this without creating files
    File hFileLocation = testFolder.newFile();
    FSDataOutputStream out = new FSDataOutputStream(new FileOutputStream(hFileLocation), null);
    try {
        hFileFactory.withOutputStream(out);
        hFileFactory.withFileContext(new HFileContextBuilder().build());
        HFile.Writer writer = hFileFactory.create();
        try {
            writer.append(new KeyValue(cellBuilder.build()));
        } finally {
            writer.close();
        }
    } finally {
        out.close();
    }
    return hFileLocation.getAbsoluteFile().getAbsolutePath();
}

Example 23 with HFileContextBuilder

use of org.apache.hadoop.hbase.io.hfile.HFileContextBuilder in project hbase by apache.

the class TestBulkloadBase method createHFileForFamilies.

private String createHFileForFamilies(byte[] family) throws IOException {
    HFile.WriterFactory hFileFactory = HFile.getWriterFactoryNoCache(conf);
    // TODO We need a way to do this without creating files
    File hFileLocation = testFolder.newFile(generateUniqueName(null));
    FSDataOutputStream out = new FSDataOutputStream(new FileOutputStream(hFileLocation), null);
    try {
        hFileFactory.withOutputStream(out);
        hFileFactory.withFileContext(new HFileContextBuilder().build());
        HFile.Writer writer = hFileFactory.create();
        try {
            writer.append(new KeyValue(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(randomBytes).setFamily(family).setQualifier(randomBytes).setTimestamp(0L).setType(KeyValue.Type.Put.getCode()).setValue(randomBytes).build()));
        } finally {
            writer.close();
        }
    } finally {
        out.close();
    }
    return hFileLocation.getAbsoluteFile().getAbsolutePath();
}

Example 24 with HFileContextBuilder

use of org.apache.hadoop.hbase.io.hfile.HFileContextBuilder in project hbase by apache.

the class DataBlockEncodingTool method checkStatistics.

/**
 * Check statistics for given HFile for different data block encoders.
 * @param scanner Of file which will be compressed.
 * @param kvLimit Maximal count of KeyValue which will be processed.
 * @throws IOException thrown if scanner is invalid
 */
public void checkStatistics(final KeyValueScanner scanner, final int kvLimit) throws IOException {
    scanner.seek(KeyValue.LOWESTKEY);
    KeyValue currentKV;
    byte[] previousKey = null;
    byte[] currentKey;
    DataBlockEncoding[] encodings = DataBlockEncoding.values();
    ByteArrayOutputStream uncompressedOutputStream = new ByteArrayOutputStream();
    int j = 0;
    while ((currentKV = KeyValueUtil.ensureKeyValue(scanner.next())) != null && j < kvLimit) {
        // Iterates through key/value pairs
        j++;
        currentKey = currentKV.getKey();
        if (previousKey != null) {
            for (int i = 0; i < previousKey.length && i < currentKey.length && previousKey[i] == currentKey[i]; ++i) {
                totalKeyRedundancyLength++;
            }
        }
        // if the cell tagsLen equals 0, it means other cells may have tags.
        if (USE_TAG && currentKV.getTagsLength() == 0) {
            uncompressedOutputStream.write(currentKV.getBuffer(), currentKV.getOffset(), currentKV.getLength());
            // write tagsLen = 0.
            uncompressedOutputStream.write(Bytes.toBytes((short) 0));
        } else {
            uncompressedOutputStream.write(currentKV.getBuffer(), currentKV.getOffset(), currentKV.getLength());
        }
        if (includesMemstoreTS) {
            WritableUtils.writeVLong(new DataOutputStream(uncompressedOutputStream), currentKV.getSequenceId());
        }
        previousKey = currentKey;
        int kLen = currentKV.getKeyLength();
        int vLen = currentKV.getValueLength();
        int cfOffset = currentKV.getFamilyOffset();
        int cfLen = currentKV.getFamilyLength();
        int restLen = currentKV.getLength() - kLen - vLen;
        totalKeyLength += kLen;
        totalValueLength += vLen;
        totalPrefixLength += restLen;
        totalCFLength += cfLen;
    }
    rawKVs = uncompressedOutputStream.toByteArray();
    for (DataBlockEncoding encoding : encodings) {
        if (encoding == DataBlockEncoding.NONE) {
            continue;
        }
        DataBlockEncoder d = encoding.getEncoder();
        HFileContext meta = new HFileContextBuilder().withDataBlockEncoding(encoding).withCompression(Compression.Algorithm.NONE).withIncludesMvcc(includesMemstoreTS).withIncludesTags(USE_TAG).build();
        codecs.add(new EncodedDataBlock(conf, d, encoding, rawKVs, meta));
    }
}

Example 25 with HFileContextBuilder

use of org.apache.hadoop.hbase.io.hfile.HFileContextBuilder in project hbase by apache.

the class TestBulkLoadReplicationHFileRefs method createHFileForFamilies.

private String createHFileForFamilies(byte[] family) throws IOException {
    CellBuilder cellBuilder = CellBuilderFactory.create(CellBuilderType.DEEP_COPY);
    cellBuilder.setRow(row).setFamily(family).setQualifier(qualifier).setValue(value).setType(Cell.Type.Put);
    HFile.WriterFactory hFileFactory = HFile.getWriterFactoryNoCache(UTIL1.getConfiguration());
    File hFileLocation = testFolder.newFile();
    FSDataOutputStream out = new FSDataOutputStream(new FileOutputStream(hFileLocation), null);
    try {
        hFileFactory.withOutputStream(out);
        hFileFactory.withFileContext(new HFileContextBuilder().build());
        HFile.Writer writer = hFileFactory.create();
        try {
            writer.append(new KeyValue(cellBuilder.build()));
        } finally {
            writer.close();
        }
    } finally {
        out.close();
    }
    return hFileLocation.getAbsoluteFile().getAbsolutePath();
}