Examples with HFile org.apache.hadoop.hbase.io.hfile.HFile used on opensource projects

Example 16 with HFile

use of org.apache.hadoop.hbase.io.hfile.HFile in project hbase by apache.

the class TestMobStoreCompaction method createHFile.

/**
 * Create an HFile with the given number of bytes
 */
private void createHFile(Path path, int rowIdx, byte[] dummyData) throws IOException {
    HFileContext meta = new HFileContextBuilder().build();
    HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf)).withPath(fs, path).withFileContext(meta).create();
    long now = EnvironmentEdgeManager.currentTime();
    try {
        KeyValue kv = new KeyValue(Bytes.add(STARTROW, Bytes.toBytes(rowIdx)), COLUMN_FAMILY, Bytes.toBytes("colX"), now, dummyData);
        writer.append(kv);
    } finally {
        writer.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(EnvironmentEdgeManager.currentTime()));
        writer.close();
    }
}

Example 17 with HFile

use of org.apache.hadoop.hbase.io.hfile.HFile in project hbase by apache.

the class TestHalfStoreFileReader method testHalfScanAndReseek.

/**
 * Test the scanner and reseek of a half hfile scanner. The scanner API demands that seekTo and
 * reseekTo() only return < 0 if the key lies before the start of the file (with no position on
 * the scanner). Returning 0 if perfect match (rare), and return > 1 if we got an imperfect match.
 * The latter case being the most common, we should generally be returning 1, and if we do, there
 * may or may not be a 'next' in the scanner/file. A bug in the half file scanner was returning -1
 * at the end of the bottom half, and that was causing the infrastructure above to go null causing
 * NPEs and other problems. This test reproduces that failure, and also tests both the bottom and
 * top of the file while we are at it.
 * @throws IOException
 */
@Test
public void testHalfScanAndReseek() throws IOException {
    String root_dir = TEST_UTIL.getDataTestDir().toString();
    Path p = new Path(root_dir, "test");
    Configuration conf = TEST_UTIL.getConfiguration();
    FileSystem fs = FileSystem.get(conf);
    CacheConfig cacheConf = new CacheConfig(conf);
    HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build();
    HFile.Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(fs, p).withFileContext(meta).create();
    // write some things.
    List<KeyValue> items = genSomeKeys();
    for (KeyValue kv : items) {
        w.append(kv);
    }
    w.close();
    HFile.Reader r = HFile.createReader(fs, p, cacheConf, true, conf);
    Cell midKV = r.midKey().get();
    byte[] midkey = CellUtil.cloneRow(midKV);
    // System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));
    Reference bottom = new Reference(midkey, Reference.Range.bottom);
    doTestOfScanAndReseek(p, fs, bottom, cacheConf);
    Reference top = new Reference(midkey, Reference.Range.top);
    doTestOfScanAndReseek(p, fs, top, cacheConf);
    r.close();
}

Example 18 with HFile

use of org.apache.hadoop.hbase.io.hfile.HFile in project hbase by apache.

the class HFileTestBase method doTest.

@SuppressWarnings("deprecation")
public void doTest(Configuration conf, Path path, Compression.Algorithm compression) throws Exception {
    // Create 10000 random test KVs
    RedundantKVGenerator generator = new RedundantKVGenerator();
    List<KeyValue> testKvs = generator.generateTestKeyValues(10000);
    // Iterate through data block encoding and compression combinations
    CacheConfig cacheConf = new CacheConfig(conf);
    HFileContext fileContext = new HFileContextBuilder().withBlockSize(// small block
    4096).withCompression(compression).build();
    // write a new test HFile
    LOG.info("Writing with " + fileContext);
    FSDataOutputStream out = FS.create(path);
    HFile.Writer writer = HFile.getWriterFactory(conf, cacheConf).withOutputStream(out).withFileContext(fileContext).create();
    try {
        for (KeyValue kv : testKvs) {
            writer.append(kv);
        }
    } finally {
        writer.close();
        out.close();
    }
    // read it back in
    LOG.info("Reading with " + fileContext);
    int i = 0;
    HFileScanner scanner = null;
    HFile.Reader reader = HFile.createReader(FS, path, cacheConf, true, conf);
    try {
        scanner = reader.getScanner(conf, false, false);
        assertTrue("Initial seekTo failed", scanner.seekTo());
        do {
            Cell kv = scanner.getCell();
            assertTrue("Read back an unexpected or invalid KV", testKvs.contains(KeyValueUtil.ensureKeyValue(kv)));
            i++;
        } while (scanner.next());
    } finally {
        reader.close();
        scanner.close();
    }
    assertEquals("Did not read back as many KVs as written", i, testKvs.size());
    // Test random seeks with pread
    LOG.info("Random seeking with " + fileContext);
    reader = HFile.createReader(FS, path, cacheConf, true, conf);
    try {
        scanner = reader.getScanner(conf, false, true);
        assertTrue("Initial seekTo failed", scanner.seekTo());
        for (i = 0; i < 100; i++) {
            KeyValue kv = testKvs.get(RNG.nextInt(testKvs.size()));
            assertEquals("Unable to find KV as expected: " + kv, 0, scanner.seekTo(kv));
        }
    } finally {
        scanner.close();
        reader.close();
    }
}

Example 19 with HFile

use of org.apache.hadoop.hbase.io.hfile.HFile in project hbase by apache.

the class TestHRegionServerBulkLoad method createHFile.

/**
 * Create an HFile with the given number of rows with a specified value.
 */
public static void createHFile(FileSystem fs, Path path, byte[] family, byte[] qualifier, byte[] value, int numRows) throws IOException {
    HFileContext context = new HFileContextBuilder().withBlockSize(BLOCKSIZE).withCompression(COMPRESSION).build();
    HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf)).withPath(fs, path).withFileContext(context).create();
    long now = EnvironmentEdgeManager.currentTime();
    try {
        // subtract 2 since iterateOnSplits doesn't include boundary keys
        for (int i = 0; i < numRows; i++) {
            KeyValue kv = new KeyValue(rowkey(i), family, qualifier, now, value);
            writer.append(kv);
        }
        writer.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(now));
    } finally {
        writer.close();
    }
}