Examples with HFileContextBuilder org.apache.hadoop.hbase.io.hfile.HFileContextBuilder used on opensource projects

Example 76 with HFileContextBuilder

use of org.apache.hadoop.hbase.io.hfile.HFileContextBuilder in project hbase by apache.

the class TestReversibleScanners method testReversibleKeyValueHeap.

@Test
public void testReversibleKeyValueHeap() throws IOException {
    // write data to one memstore and two store files
    FileSystem fs = TEST_UTIL.getTestFileSystem();
    Path hfilePath = new Path(new Path(TEST_UTIL.getDataTestDir("testReversibleKeyValueHeap"), "regionname"), "familyname");
    CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration());
    HFileContextBuilder hcBuilder = new HFileContextBuilder();
    hcBuilder.withBlockSize(2 * 1024);
    HFileContext hFileContext = hcBuilder.build();
    StoreFileWriter writer1 = new StoreFileWriter.Builder(TEST_UTIL.getConfiguration(), cacheConf, fs).withOutputDir(hfilePath).withFileContext(hFileContext).build();
    StoreFileWriter writer2 = new StoreFileWriter.Builder(TEST_UTIL.getConfiguration(), cacheConf, fs).withOutputDir(hfilePath).withFileContext(hFileContext).build();
    MemStore memstore = new DefaultMemStore();
    writeMemstoreAndStoreFiles(memstore, new StoreFileWriter[] { writer1, writer2 });
    HStoreFile sf1 = new HStoreFile(fs, writer1.getPath(), TEST_UTIL.getConfiguration(), cacheConf, BloomType.NONE, true);
    HStoreFile sf2 = new HStoreFile(fs, writer2.getPath(), TEST_UTIL.getConfiguration(), cacheConf, BloomType.NONE, true);
    /**
     * Test without MVCC
     */
    int startRowNum = ROWSIZE / 2;
    ReversedKeyValueHeap kvHeap = getReversibleKeyValueHeap(memstore, sf1, sf2, ROWS[startRowNum], MAXMVCC);
    internalTestSeekAndNextForReversibleKeyValueHeap(kvHeap, startRowNum);
    startRowNum = ROWSIZE - 1;
    kvHeap = getReversibleKeyValueHeap(memstore, sf1, sf2, HConstants.EMPTY_START_ROW, MAXMVCC);
    internalTestSeekAndNextForReversibleKeyValueHeap(kvHeap, startRowNum);
    /**
     * Test with MVCC
     */
    for (int readPoint = 0; readPoint < MAXMVCC; readPoint++) {
        LOG.info("Setting read point to " + readPoint);
        startRowNum = ROWSIZE - 1;
        kvHeap = getReversibleKeyValueHeap(memstore, sf1, sf2, HConstants.EMPTY_START_ROW, readPoint);
        for (int i = startRowNum; i >= 0; i--) {
            if (i - 2 < 0)
                break;
            i = i - 2;
            kvHeap.seekToPreviousRow(KeyValueUtil.createFirstOnRow(ROWS[i + 1]));
            Pair<Integer, Integer> nextReadableNum = getNextReadableNumWithBackwardScan(i, 0, readPoint);
            if (nextReadableNum == null)
                break;
            KeyValue expecedKey = makeKV(nextReadableNum.getFirst(), nextReadableNum.getSecond());
            assertEquals(expecedKey, kvHeap.peek());
            i = nextReadableNum.getFirst();
            int qualNum = nextReadableNum.getSecond();
            if (qualNum + 1 < QUALSIZE) {
                kvHeap.backwardSeek(makeKV(i, qualNum + 1));
                nextReadableNum = getNextReadableNumWithBackwardScan(i, qualNum + 1, readPoint);
                if (nextReadableNum == null)
                    break;
                expecedKey = makeKV(nextReadableNum.getFirst(), nextReadableNum.getSecond());
                assertEquals(expecedKey, kvHeap.peek());
                i = nextReadableNum.getFirst();
                qualNum = nextReadableNum.getSecond();
            }
            kvHeap.next();
            if (qualNum + 1 >= QUALSIZE) {
                nextReadableNum = getNextReadableNumWithBackwardScan(i - 1, 0, readPoint);
            } else {
                nextReadableNum = getNextReadableNumWithBackwardScan(i, qualNum + 1, readPoint);
            }
            if (nextReadableNum == null)
                break;
            expecedKey = makeKV(nextReadableNum.getFirst(), nextReadableNum.getSecond());
            assertEquals(expecedKey, kvHeap.peek());
            i = nextReadableNum.getFirst();
        }
    }
}

Example 77 with HFileContextBuilder

use of org.apache.hadoop.hbase.io.hfile.HFileContextBuilder in project hbase by apache.

the class TestIgnoreUnknownFamily method addStoreFileToKnownFamily.

private void addStoreFileToKnownFamily(RegionInfo region) throws IOException {
    MasterFileSystem mfs = UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
    Path regionDir = FSUtils.getRegionDirFromRootDir(CommonFSUtils.getRootDir(mfs.getConfiguration()), region);
    Path familyDir = new Path(regionDir, Bytes.toString(UNKNOWN_FAMILY));
    StoreFileWriter writer = new StoreFileWriter.Builder(mfs.getConfiguration(), mfs.getFileSystem()).withOutputDir(familyDir).withFileContext(new HFileContextBuilder().build()).build();
    writer.close();
}

Example 78 with HFileContextBuilder

use of org.apache.hadoop.hbase.io.hfile.HFileContextBuilder in project hbase by apache.

the class CreateRandomStoreFile method run.

/**
 * Runs the tools.
 *
 * @param args command-line arguments
 * @return true in case of success
 * @throws IOException
 */
public boolean run(String[] args) throws IOException {
    options.addOption(OUTPUT_DIR_OPTION, "output_dir", true, "Output directory");
    options.addOption(NUM_KV_OPTION, "num_kv", true, "Number of key/value pairs");
    options.addOption(KEY_SIZE_OPTION, "key_size", true, "Average key size");
    options.addOption(VALUE_SIZE_OPTION, "value_size", true, "Average value size");
    options.addOption(HFILE_VERSION_OPTION, "hfile_version", true, "HFile version to create");
    options.addOption(COMPRESSION_OPTION, "compression", true, " Compression type, one of " + Arrays.toString(Compression.Algorithm.values()));
    options.addOption(BLOOM_FILTER_OPTION, "bloom_filter", true, "Bloom filter type, one of " + Arrays.toString(BloomType.values()));
    options.addOption(BLOOM_FILTER_PARAM_OPTION, "bloom_param", true, "the parameter of the bloom filter");
    options.addOption(BLOCK_SIZE_OPTION, "block_size", true, "HFile block size");
    options.addOption(BLOOM_BLOCK_SIZE_OPTION, "bloom_block_size", true, "Compound Bloom filters block size");
    options.addOption(INDEX_BLOCK_SIZE_OPTION, "index_block_size", true, "Index block size");
    if (args.length == 0) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp(CreateRandomStoreFile.class.getSimpleName(), options, true);
        return false;
    }
    CommandLineParser parser = new PosixParser();
    CommandLine cmdLine;
    try {
        cmdLine = parser.parse(options, args);
    } catch (ParseException ex) {
        LOG.error(ex.toString(), ex);
        return false;
    }
    if (!cmdLine.hasOption(OUTPUT_DIR_OPTION)) {
        LOG.error("Output directory is not specified");
        return false;
    }
    if (!cmdLine.hasOption(NUM_KV_OPTION)) {
        LOG.error("The number of keys/values not specified");
        return false;
    }
    if (!cmdLine.hasOption(KEY_SIZE_OPTION)) {
        LOG.error("Key size is not specified");
        return false;
    }
    if (!cmdLine.hasOption(VALUE_SIZE_OPTION)) {
        LOG.error("Value size not specified");
        return false;
    }
    Configuration conf = HBaseConfiguration.create();
    Path outputDir = new Path(cmdLine.getOptionValue(OUTPUT_DIR_OPTION));
    long numKV = Long.parseLong(cmdLine.getOptionValue(NUM_KV_OPTION));
    configureKeyValue(numKV, Integer.parseInt(cmdLine.getOptionValue(KEY_SIZE_OPTION)), Integer.parseInt(cmdLine.getOptionValue(VALUE_SIZE_OPTION)));
    FileSystem fs = FileSystem.get(conf);
    Compression.Algorithm compr = Compression.Algorithm.NONE;
    if (cmdLine.hasOption(COMPRESSION_OPTION)) {
        compr = Compression.Algorithm.valueOf(cmdLine.getOptionValue(COMPRESSION_OPTION));
    }
    BloomType bloomType = BloomType.NONE;
    if (cmdLine.hasOption(BLOOM_FILTER_OPTION)) {
        bloomType = BloomType.valueOf(cmdLine.getOptionValue(BLOOM_FILTER_OPTION));
    }
    if (bloomType == BloomType.ROWPREFIX_FIXED_LENGTH) {
        if (!cmdLine.hasOption(BLOOM_FILTER_PARAM_OPTION)) {
            LOG.error("the parameter of bloom filter is not specified");
            return false;
        } else {
            conf.set(BloomFilterUtil.PREFIX_LENGTH_KEY, cmdLine.getOptionValue(BLOOM_FILTER_PARAM_OPTION));
        }
    }
    int blockSize = HConstants.DEFAULT_BLOCKSIZE;
    if (cmdLine.hasOption(BLOCK_SIZE_OPTION))
        blockSize = Integer.valueOf(cmdLine.getOptionValue(BLOCK_SIZE_OPTION));
    if (cmdLine.hasOption(BLOOM_BLOCK_SIZE_OPTION)) {
        conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_BLOCK_SIZE, Integer.valueOf(cmdLine.getOptionValue(BLOOM_BLOCK_SIZE_OPTION)));
    }
    if (cmdLine.hasOption(INDEX_BLOCK_SIZE_OPTION)) {
        conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, Integer.valueOf(cmdLine.getOptionValue(INDEX_BLOCK_SIZE_OPTION)));
    }
    HFileContext meta = new HFileContextBuilder().withCompression(compr).withBlockSize(blockSize).build();
    StoreFileWriter sfw = new StoreFileWriter.Builder(conf, new CacheConfig(conf), fs).withOutputDir(outputDir).withBloomType(bloomType).withMaxKeyCount(numKV).withFileContext(meta).build();
    rand = new Random();
    LOG.info("Writing " + numKV + " key/value pairs");
    for (long i = 0; i < numKV; ++i) {
        sfw.append(generateKeyValue(i));
    }
    int numMetaBlocks = rand.nextInt(10) + 1;
    LOG.info("Writing " + numMetaBlocks + " meta blocks");
    for (int metaI = 0; metaI < numMetaBlocks; ++metaI) {
        sfw.getHFileWriter().appendMetaBlock(generateString(), new BytesWritable(generateValue()));
    }
    sfw.close();
    Path storeFilePath = sfw.getPath();
    long fileSize = fs.getFileStatus(storeFilePath).getLen();
    LOG.info("Created {}, {} bytes, compression={}", storeFilePath, fileSize, compr.toString());
    return true;
}

Example 79 with HFileContextBuilder

use of org.apache.hadoop.hbase.io.hfile.HFileContextBuilder in project hbase by apache.

the class TestHStoreFile method testReferenceToHFileLink.

/**
 * This test creates an hfile and then the dir structures and files to verify that references to
 * hfilelinks (created by snapshot clones) can be properly interpreted.
 */
@Test
public void testReferenceToHFileLink() throws IOException {
    // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/
    Configuration testConf = new Configuration(this.conf);
    CommonFSUtils.setRootDir(testConf, testDir);
    // adding legal table name chars to verify regex handles it.
    RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf("_original-evil-name")).build();
    HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs, CommonFSUtils.getTableDir(testDir, hri.getTable()), hri);
    HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
    // Make a store file and write data to it. <root>/<tablename>/<rgn>/<cf>/<file>
    StoreFileWriter writer = new StoreFileWriter.Builder(testConf, cacheConf, this.fs).withFilePath(regionFs.createTempName()).withFileContext(meta).build();
    writeStoreFile(writer);
    Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath());
    // create link to store file. <root>/clone/region/<cf>/<hfile>-<region>-<table>
    RegionInfo hriClone = RegionInfoBuilder.newBuilder(TableName.valueOf("clone")).build();
    HRegionFileSystem cloneRegionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs, CommonFSUtils.getTableDir(testDir, hri.getTable()), hriClone);
    Path dstPath = cloneRegionFs.getStoreDir(TEST_FAMILY);
    HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName());
    Path linkFilePath = new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName()));
    // create splits of the link.
    // <root>/clone/splitA/<cf>/<reftohfilelink>,
    // <root>/clone/splitB/<cf>/<reftohfilelink>
    RegionInfo splitHriA = RegionInfoBuilder.newBuilder(hri.getTable()).setEndKey(SPLITKEY).build();
    RegionInfo splitHriB = RegionInfoBuilder.newBuilder(hri.getTable()).setStartKey(SPLITKEY).build();
    HStoreFile f = new HStoreFile(fs, linkFilePath, testConf, cacheConf, BloomType.NONE, true);
    f.initReader();
    // top
    Path pathA = splitStoreFile(cloneRegionFs, splitHriA, TEST_FAMILY, f, SPLITKEY, true);
    // bottom
    Path pathB = splitStoreFile(cloneRegionFs, splitHriB, TEST_FAMILY, f, SPLITKEY, false);
    f.closeStoreFile(true);
    // OK test the thing
    CommonFSUtils.logFileSystemState(fs, testDir, LOG);
    // There is a case where a file with the hfilelink pattern is actually a daughter
    // reference to a hfile link. This code in StoreFile that handles this case.
    // Try to open store file from link
    HStoreFile hsfA = new HStoreFile(this.fs, pathA, testConf, cacheConf, BloomType.NONE, true);
    hsfA.initReader();
    // Now confirm that I can read from the ref to link
    int count = 1;
    HFileScanner s = hsfA.getReader().getScanner(false, false);
    s.seekTo();
    while (s.next()) {
        count++;
    }
    // read some rows here
    assertTrue(count > 0);
    // Try to open store file from link
    HStoreFile hsfB = new HStoreFile(this.fs, pathB, testConf, cacheConf, BloomType.NONE, true);
    hsfB.initReader();
    // Now confirm that I can read from the ref to link
    HFileScanner sB = hsfB.getReader().getScanner(false, false);
    sB.seekTo();
    // count++ as seekTo() will advance the scanner
    count++;
    while (sB.next()) {
        count++;
    }
    // read the rest of the rows
    assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);
}

Example 80 with HFileContextBuilder

use of org.apache.hadoop.hbase.io.hfile.HFileContextBuilder in project hbase by apache.

the class TestHStoreFile method testBloomTypes.

@Test
public void testBloomTypes() throws Exception {
    float err = (float) 0.01;
    FileSystem fs = FileSystem.getLocal(conf);
    conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, err);
    conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true);
    int rowCount = 50;
    int colCount = 10;
    int versions = 2;
    // run once using columns and once using rows
    BloomType[] bt = { BloomType.ROWCOL, BloomType.ROW };
    int[] expKeys = { rowCount * colCount, rowCount };
    // below line deserves commentary. it is expected bloom false positives
    // column = rowCount*2*colCount inserts
    // row-level = only rowCount*2 inserts, but failures will be magnified by
    // 2nd for loop for every column (2*colCount)
    float[] expErr = { 2 * rowCount * colCount * err, 2 * rowCount * 2 * colCount * err };
    for (int x : new int[] { 0, 1 }) {
        // write the file
        Path f = new Path(ROOT_DIR, name.getMethodName() + x);
        HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).withChecksumType(CKTYPE).withBytesPerCheckSum(CKBYTES).build();
        // Make a store file and write data to it.
        StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs).withFilePath(f).withBloomType(bt[x]).withMaxKeyCount(expKeys[x]).withFileContext(meta).build();
        long now = EnvironmentEdgeManager.currentTime();
        for (int i = 0; i < rowCount * 2; i += 2) {
            // rows
            for (int j = 0; j < colCount * 2; j += 2) {
                // column qualifiers
                String row = String.format(localFormatter, i);
                String col = String.format(localFormatter, j);
                for (int k = 0; k < versions; ++k) {
                    // versions
                    KeyValue kv = new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("col" + col), now - k, Bytes.toBytes(-1L));
                    writer.append(kv);
                }
            }
        }
        writer.close();
        ReaderContext context = new ReaderContextBuilder().withFilePath(f).withFileSize(fs.getFileStatus(f).getLen()).withFileSystem(fs).withInputStreamWrapper(new FSDataInputStreamWrapper(fs, f)).build();
        HFileInfo fileInfo = new HFileInfo(context, conf);
        StoreFileReader reader = new StoreFileReader(context, fileInfo, cacheConf, new AtomicInteger(0), conf);
        fileInfo.initMetaAndIndex(reader.getHFileReader());
        reader.loadFileInfo();
        reader.loadBloomfilter();
        StoreFileScanner scanner = getStoreFileScanner(reader, false, false);
        assertEquals(expKeys[x], reader.getGeneralBloomFilter().getKeyCount());
        HStore store = mock(HStore.class);
        when(store.getColumnFamilyDescriptor()).thenReturn(ColumnFamilyDescriptorBuilder.of("family"));
        // check false positives rate
        int falsePos = 0;
        int falseNeg = 0;
        for (int i = 0; i < rowCount * 2; ++i) {
            // rows
            for (int j = 0; j < colCount * 2; ++j) {
                // column qualifiers
                String row = String.format(localFormatter, i);
                String col = String.format(localFormatter, j);
                TreeSet<byte[]> columns = new TreeSet<>(Bytes.BYTES_COMPARATOR);
                columns.add(Bytes.toBytes("col" + col));
                Scan scan = new Scan().withStartRow(Bytes.toBytes(row)).withStopRow(Bytes.toBytes(row), true);
                scan.addColumn(Bytes.toBytes("family"), Bytes.toBytes(("col" + col)));
                boolean exists = scanner.shouldUseScanner(scan, store, Long.MIN_VALUE);
                boolean shouldRowExist = i % 2 == 0;
                boolean shouldColExist = j % 2 == 0;
                shouldColExist = shouldColExist || bt[x] == BloomType.ROW;
                if (shouldRowExist && shouldColExist) {
                    if (!exists) {
                        falseNeg++;
                    }
                } else {
                    if (exists) {
                        falsePos++;
                    }
                }
            }
        }
        // evict because we are about to delete the file
        reader.close(true);
        fs.delete(f, true);
        System.out.println(bt[x].toString());
        System.out.println("  False negatives: " + falseNeg);
        System.out.println("  False positives: " + falsePos);
        assertEquals(0, falseNeg);
        assertTrue(falsePos < 2 * expErr[x]);
    }
}