Examples with CacheConfig org.apache.hadoop.hbase.io.hfile.CacheConfig used on opensource projects

Example 46 with CacheConfig

use of org.apache.hadoop.hbase.io.hfile.CacheConfig in project hbase by apache.

the class MobUtils method createWriter.

/**
   * Creates a writer for the mob file in temp directory.
   * @param conf The current configuration.
   * @param fs The current file system.
   * @param family The descriptor of the current column family.
   * @param path The path for a temp directory.
   * @param maxKeyCount The key count.
   * @param compression The compression algorithm.
   * @param cacheConfig The current cache config.
   * @param cryptoContext The encryption context.
   * @param checksumType The checksum type.
   * @param bytesPerChecksum The bytes per checksum.
   * @param blocksize The HFile block size.
   * @param bloomType The bloom filter type.
   * @param isCompaction If the writer is used in compaction.
   * @return The writer for the mob file.
   * @throws IOException
   */
public static StoreFileWriter createWriter(Configuration conf, FileSystem fs, HColumnDescriptor family, Path path, long maxKeyCount, Compression.Algorithm compression, CacheConfig cacheConfig, Encryption.Context cryptoContext, ChecksumType checksumType, int bytesPerChecksum, int blocksize, BloomType bloomType, boolean isCompaction) throws IOException {
    if (compression == null) {
        compression = HFile.DEFAULT_COMPRESSION_ALGORITHM;
    }
    final CacheConfig writerCacheConf;
    if (isCompaction) {
        writerCacheConf = new CacheConfig(cacheConfig);
        writerCacheConf.setCacheDataOnWrite(false);
    } else {
        writerCacheConf = cacheConfig;
    }
    HFileContext hFileContext = new HFileContextBuilder().withCompression(compression).withIncludesMvcc(true).withIncludesTags(true).withCompressTags(family.isCompressTags()).withChecksumType(checksumType).withBytesPerCheckSum(bytesPerChecksum).withBlockSize(blocksize).withHBaseCheckSum(true).withDataBlockEncoding(family.getDataBlockEncoding()).withEncryptionContext(cryptoContext).withCreateTime(EnvironmentEdgeManager.currentTime()).build();
    StoreFileWriter w = new StoreFileWriter.Builder(conf, writerCacheConf, fs).withFilePath(path).withComparator(CellComparator.COMPARATOR).withBloomType(bloomType).withMaxKeyCount(maxKeyCount).withFileContext(hFileContext).build();
    return w;
}

Example 47 with CacheConfig

use of org.apache.hadoop.hbase.io.hfile.CacheConfig in project hbase by apache.

the class TestHalfStoreFileReader method testHalfScanAndReseek.

/**
   * Test the scanner and reseek of a half hfile scanner. The scanner API
   * demands that seekTo and reseekTo() only return < 0 if the key lies
   * before the start of the file (with no position on the scanner). Returning
   * 0 if perfect match (rare), and return > 1 if we got an imperfect match.
   *
   * The latter case being the most common, we should generally be returning 1,
   * and if we do, there may or may not be a 'next' in the scanner/file.
   *
   * A bug in the half file scanner was returning -1 at the end of the bottom
   * half, and that was causing the infrastructure above to go null causing NPEs
   * and other problems.  This test reproduces that failure, and also tests
   * both the bottom and top of the file while we are at it.
   *
   * @throws IOException
   */
@Test
public void testHalfScanAndReseek() throws IOException {
    String root_dir = TEST_UTIL.getDataTestDir().toString();
    Path p = new Path(root_dir, "test");
    Configuration conf = TEST_UTIL.getConfiguration();
    FileSystem fs = FileSystem.get(conf);
    CacheConfig cacheConf = new CacheConfig(conf);
    HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build();
    HFile.Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(fs, p).withFileContext(meta).create();
    // write some things.
    List<KeyValue> items = genSomeKeys();
    for (KeyValue kv : items) {
        w.append(kv);
    }
    w.close();
    HFile.Reader r = HFile.createReader(fs, p, cacheConf, conf);
    r.loadFileInfo();
    Cell midKV = r.midkey();
    byte[] midkey = CellUtil.cloneRow(midKV);
    //System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));
    Reference bottom = new Reference(midkey, Reference.Range.bottom);
    doTestOfScanAndReseek(p, fs, bottom, cacheConf);
    Reference top = new Reference(midkey, Reference.Range.top);
    doTestOfScanAndReseek(p, fs, top, cacheConf);
    r.close();
}

Example 48 with CacheConfig

use of org.apache.hadoop.hbase.io.hfile.CacheConfig in project hbase by apache.

the class TestHFileOutputFormat2 method test_TIMERANGE.

/*
   * Test that {@link HFileOutputFormat2} creates an HFile with TIMERANGE
   * metadata used by time-restricted scans.
   */
@Ignore("Goes zombie too frequently; needs work. See HBASE-14563")
@Test
public void test_TIMERANGE() throws Exception {
    Configuration conf = new Configuration(this.util.getConfiguration());
    RecordWriter<ImmutableBytesWritable, Cell> writer = null;
    TaskAttemptContext context = null;
    Path dir = util.getDataTestDir("test_TIMERANGE_present");
    LOG.info("Timerange dir writing to dir: " + dir);
    try {
        // build a record writer using HFileOutputFormat2
        Job job = new Job(conf);
        FileOutputFormat.setOutputPath(job, dir);
        context = createTestTaskAttemptContext(job);
        HFileOutputFormat2 hof = new HFileOutputFormat2();
        writer = hof.getRecordWriter(context);
        // Pass two key values with explicit times stamps
        final byte[] b = Bytes.toBytes("b");
        // value 1 with timestamp 2000
        KeyValue kv = new KeyValue(b, b, b, 2000, b);
        KeyValue original = kv.clone();
        writer.write(new ImmutableBytesWritable(), kv);
        assertEquals(original, kv);
        // value 2 with timestamp 1000
        kv = new KeyValue(b, b, b, 1000, b);
        original = kv.clone();
        writer.write(new ImmutableBytesWritable(), kv);
        assertEquals(original, kv);
        // verify that the file has the proper FileInfo.
        writer.close(context);
        // the generated file lives 1 directory down from the attempt directory
        // and is the only file, e.g.
        // _attempt__0000_r_000000_0/b/1979617994050536795
        FileSystem fs = FileSystem.get(conf);
        Path attemptDirectory = hof.getDefaultWorkFile(context, "").getParent();
        FileStatus[] sub1 = fs.listStatus(attemptDirectory);
        FileStatus[] file = fs.listStatus(sub1[0].getPath());
        // open as HFile Reader and pull out TIMERANGE FileInfo.
        HFile.Reader rd = HFile.createReader(fs, file[0].getPath(), new CacheConfig(conf), conf);
        Map<byte[], byte[]> finfo = rd.loadFileInfo();
        byte[] range = finfo.get("TIMERANGE".getBytes());
        assertNotNull(range);
        // unmarshall and check values.
        TimeRangeTracker timeRangeTracker = new TimeRangeTracker();
        Writables.copyWritable(range, timeRangeTracker);
        LOG.info(timeRangeTracker.getMin() + "...." + timeRangeTracker.getMax());
        assertEquals(1000, timeRangeTracker.getMin());
        assertEquals(2000, timeRangeTracker.getMax());
        rd.close();
    } finally {
        if (writer != null && context != null)
            writer.close(context);
        dir.getFileSystem(conf).delete(dir, true);
    }
}

Example 49 with CacheConfig

use of org.apache.hadoop.hbase.io.hfile.CacheConfig in project hbase by apache.

the class TestHFileOutputFormat2 method test_WritingTagData.

/**
   * Test that {@link HFileOutputFormat2} RecordWriter writes tags such as ttl into
   * hfile.
   */
@Test
public void test_WritingTagData() throws Exception {
    Configuration conf = new Configuration(this.util.getConfiguration());
    final String HFILE_FORMAT_VERSION_CONF_KEY = "hfile.format.version";
    conf.setInt(HFILE_FORMAT_VERSION_CONF_KEY, HFile.MIN_FORMAT_VERSION_WITH_TAGS);
    RecordWriter<ImmutableBytesWritable, Cell> writer = null;
    TaskAttemptContext context = null;
    Path dir = util.getDataTestDir("WritingTagData");
    try {
        Job job = new Job(conf);
        FileOutputFormat.setOutputPath(job, dir);
        context = createTestTaskAttemptContext(job);
        HFileOutputFormat2 hof = new HFileOutputFormat2();
        writer = hof.getRecordWriter(context);
        final byte[] b = Bytes.toBytes("b");
        List<Tag> tags = new ArrayList<>();
        tags.add(new ArrayBackedTag(TagType.TTL_TAG_TYPE, Bytes.toBytes(978670)));
        KeyValue kv = new KeyValue(b, b, b, HConstants.LATEST_TIMESTAMP, b, tags);
        writer.write(new ImmutableBytesWritable(), kv);
        writer.close(context);
        writer = null;
        FileSystem fs = dir.getFileSystem(conf);
        RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(dir, true);
        while (iterator.hasNext()) {
            LocatedFileStatus keyFileStatus = iterator.next();
            HFile.Reader reader = HFile.createReader(fs, keyFileStatus.getPath(), new CacheConfig(conf), conf);
            HFileScanner scanner = reader.getScanner(false, false, false);
            scanner.seekTo();
            Cell cell = scanner.getCell();
            List<Tag> tagsFromCell = TagUtil.asList(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength());
            assertTrue(tagsFromCell.size() > 0);
            for (Tag tag : tagsFromCell) {
                assertTrue(tag.getType() == TagType.TTL_TAG_TYPE);
            }
        }
    } finally {
        if (writer != null && context != null)
            writer.close(context);
        dir.getFileSystem(conf).delete(dir, true);
    }
}

Example 50 with CacheConfig

use of org.apache.hadoop.hbase.io.hfile.CacheConfig in project hbase by apache.

the class CreateRandomStoreFile method run.

/**
   * Runs the tools.
   *
   * @param args command-line arguments
   * @return true in case of success
   * @throws IOException
   */
public boolean run(String[] args) throws IOException {
    options.addOption(OUTPUT_DIR_OPTION, "output_dir", true, "Output directory");
    options.addOption(NUM_KV_OPTION, "num_kv", true, "Number of key/value pairs");
    options.addOption(KEY_SIZE_OPTION, "key_size", true, "Average key size");
    options.addOption(VALUE_SIZE_OPTION, "value_size", true, "Average value size");
    options.addOption(HFILE_VERSION_OPTION, "hfile_version", true, "HFile version to create");
    options.addOption(COMPRESSION_OPTION, "compression", true, " Compression type, one of " + Arrays.toString(Compression.Algorithm.values()));
    options.addOption(BLOOM_FILTER_OPTION, "bloom_filter", true, "Bloom filter type, one of " + Arrays.toString(BloomType.values()));
    options.addOption(BLOCK_SIZE_OPTION, "block_size", true, "HFile block size");
    options.addOption(BLOOM_BLOCK_SIZE_OPTION, "bloom_block_size", true, "Compound Bloom filters block size");
    options.addOption(INDEX_BLOCK_SIZE_OPTION, "index_block_size", true, "Index block size");
    if (args.length == 0) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp(CreateRandomStoreFile.class.getSimpleName(), options, true);
        return false;
    }
    CommandLineParser parser = new PosixParser();
    CommandLine cmdLine;
    try {
        cmdLine = parser.parse(options, args);
    } catch (ParseException ex) {
        LOG.error(ex);
        return false;
    }
    if (!cmdLine.hasOption(OUTPUT_DIR_OPTION)) {
        LOG.error("Output directory is not specified");
        return false;
    }
    if (!cmdLine.hasOption(NUM_KV_OPTION)) {
        LOG.error("The number of keys/values not specified");
        return false;
    }
    if (!cmdLine.hasOption(KEY_SIZE_OPTION)) {
        LOG.error("Key size is not specified");
        return false;
    }
    if (!cmdLine.hasOption(VALUE_SIZE_OPTION)) {
        LOG.error("Value size not specified");
        return false;
    }
    Configuration conf = HBaseConfiguration.create();
    Path outputDir = new Path(cmdLine.getOptionValue(OUTPUT_DIR_OPTION));
    long numKV = Long.parseLong(cmdLine.getOptionValue(NUM_KV_OPTION));
    configureKeyValue(numKV, Integer.parseInt(cmdLine.getOptionValue(KEY_SIZE_OPTION)), Integer.parseInt(cmdLine.getOptionValue(VALUE_SIZE_OPTION)));
    FileSystem fs = FileSystem.get(conf);
    Compression.Algorithm compr = Compression.Algorithm.NONE;
    if (cmdLine.hasOption(COMPRESSION_OPTION)) {
        compr = Compression.Algorithm.valueOf(cmdLine.getOptionValue(COMPRESSION_OPTION));
    }
    BloomType bloomType = BloomType.NONE;
    if (cmdLine.hasOption(BLOOM_FILTER_OPTION)) {
        bloomType = BloomType.valueOf(cmdLine.getOptionValue(BLOOM_FILTER_OPTION));
    }
    int blockSize = HConstants.DEFAULT_BLOCKSIZE;
    if (cmdLine.hasOption(BLOCK_SIZE_OPTION))
        blockSize = Integer.valueOf(cmdLine.getOptionValue(BLOCK_SIZE_OPTION));
    if (cmdLine.hasOption(BLOOM_BLOCK_SIZE_OPTION)) {
        conf.setInt(BloomFilterFactory.IO_STOREFILE_BLOOM_BLOCK_SIZE, Integer.valueOf(cmdLine.getOptionValue(BLOOM_BLOCK_SIZE_OPTION)));
    }
    if (cmdLine.hasOption(INDEX_BLOCK_SIZE_OPTION)) {
        conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, Integer.valueOf(cmdLine.getOptionValue(INDEX_BLOCK_SIZE_OPTION)));
    }
    HFileContext meta = new HFileContextBuilder().withCompression(compr).withBlockSize(blockSize).build();
    StoreFileWriter sfw = new StoreFileWriter.Builder(conf, new CacheConfig(conf), fs).withOutputDir(outputDir).withBloomType(bloomType).withMaxKeyCount(numKV).withFileContext(meta).build();
    rand = new Random();
    LOG.info("Writing " + numKV + " key/value pairs");
    for (long i = 0; i < numKV; ++i) {
        sfw.append(generateKeyValue(i));
    }
    int numMetaBlocks = rand.nextInt(10) + 1;
    LOG.info("Writing " + numMetaBlocks + " meta blocks");
    for (int metaI = 0; metaI < numMetaBlocks; ++metaI) {
        sfw.getHFileWriter().appendMetaBlock(generateString(), new BytesWritable(generateValue()));
    }
    sfw.close();
    Path storeFilePath = sfw.getPath();
    long fileSize = fs.getFileStatus(storeFilePath).getLen();
    LOG.info("Created " + storeFilePath + ", " + fileSize + " bytes");
    return true;
}