Examples with BlockBasedTableConfig org.rocksdb.BlockBasedTableConfig used on opensource projects

Example 6 with BlockBasedTableConfig

use of org.rocksdb.BlockBasedTableConfig in project bookkeeper by apache.

the class RocksdbKVStore method openRocksdb.

protected void openRocksdb(StateStoreSpec spec) throws StateStoreException {
    // initialize the db options
    final BlockBasedTableConfig tableConfig = new BlockBasedTableConfig();
    tableConfig.setBlockCacheSize(BLOCK_CACHE_SIZE);
    tableConfig.setBlockSize(BLOCK_SIZE);
    tableConfig.setChecksumType(DEFAULT_CHECKSUM_TYPE);
    dbOpts = new DBOptions();
    dbOpts.setCreateIfMissing(true);
    dbOpts.setErrorIfExists(false);
    dbOpts.setInfoLogLevel(DEFAULT_LOG_LEVEL);
    dbOpts.setIncreaseParallelism(DEFAULT_PARALLELISM);
    dbOpts.setCreateMissingColumnFamilies(true);
    cfOpts = new ColumnFamilyOptions();
    cfOpts.setTableFormatConfig(tableConfig);
    cfOpts.setWriteBufferSize(WRITE_BUFFER_SIZE);
    cfOpts.setCompressionType(DEFAULT_COMPRESSION_TYPE);
    cfOpts.setCompactionStyle(DEFAULT_COMPACTION_STYLE);
    cfOpts.setMaxWriteBufferNumber(MAX_WRITE_BUFFERS);
    // initialize the write options
    writeOpts = new WriteOptions();
    // disable wal, since the source of truth will be on distributedlog
    writeOpts.setDisableWAL(false);
    // initialize the flush options
    flushOpts = new FlushOptions();
    flushOpts.setWaitForFlush(true);
    // open the rocksdb
    this.dbDir = spec.getLocalStateStoreDir();
    Pair<RocksDB, List<ColumnFamilyHandle>> dbPair = openLocalDB(dbDir, dbOpts, cfOpts);
    this.db = dbPair.getLeft();
    this.metaCfHandle = dbPair.getRight().get(0);
    this.dataCfHandle = dbPair.getRight().get(1);
}

Example 7 with BlockBasedTableConfig

use of org.rocksdb.BlockBasedTableConfig in project storm by apache.

the class RocksDbStore method prepare.

/**
 * Create metric store instance using the configurations provided via the config map.
 *
 * @param config Storm config map
 * @param metricsRegistry The Nimbus daemon metrics registry
 * @throws MetricException on preparation error
 */
@Override
public void prepare(Map<String, Object> config, StormMetricsRegistry metricsRegistry) throws MetricException {
    validateConfig(config);
    this.failureMeter = metricsRegistry.registerMeter("RocksDB:metric-failures");
    RocksDB.loadLibrary();
    boolean createIfMissing = ObjectReader.getBoolean(config.get(DaemonConfig.STORM_ROCKSDB_CREATE_IF_MISSING), false);
    try (Options options = new Options().setCreateIfMissing(createIfMissing)) {
        // use the hash index for prefix searches
        BlockBasedTableConfig tfc = new BlockBasedTableConfig();
        tfc.setIndexType(IndexType.kHashSearch);
        options.setTableFormatConfig(tfc);
        options.useCappedPrefixExtractor(RocksDbKey.KEY_SIZE);
        String path = getRocksDbAbsoluteDir(config);
        LOG.info("Opening RocksDB from {}", path);
        db = RocksDB.open(options, path);
    } catch (RocksDBException e) {
        String message = "Error opening RockDB database";
        LOG.error(message, e);
        throw new MetricException(message, e);
    }
    // create thread to delete old metrics and metadata
    Integer retentionHours = Integer.parseInt(config.get(DaemonConfig.STORM_ROCKSDB_METRIC_RETENTION_HOURS).toString());
    Integer deletionPeriod = 0;
    if (config.containsKey(DaemonConfig.STORM_ROCKSDB_METRIC_DELETION_PERIOD_HOURS)) {
        deletionPeriod = Integer.parseInt(config.get(DaemonConfig.STORM_ROCKSDB_METRIC_DELETION_PERIOD_HOURS).toString());
    }
    metricsCleaner = new MetricsCleaner(this, retentionHours, deletionPeriod, failureMeter, metricsRegistry);
    // create thread to process insertion of all metrics
    metricsWriter = new RocksDbMetricsWriter(this, this.queue, this.failureMeter);
    int cacheCapacity = Integer.parseInt(config.get(DaemonConfig.STORM_ROCKSDB_METADATA_STRING_CACHE_CAPACITY).toString());
    StringMetadataCache.init(metricsWriter, cacheCapacity);
    readOnlyStringMetadataCache = StringMetadataCache.getReadOnlyStringMetadataCache();
    // init the writer once the cache is setup
    metricsWriter.init();
    // start threads after metadata cache created
    Thread thread = new Thread(metricsCleaner, "RocksDbMetricsCleaner");
    thread.setDaemon(true);
    thread.start();
    thread = new Thread(metricsWriter, "RocksDbMetricsWriter");
    thread.setDaemon(true);
    thread.start();
}

Example 8 with BlockBasedTableConfig

use of org.rocksdb.BlockBasedTableConfig in project samza by apache.

the class RocksDbOptionsHelper method options.

public static Options options(Config storeConfig, int numTasksForContainer, File storeDir, StorageEngineFactory.StoreMode storeMode) {
    Options options = new Options();
    if (storeConfig.getBoolean(ROCKSDB_WAL_ENABLED, false)) {
        // store.flush() will flushWAL(sync = true) instead
        options.setManualWalFlush(true);
        options.setWalRecoveryMode(WALRecoveryMode.AbsoluteConsistency);
    }
    Long writeBufSize = storeConfig.getLong("container.write.buffer.size.bytes", 32 * 1024 * 1024);
    // Cache size and write buffer size are specified on a per-container basis.
    options.setWriteBufferSize((int) (writeBufSize / numTasksForContainer));
    CompressionType compressionType = CompressionType.SNAPPY_COMPRESSION;
    String compressionInConfig = storeConfig.get(ROCKSDB_COMPRESSION, "snappy");
    switch(compressionInConfig) {
        case "snappy":
            compressionType = CompressionType.SNAPPY_COMPRESSION;
            break;
        case "bzip2":
            compressionType = CompressionType.BZLIB2_COMPRESSION;
            break;
        case "zlib":
            compressionType = CompressionType.ZLIB_COMPRESSION;
            break;
        case "lz4":
            compressionType = CompressionType.LZ4_COMPRESSION;
            break;
        case "lz4hc":
            compressionType = CompressionType.LZ4HC_COMPRESSION;
            break;
        case "none":
            compressionType = CompressionType.NO_COMPRESSION;
            break;
        default:
            log.warn("Unknown rocksdb.compression codec " + compressionInConfig + ", overwriting to " + compressionType.name());
    }
    options.setCompressionType(compressionType);
    long blockCacheSize = getBlockCacheSize(storeConfig, numTasksForContainer);
    int blockSize = storeConfig.getInt(ROCKSDB_BLOCK_SIZE_BYTES, 4096);
    BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
    tableOptions.setBlockCacheSize(blockCacheSize).setBlockSize(blockSize);
    options.setTableFormatConfig(tableOptions);
    setCompactionOptions(storeConfig, options);
    options.setMaxWriteBufferNumber(storeConfig.getInt(ROCKSDB_NUM_WRITE_BUFFERS, 3));
    options.setCreateIfMissing(true);
    options.setErrorIfExists(false);
    options.setMaxLogFileSize(storeConfig.getLong(ROCKSDB_MAX_LOG_FILE_SIZE_BYTES, 64 * 1024 * 1024L));
    options.setKeepLogFileNum(storeConfig.getLong(ROCKSDB_KEEP_LOG_FILE_NUM, 2));
    options.setDeleteObsoleteFilesPeriodMicros(storeConfig.getLong(ROCKSDB_DELETE_OBSOLETE_FILES_PERIOD_MICROS, 21600000000L));
    options.setMaxOpenFiles(storeConfig.getInt(ROCKSDB_MAX_OPEN_FILES, -1));
    options.setMaxFileOpeningThreads(storeConfig.getInt(ROCKSDB_MAX_FILE_OPENING_THREADS, 16));
    // The default for rocksdb is 18446744073709551615, which is larger than java Long.MAX_VALUE. Hence setting it only if it's passed.
    if (storeConfig.containsKey(ROCKSDB_MAX_MANIFEST_FILE_SIZE)) {
        options.setMaxManifestFileSize(storeConfig.getLong(ROCKSDB_MAX_MANIFEST_FILE_SIZE));
    }
    // use prepareForBulk load only when i. the store is being requested in BulkLoad mode
    // and ii. the storeDirectory does not exist (fresh restore), because bulk load does not work seamlessly with
    // existing stores : https://github.com/facebook/rocksdb/issues/2734
    StorageManagerUtil storageManagerUtil = new StorageManagerUtil();
    if (storeMode.equals(StorageEngineFactory.StoreMode.BulkLoad) && !storageManagerUtil.storeExists(storeDir)) {
        log.info("Using prepareForBulkLoad for restore to " + storeDir);
        options.prepareForBulkLoad();
    }
    return options;
}

Example 9 with BlockBasedTableConfig

use of org.rocksdb.BlockBasedTableConfig in project kafka by apache.

the class RocksDBStore method addValueProvidersToMetricsRecorder.

private void addValueProvidersToMetricsRecorder() {
    final TableFormatConfig tableFormatConfig = userSpecifiedOptions.tableFormatConfig();
    final Statistics statistics = userSpecifiedStatistics ? null : userSpecifiedOptions.statistics();
    if (tableFormatConfig instanceof BlockBasedTableConfigWithAccessibleCache) {
        final Cache cache = ((BlockBasedTableConfigWithAccessibleCache) tableFormatConfig).blockCache();
        metricsRecorder.addValueProviders(name, db, cache, statistics);
    } else if (tableFormatConfig instanceof BlockBasedTableConfig) {
        throw new ProcessorStateException("The used block-based table format configuration does not expose the " + "block cache. Use the BlockBasedTableConfig instance provided by Options#tableFormatConfig() to configure " + "the block-based table format of RocksDB. Do not provide a new instance of BlockBasedTableConfig to " + "the RocksDB options.");
    } else {
        metricsRecorder.addValueProviders(name, db, null, statistics);
    }
}

Example 10 with BlockBasedTableConfig

use of org.rocksdb.BlockBasedTableConfig in project flink by apache.

the class DefaultConfigurableOptionsFactory method createColumnOptions.

@Override
public ColumnFamilyOptions createColumnOptions(ColumnFamilyOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
    if (isOptionConfigured(COMPACTION_STYLE)) {
        currentOptions.setCompactionStyle(getCompactionStyle());
    }
    if (isOptionConfigured(USE_DYNAMIC_LEVEL_SIZE)) {
        currentOptions.setLevelCompactionDynamicLevelBytes(getUseDynamicLevelSize());
    }
    if (isOptionConfigured(TARGET_FILE_SIZE_BASE)) {
        currentOptions.setTargetFileSizeBase(getTargetFileSizeBase());
    }
    if (isOptionConfigured(MAX_SIZE_LEVEL_BASE)) {
        currentOptions.setMaxBytesForLevelBase(getMaxSizeLevelBase());
    }
    if (isOptionConfigured(WRITE_BUFFER_SIZE)) {
        currentOptions.setWriteBufferSize(getWriteBufferSize());
    }
    if (isOptionConfigured(MAX_WRITE_BUFFER_NUMBER)) {
        currentOptions.setMaxWriteBufferNumber(getMaxWriteBufferNumber());
    }
    if (isOptionConfigured(MIN_WRITE_BUFFER_NUMBER_TO_MERGE)) {
        currentOptions.setMinWriteBufferNumberToMerge(getMinWriteBufferNumberToMerge());
    }
    TableFormatConfig tableFormatConfig = currentOptions.tableFormatConfig();
    BlockBasedTableConfig blockBasedTableConfig;
    if (tableFormatConfig == null) {
        blockBasedTableConfig = new BlockBasedTableConfig();
    } else {
        if (tableFormatConfig instanceof PlainTableConfig) {
            // column-family options
            return currentOptions;
        } else {
            blockBasedTableConfig = (BlockBasedTableConfig) tableFormatConfig;
        }
    }
    if (isOptionConfigured(BLOCK_SIZE)) {
        blockBasedTableConfig.setBlockSize(getBlockSize());
    }
    if (isOptionConfigured(METADATA_BLOCK_SIZE)) {
        blockBasedTableConfig.setMetadataBlockSize(getMetadataBlockSize());
    }
    if (isOptionConfigured(BLOCK_CACHE_SIZE)) {
        blockBasedTableConfig.setBlockCacheSize(getBlockCacheSize());
    }
    if (isOptionConfigured(USE_BLOOM_FILTER)) {
        final boolean enabled = Boolean.parseBoolean(getInternal(USE_BLOOM_FILTER.key()));
        if (enabled) {
            final double bitsPerKey = isOptionConfigured(BLOOM_FILTER_BITS_PER_KEY) ? Double.parseDouble(getInternal(BLOOM_FILTER_BITS_PER_KEY.key())) : BLOOM_FILTER_BITS_PER_KEY.defaultValue();
            final boolean blockBasedMode = isOptionConfigured(BLOOM_FILTER_BLOCK_BASED_MODE) ? Boolean.parseBoolean(getInternal(BLOOM_FILTER_BLOCK_BASED_MODE.key())) : BLOOM_FILTER_BLOCK_BASED_MODE.defaultValue();
            BloomFilter bloomFilter = new BloomFilter(bitsPerKey, blockBasedMode);
            handlesToClose.add(bloomFilter);
            blockBasedTableConfig.setFilterPolicy(bloomFilter);
        }
    }
    return currentOptions.setTableFormatConfig(blockBasedTableConfig);
}