Examples with ColumnFamilyOptions org.rocksdb.ColumnFamilyOptions used on opensource projects

Example 11 with ColumnFamilyOptions

use of org.rocksdb.ColumnFamilyOptions in project kafka by apache.

the class RocksDBStore method openDB.

@SuppressWarnings("unchecked")
void openDB(final Map<String, Object> configs, final File stateDir) {
    // initialize the default rocksdb options
    final DBOptions dbOptions = new DBOptions();
    final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions();
    userSpecifiedOptions = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(dbOptions, columnFamilyOptions);
    final BlockBasedTableConfigWithAccessibleCache tableConfig = new BlockBasedTableConfigWithAccessibleCache();
    cache = new LRUCache(BLOCK_CACHE_SIZE);
    tableConfig.setBlockCache(cache);
    tableConfig.setBlockSize(BLOCK_SIZE);
    filter = new BloomFilter();
    tableConfig.setFilterPolicy(filter);
    userSpecifiedOptions.optimizeFiltersForHits();
    userSpecifiedOptions.setTableFormatConfig(tableConfig);
    userSpecifiedOptions.setWriteBufferSize(WRITE_BUFFER_SIZE);
    userSpecifiedOptions.setCompressionType(COMPRESSION_TYPE);
    userSpecifiedOptions.setCompactionStyle(COMPACTION_STYLE);
    userSpecifiedOptions.setMaxWriteBufferNumber(MAX_WRITE_BUFFERS);
    userSpecifiedOptions.setCreateIfMissing(true);
    userSpecifiedOptions.setErrorIfExists(false);
    userSpecifiedOptions.setInfoLogLevel(InfoLogLevel.ERROR_LEVEL);
    // this is the recommended way to increase parallelism in RocksDb
    // note that the current implementation of setIncreaseParallelism affects the number
    // of compaction threads but not flush threads (the latter remains one). Also,
    // the parallelism value needs to be at least two because of the code in
    // https://github.com/facebook/rocksdb/blob/62ad0a9b19f0be4cefa70b6b32876e764b7f3c11/util/options.cc#L580
    // subtracts one from the value passed to determine the number of compaction threads
    // (this could be a bug in the RocksDB code and their devs have been contacted).
    userSpecifiedOptions.setIncreaseParallelism(Math.max(Runtime.getRuntime().availableProcessors(), 2));
    wOptions = new WriteOptions();
    wOptions.setDisableWAL(true);
    fOptions = new FlushOptions();
    fOptions.setWaitForFlush(true);
    final Class<RocksDBConfigSetter> configSetterClass = (Class<RocksDBConfigSetter>) configs.get(StreamsConfig.ROCKSDB_CONFIG_SETTER_CLASS_CONFIG);
    if (configSetterClass != null) {
        configSetter = Utils.newInstance(configSetterClass);
        configSetter.setConfig(name, userSpecifiedOptions, configs);
    }
    dbDir = new File(new File(stateDir, parentDir), name);
    try {
        Files.createDirectories(dbDir.getParentFile().toPath());
        Files.createDirectories(dbDir.getAbsoluteFile().toPath());
    } catch (final IOException fatal) {
        throw new ProcessorStateException(fatal);
    }
    // Setup statistics before the database is opened, otherwise the statistics are not updated
    // with the measurements from Rocks DB
    maybeSetUpStatistics(configs);
    openRocksDB(dbOptions, columnFamilyOptions);
    open = true;
    addValueProvidersToMetricsRecorder();
}

Example 12 with ColumnFamilyOptions

use of org.rocksdb.ColumnFamilyOptions in project voldemort by voldemort.

the class RocksDbStorageConfiguration method getStore.

@Override
public StorageEngine<ByteArray, byte[], byte[]> getStore(StoreDefinition storeDef, RoutingStrategy strategy) {
    String storeName = storeDef.getName();
    if (!stores.containsKey(storeName)) {
        String dataDir = this.voldemortconfig.getRdbDataDirectory() + "/" + storeName;
        new File(dataDir).mkdirs();
        Properties dbProperties = parseProperties(VoldemortConfig.ROCKSDB_DB_OPTIONS);
        DBOptions dbOptions = (dbProperties.size() > 0) ? DBOptions.getDBOptionsFromProps(dbProperties) : new DBOptions();
        if (dbOptions == null) {
            throw new StorageInitializationException("Unable to parse Data Base Options.");
        }
        dbOptions.setCreateIfMissing(true);
        dbOptions.setCreateMissingColumnFamilies(true);
        dbOptions.createStatistics();
        Properties cfProperties = parseProperties(VoldemortConfig.ROCKSDB_CF_OPTIONS);
        if (this.voldemortconfig.getRocksdbPrefixKeysWithPartitionId()) {
            cfProperties.setProperty("prefix_extractor", "fixed:" + StoreBinaryFormat.PARTITIONID_PREFIX_SIZE);
        }
        ColumnFamilyOptions cfOptions = (cfProperties.size() > 0) ? ColumnFamilyOptions.getColumnFamilyOptionsFromProps(cfProperties) : new ColumnFamilyOptions();
        if (cfOptions == null) {
            throw new StorageInitializationException("Unable to parse Column Family Options.");
        }
        // Create a non default Column Family tp hold the store data.
        List<ColumnFamilyDescriptor> descriptors = new ArrayList<ColumnFamilyDescriptor>();
        descriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions));
        descriptors.add(new ColumnFamilyDescriptor(storeName.getBytes(), cfOptions));
        List<ColumnFamilyHandle> handles = new ArrayList<ColumnFamilyHandle>();
        try {
            RocksDB rdbStore = RocksDB.open(dbOptions, dataDir, descriptors, handles);
            // Dispose of the default Column Family immediately.  We don't use it and if it has not been disposed
            // by the time the DB is closed then the RocksDB code can terminate abnormally (if the RocksDB code is
            // built with assertions enabled). The handle will go out of scope on its own and the Java finalizer
            // will (eventually) do this for us, but, that is not fast enough for the unit tests.
            handles.get(0).dispose();
            ColumnFamilyHandle storeHandle = handles.get(1);
            RocksDbStorageEngine rdbStorageEngine;
            if (this.voldemortconfig.getRocksdbPrefixKeysWithPartitionId()) {
                rdbStorageEngine = new PartitionPrefixedRocksDbStorageEngine(storeName, rdbStore, storeHandle, cfOptions, lockStripes, strategy, voldemortconfig.isRocksdbEnableReadLocks());
            } else {
                rdbStorageEngine = new RocksDbStorageEngine(storeName, rdbStore, storeHandle, cfOptions, lockStripes, voldemortconfig.isRocksdbEnableReadLocks());
            }
            stores.put(storeName, rdbStorageEngine);
        } catch (Exception e) {
            throw new StorageInitializationException(e);
        }
    }
    return stores.get(storeName);
}

Example 13 with ColumnFamilyOptions

use of org.rocksdb.ColumnFamilyOptions in project jstorm by alibaba.

the class RocksDbHdfsState method initRocksDb.

protected void initRocksDb() {
    RocksDbOptionsFactory optionFactory = new RocksDbOptionsFactory.Defaults();
    Options options = optionFactory.createOptions(null);
    DBOptions dbOptions = optionFactory.createDbOptions(null);
    ColumnFamilyOptions cfOptions = optionFactory.createColumnFamilyOptions(null);
    String optionsFactoryClass = (String) conf.get(ConfigExtension.ROCKSDB_OPTIONS_FACTORY_CLASS);
    if (optionsFactoryClass != null) {
        RocksDbOptionsFactory udfOptionFactory = (RocksDbOptionsFactory) Utils.newInstance(optionsFactoryClass);
        options = udfOptionFactory.createOptions(options);
        dbOptions = udfOptionFactory.createDbOptions(dbOptions);
        cfOptions = udfOptionFactory.createColumnFamilyOptions(cfOptions);
    }
    try {
        ttlTimeSec = ConfigExtension.getStateTtlTime(conf);
        if (ttlTimeSec > 0)
            rocksDb = TtlDB.open(options, rocksDbDir, ttlTimeSec, false);
        else
            rocksDb = RocksDB.open(options, rocksDbDir);
        // enable compaction
        rocksDb.compactRange();
        LOG.info("Finish the initialization of RocksDB");
    } catch (RocksDBException e) {
        LOG.error("Failed to open rocksdb located at " + rocksDbDir, e);
        throw new RuntimeException(e.getMessage());
    }
    lastCheckpointFiles = new HashSet<String>();
    lastCleanTime = System.currentTimeMillis();
    lastSuccessBatchId = -1;
}

Example 14 with ColumnFamilyOptions

use of org.rocksdb.ColumnFamilyOptions in project jstorm by alibaba.

the class WindowedRocksDbHdfsState method initRocksDb.

@Override
protected void initRocksDb() {
    windowToCFHandler = new HashMap<>();
    RocksDbOptionsFactory optionFactory = new RocksDbOptionsFactory.Defaults();
    Options options = optionFactory.createOptions(null);
    DBOptions dbOptions = optionFactory.createDbOptions(null);
    ColumnFamilyOptions cfOptions = optionFactory.createColumnFamilyOptions(null);
    String optionsFactoryClass = (String) conf.get(ConfigExtension.ROCKSDB_OPTIONS_FACTORY_CLASS);
    if (optionsFactoryClass != null) {
        RocksDbOptionsFactory udfOptionFactory = (RocksDbOptionsFactory) Utils.newInstance(optionsFactoryClass);
        options = udfOptionFactory.createOptions(options);
        dbOptions = udfOptionFactory.createDbOptions(dbOptions);
        cfOptions = udfOptionFactory.createColumnFamilyOptions(cfOptions);
    }
    try {
        ttlTimeSec = ConfigExtension.getStateTtlTime(conf);
        List<Integer> ttlValues = new ArrayList<>();
        List<byte[]> families = RocksDB.listColumnFamilies(options, rocksDbDir);
        List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
        List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
        if (families != null) {
            for (byte[] bytes : families) {
                columnFamilyDescriptors.add(new ColumnFamilyDescriptor(bytes, cfOptions));
                LOG.debug("Load colum family of {}", new String(bytes));
                if (ttlTimeSec > 0)
                    ttlValues.add(ttlTimeSec);
            }
        }
        if (columnFamilyDescriptors.size() > 0) {
            if (ttlTimeSec > 0)
                rocksDb = TtlDB.open(dbOptions, rocksDbDir, columnFamilyDescriptors, columnFamilyHandles, ttlValues, false);
            else
                rocksDb = RocksDB.open(dbOptions, rocksDbDir, columnFamilyDescriptors, columnFamilyHandles);
            int n = Math.min(columnFamilyDescriptors.size(), columnFamilyHandles.size());
            LOG.info("Try to load RocksDB with column family, desc_num={}, handler_num={}", columnFamilyDescriptors.size(), columnFamilyHandles.size());
            // skip default column
            for (int i = 1; i < n; i++) {
                windowToCFHandler.put((TimeWindow) serializer.deserialize(columnFamilyDescriptors.get(i).columnFamilyName()), columnFamilyHandles.get(i));
            }
        } else {
            rocksDb = RocksDB.open(options, rocksDbDir);
        }
        rocksDb.compactRange();
        LOG.info("Finish the initialization of RocksDB");
    } catch (RocksDBException e) {
        LOG.error("Failed to open rocksdb located at " + rocksDbDir, e);
        throw new RuntimeException(e.getMessage());
    }
    lastCheckpointFiles = new HashSet<String>();
    lastCleanTime = System.currentTimeMillis();
    lastSuccessBatchId = -1;
}

Example 15 with ColumnFamilyOptions

use of org.rocksdb.ColumnFamilyOptions in project flink by apache.

the class KVStateRequestSerializerRocksDBTest method testMapSerialization.

/**
	 * Tests map serialization and deserialization match.
	 *
	 * @see KvStateRequestSerializerTest#testMapSerialization()
	 * KvStateRequestSerializerTest#testMapSerialization() using the heap state back-end
	 * test
	 */
@Test
public void testMapSerialization() throws Exception {
    final long key = 0L;
    // objects for RocksDB state list serialisation
    DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
    dbOptions.setCreateIfMissing(true);
    ColumnFamilyOptions columnFamilyOptions = PredefinedOptions.DEFAULT.createColumnOptions();
    final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend = new RocksDBKeyedStateBackend<>(new JobID(), "no-op", ClassLoader.getSystemClassLoader(), temporaryFolder.getRoot(), dbOptions, columnFamilyOptions, mock(TaskKvStateRegistry.class), LongSerializer.INSTANCE, 1, new KeyGroupRange(0, 0), new ExecutionConfig());
    longHeapKeyedStateBackend.setCurrentKey(key);
    final InternalMapState<VoidNamespace, Long, String> mapState = (InternalMapState<VoidNamespace, Long, String>) longHeapKeyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));
    KvStateRequestSerializerTest.testMapSerialization(key, mapState);
}