Search in sources :

Example 6 with Options

use of org.rocksdb.Options in project samza by apache.

the class TestKeyValueSizeHistogramMetric method setup.

@Before
public void setup() {
    Config config = new MapConfig();
    Options options = new Options();
    options.setCreateIfMissing(true);
    File dbDir = new File(System.getProperty("java.io.tmpdir") + "/dbStore" + System.currentTimeMillis());
    RocksDbKeyValueStore kvStore = new RocksDbKeyValueStore(dbDir, options, config, false, "dbStore", new WriteOptions(), new FlushOptions(), new KeyValueStoreMetrics("dbStore", new MetricsRegistryMap()));
    KeyValueStore<String, String> serializedStore = new SerializedKeyValueStore<>(kvStore, stringSerde, stringSerde, serializedKeyValueStoreMetrics);
    store = new NullSafeKeyValueStore<>(serializedStore);
}
Also used : FlushOptions(org.rocksdb.FlushOptions) WriteOptions(org.rocksdb.WriteOptions) Options(org.rocksdb.Options) Config(org.apache.samza.config.Config) MapConfig(org.apache.samza.config.MapConfig) FlushOptions(org.rocksdb.FlushOptions) WriteOptions(org.rocksdb.WriteOptions) MapConfig(org.apache.samza.config.MapConfig) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) File(java.io.File) Before(org.junit.Before)

Example 7 with Options

use of org.rocksdb.Options in project jstorm by alibaba.

the class RocksDbHdfsState method initRocksDb.

protected void initRocksDb() {
    RocksDbOptionsFactory optionFactory = new RocksDbOptionsFactory.Defaults();
    Options options = optionFactory.createOptions(null);
    DBOptions dbOptions = optionFactory.createDbOptions(null);
    ColumnFamilyOptions cfOptions = optionFactory.createColumnFamilyOptions(null);
    String optionsFactoryClass = (String) conf.get(ConfigExtension.ROCKSDB_OPTIONS_FACTORY_CLASS);
    if (optionsFactoryClass != null) {
        RocksDbOptionsFactory udfOptionFactory = (RocksDbOptionsFactory) Utils.newInstance(optionsFactoryClass);
        options = udfOptionFactory.createOptions(options);
        dbOptions = udfOptionFactory.createDbOptions(dbOptions);
        cfOptions = udfOptionFactory.createColumnFamilyOptions(cfOptions);
    }
    try {
        ttlTimeSec = ConfigExtension.getStateTtlTime(conf);
        if (ttlTimeSec > 0)
            rocksDb = TtlDB.open(options, rocksDbDir, ttlTimeSec, false);
        else
            rocksDb = RocksDB.open(options, rocksDbDir);
        // enable compaction
        rocksDb.compactRange();
        LOG.info("Finish the initialization of RocksDB");
    } catch (RocksDBException e) {
        LOG.error("Failed to open rocksdb located at " + rocksDbDir, e);
        throw new RuntimeException(e.getMessage());
    }
    lastCheckpointFiles = new HashSet<String>();
    lastCleanTime = System.currentTimeMillis();
    lastSuccessBatchId = -1;
}
Also used : ColumnFamilyOptions(org.rocksdb.ColumnFamilyOptions) FlushOptions(org.rocksdb.FlushOptions) ColumnFamilyOptions(org.rocksdb.ColumnFamilyOptions) DBOptions(org.rocksdb.DBOptions) WriteOptions(org.rocksdb.WriteOptions) Options(org.rocksdb.Options) RocksDBException(org.rocksdb.RocksDBException) RocksDbOptionsFactory(com.alibaba.jstorm.cache.rocksdb.RocksDbOptionsFactory) DBOptions(org.rocksdb.DBOptions)

Example 8 with Options

use of org.rocksdb.Options in project jstorm by alibaba.

the class WindowedRocksDbHdfsState method initRocksDb.

@Override
protected void initRocksDb() {
    windowToCFHandler = new HashMap<>();
    RocksDbOptionsFactory optionFactory = new RocksDbOptionsFactory.Defaults();
    Options options = optionFactory.createOptions(null);
    DBOptions dbOptions = optionFactory.createDbOptions(null);
    ColumnFamilyOptions cfOptions = optionFactory.createColumnFamilyOptions(null);
    String optionsFactoryClass = (String) conf.get(ConfigExtension.ROCKSDB_OPTIONS_FACTORY_CLASS);
    if (optionsFactoryClass != null) {
        RocksDbOptionsFactory udfOptionFactory = (RocksDbOptionsFactory) Utils.newInstance(optionsFactoryClass);
        options = udfOptionFactory.createOptions(options);
        dbOptions = udfOptionFactory.createDbOptions(dbOptions);
        cfOptions = udfOptionFactory.createColumnFamilyOptions(cfOptions);
    }
    try {
        ttlTimeSec = ConfigExtension.getStateTtlTime(conf);
        List<Integer> ttlValues = new ArrayList<>();
        List<byte[]> families = RocksDB.listColumnFamilies(options, rocksDbDir);
        List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
        List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
        if (families != null) {
            for (byte[] bytes : families) {
                columnFamilyDescriptors.add(new ColumnFamilyDescriptor(bytes, cfOptions));
                LOG.debug("Load colum family of {}", new String(bytes));
                if (ttlTimeSec > 0)
                    ttlValues.add(ttlTimeSec);
            }
        }
        if (columnFamilyDescriptors.size() > 0) {
            if (ttlTimeSec > 0)
                rocksDb = TtlDB.open(dbOptions, rocksDbDir, columnFamilyDescriptors, columnFamilyHandles, ttlValues, false);
            else
                rocksDb = RocksDB.open(dbOptions, rocksDbDir, columnFamilyDescriptors, columnFamilyHandles);
            int n = Math.min(columnFamilyDescriptors.size(), columnFamilyHandles.size());
            LOG.info("Try to load RocksDB with column family, desc_num={}, handler_num={}", columnFamilyDescriptors.size(), columnFamilyHandles.size());
            // skip default column
            for (int i = 1; i < n; i++) {
                windowToCFHandler.put((TimeWindow) serializer.deserialize(columnFamilyDescriptors.get(i).columnFamilyName()), columnFamilyHandles.get(i));
            }
        } else {
            rocksDb = RocksDB.open(options, rocksDbDir);
        }
        rocksDb.compactRange();
        LOG.info("Finish the initialization of RocksDB");
    } catch (RocksDBException e) {
        LOG.error("Failed to open rocksdb located at " + rocksDbDir, e);
        throw new RuntimeException(e.getMessage());
    }
    lastCheckpointFiles = new HashSet<String>();
    lastCleanTime = System.currentTimeMillis();
    lastSuccessBatchId = -1;
}
Also used : ColumnFamilyOptions(org.rocksdb.ColumnFamilyOptions) DBOptions(org.rocksdb.DBOptions) WriteOptions(org.rocksdb.WriteOptions) Options(org.rocksdb.Options) RocksDBException(org.rocksdb.RocksDBException) ArrayList(java.util.ArrayList) ColumnFamilyDescriptor(org.rocksdb.ColumnFamilyDescriptor) ColumnFamilyHandle(org.rocksdb.ColumnFamilyHandle) ColumnFamilyOptions(org.rocksdb.ColumnFamilyOptions) RocksDbOptionsFactory(com.alibaba.jstorm.cache.rocksdb.RocksDbOptionsFactory) DBOptions(org.rocksdb.DBOptions)

Example 9 with Options

use of org.rocksdb.Options in project aion by aionnetwork.

the class RocksDBWrapper method open.

// IDatabase Functionality
@Override
public boolean open() {
    if (isOpen()) {
        return true;
    }
    LOG.debug("Initialising RockDB {}", this.toString());
    File f = new File(path);
    File dbRoot = f.getParentFile();
    // make the parent directory if not exists
    if (!dbRoot.exists()) {
        if (!f.getParentFile().mkdirs()) {
            LOG.error("Failed to initialize the database storage for " + this.toString() + ".");
            return false;
        }
    }
    writeOptions = setupWriteOptions();
    Options options = setupRocksDbOptions();
    try {
        db = RocksDB.open(options, f.getAbsolutePath());
    } catch (RocksDBException e) {
        if (e.getMessage().contains("lock")) {
            LOG.error("Failed to open the database " + this.toString() + "\nCheck if you have two instances running on the same database." + "\nFailure due to: ", e);
        } else {
            LOG.error("Failed to open the database " + this.toString() + " due to: ", e);
        }
        // close the connection and cleanup if needed
        close();
    }
    return isOpen();
}
Also used : ReadOptions(org.rocksdb.ReadOptions) WriteOptions(org.rocksdb.WriteOptions) Options(org.rocksdb.Options) RocksDBException(org.rocksdb.RocksDBException) File(java.io.File)

Example 10 with Options

use of org.rocksdb.Options in project flink by apache.

the class ListViaRangeSpeedMiniBenchmark method main.

public static void main(String[] args) throws Exception {
    final File rocksDir = new File("/tmp/rdb");
    FileUtils.deleteDirectory(rocksDir);
    final Options options = new Options().setCompactionStyle(CompactionStyle.LEVEL).setLevelCompactionDynamicLevelBytes(true).setIncreaseParallelism(4).setUseFsync(false).setMaxOpenFiles(-1).setDisableDataSync(true).setCreateIfMissing(true).setMergeOperator(new StringAppendOperator());
    final WriteOptions write_options = new WriteOptions().setSync(false).setDisableWAL(true);
    final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath());
    final String key = "key";
    final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";
    final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
    final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
    final byte[] keyTemplate = Arrays.copyOf(keyBytes, keyBytes.length + 4);
    final Unsafe unsafe = MemoryUtils.UNSAFE;
    final long offset = unsafe.arrayBaseOffset(byte[].class) + keyTemplate.length - 4;
    final int num = 50000;
    System.out.println("begin insert");
    final long beginInsert = System.nanoTime();
    for (int i = 0; i < num; i++) {
        unsafe.putInt(keyTemplate, offset, i);
        rocksDB.put(write_options, keyTemplate, valueBytes);
    }
    final long endInsert = System.nanoTime();
    System.out.println("end insert - duration: " + ((endInsert - beginInsert) / 1_000_000) + " ms");
    final byte[] resultHolder = new byte[num * valueBytes.length];
    final long beginGet = System.nanoTime();
    final RocksIterator iterator = rocksDB.newIterator();
    int pos = 0;
    // seek to start
    unsafe.putInt(keyTemplate, offset, 0);
    iterator.seek(keyTemplate);
    // mark end
    unsafe.putInt(keyTemplate, offset, -1);
    // iterate
    while (iterator.isValid()) {
        byte[] currKey = iterator.key();
        if (samePrefix(keyBytes, currKey)) {
            byte[] currValue = iterator.value();
            System.arraycopy(currValue, 0, resultHolder, pos, currValue.length);
            pos += currValue.length;
            iterator.next();
        } else {
            break;
        }
    }
    final long endGet = System.nanoTime();
    System.out.println("end get - duration: " + ((endGet - beginGet) / 1_000_000) + " ms");
}
Also used : Options(org.rocksdb.Options) WriteOptions(org.rocksdb.WriteOptions) WriteOptions(org.rocksdb.WriteOptions) RocksDB(org.rocksdb.RocksDB) StringAppendOperator(org.rocksdb.StringAppendOperator) Unsafe(sun.misc.Unsafe) RocksIterator(org.rocksdb.RocksIterator) File(java.io.File)

Aggregations

Options (org.rocksdb.Options)18 WriteOptions (org.rocksdb.WriteOptions)13 File (java.io.File)9 FlushOptions (org.rocksdb.FlushOptions)8 RocksDBException (org.rocksdb.RocksDBException)6 BlockBasedTableConfig (org.rocksdb.BlockBasedTableConfig)5 MapConfig (org.apache.samza.config.MapConfig)4 MetricsRegistryMap (org.apache.samza.metrics.MetricsRegistryMap)4 RocksDB (org.rocksdb.RocksDB)4 IOException (java.io.IOException)3 ArrayList (java.util.ArrayList)3 Config (org.apache.samza.config.Config)3 ColumnFamilyOptions (org.rocksdb.ColumnFamilyOptions)3 DBOptions (org.rocksdb.DBOptions)3 ReadOptions (org.rocksdb.ReadOptions)3 RocksDbOptionsFactory (com.alibaba.jstorm.cache.rocksdb.RocksDbOptionsFactory)2 ByteArrayOutputStream (java.io.ByteArrayOutputStream)2 RocksDBConfigSetter (org.apache.kafka.streams.state.RocksDBConfigSetter)2 Test (org.junit.Test)2 ColumnFamilyDescriptor (org.rocksdb.ColumnFamilyDescriptor)2