Examples with RocksDB org.rocksdb.RocksDB used on opensource projects

Example 1 with RocksDB

use of org.rocksdb.RocksDB in project flink by apache.

the class RocksDBMergeIteratorTest method testMergeIterator.

public void testMergeIterator(int maxParallelism) throws Exception {
    Random random = new Random(1234);
    File tmpDir = CommonTestUtils.createTempDirectory();
    RocksDB rocksDB = RocksDB.open(tmpDir.getAbsolutePath());
    try {
        List<Tuple2<RocksIterator, Integer>> rocksIteratorsWithKVStateId = new ArrayList<>();
        List<Tuple2<ColumnFamilyHandle, Integer>> columnFamilyHandlesWithKeyCount = new ArrayList<>();
        int totalKeysExpected = 0;
        for (int c = 0; c < NUM_KEY_VAL_STATES; ++c) {
            ColumnFamilyHandle handle = rocksDB.createColumnFamily(new ColumnFamilyDescriptor(("column-" + c).getBytes(ConfigConstants.DEFAULT_CHARSET)));
            ByteArrayOutputStreamWithPos bos = new ByteArrayOutputStreamWithPos();
            DataOutputStream dos = new DataOutputStream(bos);
            int numKeys = random.nextInt(MAX_NUM_KEYS + 1);
            for (int i = 0; i < numKeys; ++i) {
                if (maxParallelism <= Byte.MAX_VALUE) {
                    dos.writeByte(i);
                } else {
                    dos.writeShort(i);
                }
                dos.writeInt(i);
                byte[] key = bos.toByteArray();
                byte[] val = new byte[] { 42 };
                rocksDB.put(handle, key, val);
                bos.reset();
            }
            columnFamilyHandlesWithKeyCount.add(new Tuple2<>(handle, numKeys));
            totalKeysExpected += numKeys;
        }
        int id = 0;
        for (Tuple2<ColumnFamilyHandle, Integer> columnFamilyHandle : columnFamilyHandlesWithKeyCount) {
            rocksIteratorsWithKVStateId.add(new Tuple2<>(rocksDB.newIterator(columnFamilyHandle.f0), id));
            ++id;
        }
        RocksDBKeyedStateBackend.RocksDBMergeIterator mergeIterator = new RocksDBKeyedStateBackend.RocksDBMergeIterator(rocksIteratorsWithKVStateId, maxParallelism <= Byte.MAX_VALUE ? 1 : 2);
        int prevKVState = -1;
        int prevKey = -1;
        int prevKeyGroup = -1;
        int totalKeysActual = 0;
        while (mergeIterator.isValid()) {
            ByteBuffer bb = ByteBuffer.wrap(mergeIterator.key());
            int keyGroup = maxParallelism > Byte.MAX_VALUE ? bb.getShort() : bb.get();
            int key = bb.getInt();
            Assert.assertTrue(keyGroup >= prevKeyGroup);
            Assert.assertTrue(key >= prevKey);
            Assert.assertEquals(prevKeyGroup != keyGroup, mergeIterator.isNewKeyGroup());
            Assert.assertEquals(prevKVState != mergeIterator.kvStateId(), mergeIterator.isNewKeyValueState());
            prevKeyGroup = keyGroup;
            prevKVState = mergeIterator.kvStateId();
            //System.out.println(keyGroup + " " + key + " " + mergeIterator.kvStateId());
            mergeIterator.next();
            ++totalKeysActual;
        }
        Assert.assertEquals(totalKeysExpected, totalKeysActual);
        for (Tuple2<ColumnFamilyHandle, Integer> handleWithCount : columnFamilyHandlesWithKeyCount) {
            rocksDB.dropColumnFamily(handleWithCount.f0);
        }
    } finally {
        rocksDB.close();
    }
}

Example 2 with RocksDB

use of org.rocksdb.RocksDB in project flink by apache.

the class RocksDBStateBackendTest method testRunningSnapshotAfterBackendClosed.

@Test
public void testRunningSnapshotAfterBackendClosed() throws Exception {
    setupRocksKeyedStateBackend();
    RunnableFuture<KeyGroupsStateHandle> snapshot = keyedStateBackend.snapshot(0L, 0L, testStreamFactory, CheckpointOptions.forFullCheckpoint());
    RocksDB spyDB = keyedStateBackend.db;
    verify(spyDB, times(1)).getSnapshot();
    verify(spyDB, times(0)).releaseSnapshot(any(Snapshot.class));
    this.keyedStateBackend.dispose();
    verify(spyDB, times(1)).close();
    assertEquals(null, keyedStateBackend.db);
    //Ensure every RocksObjects not closed yet
    for (RocksObject rocksCloseable : allCreatedCloseables) {
        verify(rocksCloseable, times(0)).close();
    }
    Thread asyncSnapshotThread = new Thread(snapshot);
    asyncSnapshotThread.start();
    try {
        snapshot.get();
        fail();
    } catch (Exception ignored) {
    }
    asyncSnapshotThread.join();
    //Ensure every RocksObject was closed exactly once
    for (RocksObject rocksCloseable : allCreatedCloseables) {
        verify(rocksCloseable, times(1)).close();
    }
}

Example 3 with RocksDB

use of org.rocksdb.RocksDB in project flink by apache.

the class ListViaMergeSpeedMiniBenchmark method main.

public static void main(String[] args) throws Exception {
    final File rocksDir = new File("/tmp/rdb");
    FileUtils.deleteDirectory(rocksDir);
    final Options options = new Options().setCompactionStyle(CompactionStyle.LEVEL).setLevelCompactionDynamicLevelBytes(true).setIncreaseParallelism(4).setUseFsync(false).setMaxOpenFiles(-1).setDisableDataSync(true).setCreateIfMissing(true).setMergeOperator(new StringAppendOperator());
    final WriteOptions write_options = new WriteOptions().setSync(false).setDisableWAL(true);
    final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath());
    final String key = "key";
    final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";
    final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
    final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
    final int num = 50000;
    // ----- insert -----
    System.out.println("begin insert");
    final long beginInsert = System.nanoTime();
    for (int i = 0; i < num; i++) {
        rocksDB.merge(write_options, keyBytes, valueBytes);
    }
    final long endInsert = System.nanoTime();
    System.out.println("end insert - duration: " + ((endInsert - beginInsert) / 1_000_000) + " ms");
    // ----- read (attempt 1) -----
    final byte[] resultHolder = new byte[num * (valueBytes.length + 2)];
    final long beginGet1 = System.nanoTime();
    rocksDB.get(keyBytes, resultHolder);
    final long endGet1 = System.nanoTime();
    System.out.println("end get - duration: " + ((endGet1 - beginGet1) / 1_000_000) + " ms");
    // ----- read (attempt 2) -----
    final long beginGet2 = System.nanoTime();
    rocksDB.get(keyBytes, resultHolder);
    final long endGet2 = System.nanoTime();
    System.out.println("end get - duration: " + ((endGet2 - beginGet2) / 1_000_000) + " ms");
    // ----- compact -----
    System.out.println("compacting...");
    final long beginCompact = System.nanoTime();
    rocksDB.compactRange();
    final long endCompact = System.nanoTime();
    System.out.println("end compaction - duration: " + ((endCompact - beginCompact) / 1_000_000) + " ms");
    // ----- read (attempt 3) -----
    final long beginGet3 = System.nanoTime();
    rocksDB.get(keyBytes, resultHolder);
    final long endGet3 = System.nanoTime();
    System.out.println("end get - duration: " + ((endGet3 - beginGet3) / 1_000_000) + " ms");
}

Example 4 with RocksDB

use of org.rocksdb.RocksDB in project voldemort by voldemort.

the class RocksDbStorageConfiguration method getStore.

@Override
public StorageEngine<ByteArray, byte[], byte[]> getStore(StoreDefinition storeDef, RoutingStrategy strategy) {
    String storeName = storeDef.getName();
    if (!stores.containsKey(storeName)) {
        String dataDir = this.voldemortconfig.getRdbDataDirectory() + "/" + storeName;
        new File(dataDir).mkdirs();
        Properties dbProperties = parseProperties(VoldemortConfig.ROCKSDB_DB_OPTIONS);
        DBOptions dbOptions = (dbProperties.size() > 0) ? DBOptions.getDBOptionsFromProps(dbProperties) : new DBOptions();
        if (dbOptions == null) {
            throw new StorageInitializationException("Unable to parse Data Base Options.");
        }
        dbOptions.setCreateIfMissing(true);
        dbOptions.setCreateMissingColumnFamilies(true);
        dbOptions.createStatistics();
        Properties cfProperties = parseProperties(VoldemortConfig.ROCKSDB_CF_OPTIONS);
        if (this.voldemortconfig.getRocksdbPrefixKeysWithPartitionId()) {
            cfProperties.setProperty("prefix_extractor", "fixed:" + StoreBinaryFormat.PARTITIONID_PREFIX_SIZE);
        }
        ColumnFamilyOptions cfOptions = (cfProperties.size() > 0) ? ColumnFamilyOptions.getColumnFamilyOptionsFromProps(cfProperties) : new ColumnFamilyOptions();
        if (cfOptions == null) {
            throw new StorageInitializationException("Unable to parse Column Family Options.");
        }
        // Create a non default Column Family tp hold the store data.
        List<ColumnFamilyDescriptor> descriptors = new ArrayList<ColumnFamilyDescriptor>();
        descriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions));
        descriptors.add(new ColumnFamilyDescriptor(storeName.getBytes(), cfOptions));
        List<ColumnFamilyHandle> handles = new ArrayList<ColumnFamilyHandle>();
        try {
            RocksDB rdbStore = RocksDB.open(dbOptions, dataDir, descriptors, handles);
            // Dispose of the default Column Family immediately.  We don't use it and if it has not been disposed
            // by the time the DB is closed then the RocksDB code can terminate abnormally (if the RocksDB code is
            // built with assertions enabled). The handle will go out of scope on its own and the Java finalizer
            // will (eventually) do this for us, but, that is not fast enough for the unit tests.
            handles.get(0).dispose();
            ColumnFamilyHandle storeHandle = handles.get(1);
            RocksDbStorageEngine rdbStorageEngine;
            if (this.voldemortconfig.getRocksdbPrefixKeysWithPartitionId()) {
                rdbStorageEngine = new PartitionPrefixedRocksDbStorageEngine(storeName, rdbStore, storeHandle, cfOptions, lockStripes, strategy, voldemortconfig.isRocksdbEnableReadLocks());
            } else {
                rdbStorageEngine = new RocksDbStorageEngine(storeName, rdbStore, storeHandle, cfOptions, lockStripes, voldemortconfig.isRocksdbEnableReadLocks());
            }
            stores.put(storeName, rdbStorageEngine);
        } catch (Exception e) {
            throw new StorageInitializationException(e);
        }
    }
    return stores.get(storeName);
}

Example 5 with RocksDB

use of org.rocksdb.RocksDB in project flink by apache.

the class RocksDBStateBackendTest method verifyRocksObjectsReleased.

private void verifyRocksObjectsReleased() {
    //Ensure every RocksObject was closed exactly once
    for (RocksObject rocksCloseable : allCreatedCloseables) {
        verify(rocksCloseable, times(1)).close();
    }
    assertNotNull(null, keyedStateBackend.db);
    RocksDB spyDB = keyedStateBackend.db;
    verify(spyDB, times(1)).getSnapshot();
    verify(spyDB, times(1)).releaseSnapshot(any(Snapshot.class));
    keyedStateBackend.dispose();
    verify(spyDB, times(1)).close();
    assertEquals(null, keyedStateBackend.db);
}