Examples with DBOptions org.rocksdb.DBOptions used on opensource projects

Example 1 with DBOptions

use of org.rocksdb.DBOptions in project flink by apache.

the class RocksDBStateBackend method getDbOptions.

/**
	 * Gets the RocksDB {@link DBOptions} to be used for all RocksDB instances.
	 */
public DBOptions getDbOptions() {
    // initial options from pre-defined profile
    DBOptions opt = predefinedOptions.createDBOptions();
    // add user-defined options, if specified
    if (optionsFactory != null) {
        opt = optionsFactory.createDBOptions(opt);
    }
    // add necessary default options
    opt = opt.setCreateIfMissing(true);
    return opt;
}

Example 2 with DBOptions

use of org.rocksdb.DBOptions in project flink by apache.

the class KVStateRequestSerializerRocksDBTest method testListSerialization.

/**
	 * Tests list serialization and deserialization match.
	 *
	 * @see KvStateRequestSerializerTest#testListSerialization()
	 * KvStateRequestSerializerTest#testListSerialization() using the heap state back-end
	 * test
	 */
@Test
public void testListSerialization() throws Exception {
    final long key = 0L;
    // objects for RocksDB state list serialisation
    DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
    dbOptions.setCreateIfMissing(true);
    ColumnFamilyOptions columnFamilyOptions = PredefinedOptions.DEFAULT.createColumnOptions();
    final RocksDBKeyedStateBackend2<Long> longHeapKeyedStateBackend = new RocksDBKeyedStateBackend2<>(new JobID(), "no-op", ClassLoader.getSystemClassLoader(), temporaryFolder.getRoot(), dbOptions, columnFamilyOptions, mock(TaskKvStateRegistry.class), LongSerializer.INSTANCE, 1, new KeyGroupRange(0, 0), new ExecutionConfig());
    longHeapKeyedStateBackend.setCurrentKey(key);
    final InternalListState<VoidNamespace, Long> listState = longHeapKeyedStateBackend.createListState(VoidNamespaceSerializer.INSTANCE, new ListStateDescriptor<>("test", LongSerializer.INSTANCE));
    KvStateRequestSerializerTest.testListSerialization(key, listState);
}

Example 3 with DBOptions

use of org.rocksdb.DBOptions in project flink by apache.

the class RocksDBOperationsUtilsTest method testPathExceptionOnWindows.

@Test
public void testPathExceptionOnWindows() throws Exception {
    assumeTrue(OperatingSystem.isWindows());
    final File folder = TMP_DIR.newFolder();
    final File rocksDir = new File(folder, getLongString(247 - folder.getAbsolutePath().length()));
    Files.createDirectories(rocksDir.toPath());
    try (DBOptions dbOptions = new DBOptions().setCreateIfMissing(true);
        ColumnFamilyOptions colOptions = new ColumnFamilyOptions()) {
        RocksDB rocks = RocksDBOperationUtils.openDB(rocksDir.getAbsolutePath(), Collections.emptyList(), Collections.emptyList(), colOptions, dbOptions);
        rocks.close();
    // do not provoke a test failure if this passes, because some setups may actually
    // support long paths, in which case: great!
    } catch (IOException e) {
        assertThat(e.getMessage(), containsString("longer than the directory path length limit for Windows"));
    }
}

Example 4 with DBOptions

use of org.rocksdb.DBOptions in project flink by apache.

the class RocksDBKeyedStateBackendBuilder method getRocksDBRestoreOperation.

private RocksDBRestoreOperation getRocksDBRestoreOperation(int keyGroupPrefixBytes, CloseableRegistry cancelStreamRegistry, LinkedHashMap<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation, LinkedHashMap<String, HeapPriorityQueueSnapshotRestoreWrapper<?>> registeredPQStates, RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
    DBOptions dbOptions = optionsContainer.getDbOptions();
    if (restoreStateHandles.isEmpty()) {
        return new RocksDBNoneRestoreOperation<>(kvStateInformation, instanceRocksDBPath, dbOptions, columnFamilyOptionsFactory, nativeMetricOptions, metricGroup, ttlCompactFiltersManager, optionsContainer.getWriteBufferManagerCapacity());
    }
    KeyedStateHandle firstStateHandle = restoreStateHandles.iterator().next();
    if (firstStateHandle instanceof IncrementalKeyedStateHandle) {
        return new RocksDBIncrementalRestoreOperation<>(operatorIdentifier, keyGroupRange, keyGroupPrefixBytes, numberOfTransferingThreads, cancelStreamRegistry, userCodeClassLoader, kvStateInformation, keySerializerProvider, instanceBasePath, instanceRocksDBPath, dbOptions, columnFamilyOptionsFactory, nativeMetricOptions, metricGroup, restoreStateHandles, ttlCompactFiltersManager, writeBatchSize, optionsContainer.getWriteBufferManagerCapacity());
    } else if (priorityQueueStateType == EmbeddedRocksDBStateBackend.PriorityQueueStateType.HEAP) {
        return new RocksDBHeapTimersFullRestoreOperation<>(keyGroupRange, numberOfKeyGroups, userCodeClassLoader, kvStateInformation, registeredPQStates, createHeapQueueFactory(), keySerializerProvider, instanceRocksDBPath, dbOptions, columnFamilyOptionsFactory, nativeMetricOptions, metricGroup, restoreStateHandles, ttlCompactFiltersManager, writeBatchSize, optionsContainer.getWriteBufferManagerCapacity());
    } else {
        return new RocksDBFullRestoreOperation<>(keyGroupRange, userCodeClassLoader, kvStateInformation, keySerializerProvider, instanceRocksDBPath, dbOptions, columnFamilyOptionsFactory, nativeMetricOptions, metricGroup, restoreStateHandles, ttlCompactFiltersManager, writeBatchSize, optionsContainer.getWriteBufferManagerCapacity());
    }
}

Example 5 with DBOptions

use of org.rocksdb.DBOptions in project flink by apache.

the class RocksDBOperationUtils method openDB.

public static RocksDB openDB(String path, List<ColumnFamilyDescriptor> stateColumnFamilyDescriptors, List<ColumnFamilyHandle> stateColumnFamilyHandles, ColumnFamilyOptions columnFamilyOptions, DBOptions dbOptions) throws IOException {
    List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(1 + stateColumnFamilyDescriptors.size());
    // we add the required descriptor for the default CF in FIRST position, see
    // https://github.com/facebook/rocksdb/wiki/RocksJava-Basics#opening-a-database-with-column-families
    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions));
    columnFamilyDescriptors.addAll(stateColumnFamilyDescriptors);
    RocksDB dbRef;
    try {
        dbRef = RocksDB.open(Preconditions.checkNotNull(dbOptions), Preconditions.checkNotNull(path), columnFamilyDescriptors, stateColumnFamilyHandles);
    } catch (RocksDBException e) {
        IOUtils.closeQuietly(columnFamilyOptions);
        columnFamilyDescriptors.forEach((cfd) -> IOUtils.closeQuietly(cfd.getOptions()));
        // improve error reporting on Windows
        throwExceptionIfPathLengthExceededOnWindows(path, e);
        throw new IOException("Error while opening RocksDB instance.", e);
    }
    // requested + default CF
    Preconditions.checkState(1 + stateColumnFamilyDescriptors.size() == stateColumnFamilyHandles.size(), "Not all requested column family handles have been created");
    return dbRef;
}