Examples with ColumnFamilyOptions org.rocksdb.ColumnFamilyOptions used on opensource projects

Example 1 with ColumnFamilyOptions

use of org.rocksdb.ColumnFamilyOptions in project flink by apache.

the class KVStateRequestSerializerRocksDBTest method testListSerialization.

/**
	 * Tests list serialization and deserialization match.
	 *
	 * @see KvStateRequestSerializerTest#testListSerialization()
	 * KvStateRequestSerializerTest#testListSerialization() using the heap state back-end
	 * test
	 */
@Test
public void testListSerialization() throws Exception {
    final long key = 0L;
    // objects for RocksDB state list serialisation
    DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
    dbOptions.setCreateIfMissing(true);
    ColumnFamilyOptions columnFamilyOptions = PredefinedOptions.DEFAULT.createColumnOptions();
    final RocksDBKeyedStateBackend2<Long> longHeapKeyedStateBackend = new RocksDBKeyedStateBackend2<>(new JobID(), "no-op", ClassLoader.getSystemClassLoader(), temporaryFolder.getRoot(), dbOptions, columnFamilyOptions, mock(TaskKvStateRegistry.class), LongSerializer.INSTANCE, 1, new KeyGroupRange(0, 0), new ExecutionConfig());
    longHeapKeyedStateBackend.setCurrentKey(key);
    final InternalListState<VoidNamespace, Long> listState = longHeapKeyedStateBackend.createListState(VoidNamespaceSerializer.INSTANCE, new ListStateDescriptor<>("test", LongSerializer.INSTANCE));
    KvStateRequestSerializerTest.testListSerialization(key, listState);
}

Example 2 with ColumnFamilyOptions

use of org.rocksdb.ColumnFamilyOptions in project flink by apache.

the class RocksDBOperationsUtilsTest method testPathExceptionOnWindows.

@Test
public void testPathExceptionOnWindows() throws Exception {
    assumeTrue(OperatingSystem.isWindows());
    final File folder = TMP_DIR.newFolder();
    final File rocksDir = new File(folder, getLongString(247 - folder.getAbsolutePath().length()));
    Files.createDirectories(rocksDir.toPath());
    try (DBOptions dbOptions = new DBOptions().setCreateIfMissing(true);
        ColumnFamilyOptions colOptions = new ColumnFamilyOptions()) {
        RocksDB rocks = RocksDBOperationUtils.openDB(rocksDir.getAbsolutePath(), Collections.emptyList(), Collections.emptyList(), colOptions, dbOptions);
        rocks.close();
    // do not provoke a test failure if this passes, because some setups may actually
    // support long paths, in which case: great!
    } catch (IOException e) {
        assertThat(e.getMessage(), containsString("longer than the directory path length limit for Windows"));
    }
}

Example 3 with ColumnFamilyOptions

use of org.rocksdb.ColumnFamilyOptions in project flink by apache.

the class RocksDBOperationUtils method openDB.

public static RocksDB openDB(String path, List<ColumnFamilyDescriptor> stateColumnFamilyDescriptors, List<ColumnFamilyHandle> stateColumnFamilyHandles, ColumnFamilyOptions columnFamilyOptions, DBOptions dbOptions) throws IOException {
    List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(1 + stateColumnFamilyDescriptors.size());
    // we add the required descriptor for the default CF in FIRST position, see
    // https://github.com/facebook/rocksdb/wiki/RocksJava-Basics#opening-a-database-with-column-families
    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions));
    columnFamilyDescriptors.addAll(stateColumnFamilyDescriptors);
    RocksDB dbRef;
    try {
        dbRef = RocksDB.open(Preconditions.checkNotNull(dbOptions), Preconditions.checkNotNull(path), columnFamilyDescriptors, stateColumnFamilyHandles);
    } catch (RocksDBException e) {
        IOUtils.closeQuietly(columnFamilyOptions);
        columnFamilyDescriptors.forEach((cfd) -> IOUtils.closeQuietly(cfd.getOptions()));
        // improve error reporting on Windows
        throwExceptionIfPathLengthExceededOnWindows(path, e);
        throw new IOException("Error while opening RocksDB instance.", e);
    }
    // requested + default CF
    Preconditions.checkState(1 + stateColumnFamilyDescriptors.size() == stateColumnFamilyHandles.size(), "Not all requested column family handles have been created");
    return dbRef;
}

Example 4 with ColumnFamilyOptions

use of org.rocksdb.ColumnFamilyOptions in project flink by apache.

the class RocksDBResource method before.

@Override
protected void before() throws Throwable {
    this.temporaryFolder = new TemporaryFolder();
    this.temporaryFolder.create();
    final File rocksFolder = temporaryFolder.newFolder();
    this.dbOptions = optionsFactory.createDBOptions(new DBOptions().setUseFsync(false).setInfoLogLevel(InfoLogLevel.HEADER_LEVEL).setStatsDumpPeriodSec(0), handlesToClose).setCreateIfMissing(true);
    this.columnFamilyOptions = optionsFactory.createColumnOptions(new ColumnFamilyOptions(), handlesToClose);
    this.writeOptions = new WriteOptions();
    this.writeOptions.disableWAL();
    this.readOptions = new ReadOptions();
    this.columnFamilyHandles = new ArrayList<>(1);
    this.rocksDB = RocksDB.open(dbOptions, rocksFolder.getAbsolutePath(), Collections.singletonList(new ColumnFamilyDescriptor("default".getBytes(), columnFamilyOptions)), columnFamilyHandles);
    this.batchWrapper = new RocksDBWriteBatchWrapper(rocksDB, writeOptions);
}

Example 5 with ColumnFamilyOptions

use of org.rocksdb.ColumnFamilyOptions in project flink by apache.

the class RocksDBResourceContainerTest method testGetColumnFamilyOptionsWithSharedResources.

/**
 * Guard that {@link RocksDBResourceContainer#getColumnOptions()} shares the same {@link Cache}
 * instance if the {@link RocksDBResourceContainer} instance is initiated with {@link
 * OpaqueMemoryResource}.
 *
 * @throws Exception if unexpected error happened.
 */
@Test
public void testGetColumnFamilyOptionsWithSharedResources() throws Exception {
    final int optionNumber = 20;
    OpaqueMemoryResource<RocksDBSharedResources> sharedResources = getSharedResources();
    RocksDBResourceContainer container = new RocksDBResourceContainer(PredefinedOptions.DEFAULT, null, sharedResources);
    HashSet<Cache> caches = new HashSet<>();
    for (int i = 0; i < optionNumber; i++) {
        ColumnFamilyOptions columnOptions = container.getColumnOptions();
        Cache cache = getBlockCache(columnOptions);
        caches.add(cache);
    }
    assertThat(caches.size(), is(1));
    assertThat(caches.iterator().next(), is(sharedResources.getResourceHandle().getCache()));
    container.close();
}