Examples with ColumnFamilyOptions org.rocksdb.ColumnFamilyOptions used on opensource projects

Example 21 with ColumnFamilyOptions

use of org.rocksdb.ColumnFamilyOptions in project alluxio by Alluxio.

the class RocksStoreTest method backupRestore.

@Test
public void backupRestore() throws Exception {
    ColumnFamilyOptions cfOpts = new ColumnFamilyOptions().setMemTableConfig(new HashLinkedListMemTableConfig()).setCompressionType(CompressionType.NO_COMPRESSION).useFixedLengthPrefixExtractor(// We always search using the initial long key
    Longs.BYTES);
    List<ColumnFamilyDescriptor> columnDescriptors = Arrays.asList(new ColumnFamilyDescriptor("test".getBytes(), cfOpts));
    String dbDir = mFolder.newFolder("rocks").getAbsolutePath();
    String backupsDir = mFolder.newFolder("rocks-backups").getAbsolutePath();
    AtomicReference<ColumnFamilyHandle> testColumn = new AtomicReference<>();
    RocksStore store = new RocksStore("test", dbDir, backupsDir, columnDescriptors, Arrays.asList(testColumn));
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    RocksDB db = store.getDb();
    int count = 10;
    for (int i = 0; i < count; i++) {
        db.put(testColumn.get(), new WriteOptions().setDisableWAL(true), ("a" + i).getBytes(), "b".getBytes());
    }
    store.writeToCheckpoint(baos);
    String newBbDir = mFolder.newFolder("rocks-new").getAbsolutePath();
    store = new RocksStore("test-new", newBbDir, backupsDir, columnDescriptors, Arrays.asList(testColumn));
    store.restoreFromCheckpoint(new CheckpointInputStream(new ByteArrayInputStream(baos.toByteArray())));
    db = store.getDb();
    for (int i = 0; i < count; i++) {
        assertArrayEquals("b".getBytes(), db.get(testColumn.get(), ("a" + i).getBytes()));
    }
}

Example 22 with ColumnFamilyOptions

use of org.rocksdb.ColumnFamilyOptions in project flink by apache.

the class RocksDBOperationUtils method createColumnFamilyDescriptor.

/**
 * Creates a column descriptor for a state column family.
 *
 * <p>Sets TTL compaction filter if {@code ttlCompactFiltersManager} is not {@code null}.
 */
public static ColumnFamilyDescriptor createColumnFamilyDescriptor(RegisteredStateMetaInfoBase metaInfoBase, Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory, @Nullable RocksDbTtlCompactFiltersManager ttlCompactFiltersManager, @Nullable Long writeBufferManagerCapacity) {
    ColumnFamilyOptions options = createColumnFamilyOptions(columnFamilyOptionsFactory, metaInfoBase.getName());
    if (ttlCompactFiltersManager != null) {
        ttlCompactFiltersManager.setAndRegisterCompactFilterIfStateTtl(metaInfoBase, options);
    }
    byte[] nameBytes = metaInfoBase.getName().getBytes(ConfigConstants.DEFAULT_CHARSET);
    Preconditions.checkState(!Arrays.equals(RocksDB.DEFAULT_COLUMN_FAMILY, nameBytes), "The chosen state name 'default' collides with the name of the default column family!");
    if (writeBufferManagerCapacity != null) {
        // It'd be great to perform the check earlier, e.g. when creating write buffer manager.
        // Unfortunately the check needs write buffer size that was just calculated.
        sanityCheckArenaBlockSize(options.writeBufferSize(), options.arenaBlockSize(), writeBufferManagerCapacity);
    }
    return new ColumnFamilyDescriptor(nameBytes, options);
}

Example 23 with ColumnFamilyOptions

use of org.rocksdb.ColumnFamilyOptions in project flink by apache.

the class RocksDBResourceContainer method getColumnOptions.

/**
 * Gets the RocksDB {@link ColumnFamilyOptions} to be used for all RocksDB instances.
 */
public ColumnFamilyOptions getColumnOptions() {
    // initial options from common profile
    ColumnFamilyOptions opt = createBaseCommonColumnOptions();
    handlesToClose.add(opt);
    // load configurable options on top of pre-defined profile
    setColumnFamilyOptionsFromConfigurableOptions(opt, handlesToClose);
    // add user-defined options, if specified
    if (optionsFactory != null) {
        opt = optionsFactory.createColumnOptions(opt, handlesToClose);
    }
    // set necessary options for performance consideration with memory control
    if (sharedResources != null) {
        final RocksDBSharedResources rocksResources = sharedResources.getResourceHandle();
        final Cache blockCache = rocksResources.getCache();
        TableFormatConfig tableFormatConfig = opt.tableFormatConfig();
        BlockBasedTableConfig blockBasedTableConfig;
        if (tableFormatConfig == null) {
            blockBasedTableConfig = new BlockBasedTableConfig();
        } else {
            Preconditions.checkArgument(tableFormatConfig instanceof BlockBasedTableConfig, "We currently only support BlockBasedTableConfig When bounding total memory.");
            blockBasedTableConfig = (BlockBasedTableConfig) tableFormatConfig;
        }
        if (rocksResources.isUsingPartitionedIndexFilters() && overwriteFilterIfExist(blockBasedTableConfig)) {
            blockBasedTableConfig.setIndexType(IndexType.kTwoLevelIndexSearch);
            blockBasedTableConfig.setPartitionFilters(true);
            blockBasedTableConfig.setPinTopLevelIndexAndFilter(true);
        }
        blockBasedTableConfig.setBlockCache(blockCache);
        blockBasedTableConfig.setCacheIndexAndFilterBlocks(true);
        blockBasedTableConfig.setCacheIndexAndFilterBlocksWithHighPriority(true);
        blockBasedTableConfig.setPinL0FilterAndIndexBlocksInCache(true);
        opt.setTableFormatConfig(blockBasedTableConfig);
    }
    return opt;
}

Example 24 with ColumnFamilyOptions

use of org.rocksdb.ColumnFamilyOptions in project flink by apache.

the class EmbeddedRocksDBStateBackendTest method testCorrectMergeOperatorSet.

@Test
public void testCorrectMergeOperatorSet() throws Exception {
    prepareRocksDB();
    final ColumnFamilyOptions columnFamilyOptions = spy(new ColumnFamilyOptions());
    RocksDBKeyedStateBackend<Integer> test = null;
    try {
        test = RocksDBTestUtils.builderForTestDB(TEMP_FOLDER.newFolder(), IntSerializer.INSTANCE, db, defaultCFHandle, columnFamilyOptions).setEnableIncrementalCheckpointing(enableIncrementalCheckpointing).build();
        ValueStateDescriptor<String> stubState1 = new ValueStateDescriptor<>("StubState-1", StringSerializer.INSTANCE);
        test.createInternalState(StringSerializer.INSTANCE, stubState1);
        ValueStateDescriptor<String> stubState2 = new ValueStateDescriptor<>("StubState-2", StringSerializer.INSTANCE);
        test.createInternalState(StringSerializer.INSTANCE, stubState2);
        // The default CF is pre-created so sum up to 2 times (once for each stub state)
        verify(columnFamilyOptions, Mockito.times(2)).setMergeOperatorName(RocksDBKeyedStateBackend.MERGE_OPERATOR_NAME);
    } finally {
        if (test != null) {
            IOUtils.closeQuietly(test);
            test.dispose();
        }
        columnFamilyOptions.close();
    }
}

Example 25 with ColumnFamilyOptions

use of org.rocksdb.ColumnFamilyOptions in project flink by apache.

the class EmbeddedRocksDBStateBackendTest method prepareRocksDB.

public void prepareRocksDB() throws Exception {
    String dbPath = new File(TEMP_FOLDER.newFolder(), DB_INSTANCE_DIR_STRING).getAbsolutePath();
    ColumnFamilyOptions columnOptions = optionsContainer.getColumnOptions();
    ArrayList<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(1);
    db = RocksDBOperationUtils.openDB(dbPath, Collections.emptyList(), columnFamilyHandles, columnOptions, optionsContainer.getDbOptions());
    defaultCFHandle = columnFamilyHandles.remove(0);
}