Examples with ColumnFamilyDescriptor org.rocksdb.ColumnFamilyDescriptor used on opensource projects

Example 21 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project kafka by apache.

the class RocksDBStore method openRocksDB.

void openRocksDB(final DBOptions dbOptions, final ColumnFamilyOptions columnFamilyOptions) {
    final List<ColumnFamilyDescriptor> columnFamilyDescriptors = Collections.singletonList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions));
    final List<ColumnFamilyHandle> columnFamilies = new ArrayList<>(columnFamilyDescriptors.size());
    try {
        db = RocksDB.open(dbOptions, dbDir.getAbsolutePath(), columnFamilyDescriptors, columnFamilies);
        dbAccessor = new SingleColumnFamilyAccessor(columnFamilies.get(0));
    } catch (final RocksDBException e) {
        throw new ProcessorStateException("Error opening store " + name + " at location " + dbDir.toString(), e);
    }
}

Example 22 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project kafka by apache.

the class RocksDBTimestampedStore method openRocksDB.

@Override
void openRocksDB(final DBOptions dbOptions, final ColumnFamilyOptions columnFamilyOptions) {
    final List<ColumnFamilyDescriptor> columnFamilyDescriptors = asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions), new ColumnFamilyDescriptor("keyValueWithTimestamp".getBytes(StandardCharsets.UTF_8), columnFamilyOptions));
    final List<ColumnFamilyHandle> columnFamilies = new ArrayList<>(columnFamilyDescriptors.size());
    try {
        db = RocksDB.open(dbOptions, dbDir.getAbsolutePath(), columnFamilyDescriptors, columnFamilies);
        setDbAccessor(columnFamilies.get(0), columnFamilies.get(1));
    } catch (final RocksDBException e) {
        if ("Column family not found: keyValueWithTimestamp".equals(e.getMessage())) {
            try {
                db = RocksDB.open(dbOptions, dbDir.getAbsolutePath(), columnFamilyDescriptors.subList(0, 1), columnFamilies);
                columnFamilies.add(db.createColumnFamily(columnFamilyDescriptors.get(1)));
            } catch (final RocksDBException fatal) {
                throw new ProcessorStateException("Error opening store " + name + " at location " + dbDir.toString(), fatal);
            }
            setDbAccessor(columnFamilies.get(0), columnFamilies.get(1));
        } else {
            throw new ProcessorStateException("Error opening store " + name + " at location " + dbDir.toString(), e);
        }
    }
}

Example 23 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project kafka by apache.

the class RocksDBTimestampedStoreTest method shouldOpenExistingStoreInRegularMode.

@Test
public void shouldOpenExistingStoreInRegularMode() throws Exception {
    // prepare store
    rocksDBStore.init((StateStoreContext) context, rocksDBStore);
    rocksDBStore.put(new Bytes("key".getBytes()), "timestamped".getBytes());
    rocksDBStore.close();
    // re-open store
    try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(RocksDBTimestampedStore.class)) {
        rocksDBStore.init((StateStoreContext) context, rocksDBStore);
        assertThat(appender.getMessages(), hasItem("Opening store " + DB_NAME + " in regular mode"));
    } finally {
        rocksDBStore.close();
    }
    // verify store
    final DBOptions dbOptions = new DBOptions();
    final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions();
    final List<ColumnFamilyDescriptor> columnFamilyDescriptors = asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions), new ColumnFamilyDescriptor("keyValueWithTimestamp".getBytes(StandardCharsets.UTF_8), columnFamilyOptions));
    final List<ColumnFamilyHandle> columnFamilies = new ArrayList<>(columnFamilyDescriptors.size());
    RocksDB db = null;
    ColumnFamilyHandle noTimestampColumnFamily = null, withTimestampColumnFamily = null;
    try {
        db = RocksDB.open(dbOptions, new File(new File(context.stateDir(), "rocksdb"), DB_NAME).getAbsolutePath(), columnFamilyDescriptors, columnFamilies);
        noTimestampColumnFamily = columnFamilies.get(0);
        withTimestampColumnFamily = columnFamilies.get(1);
        assertThat(db.get(noTimestampColumnFamily, "key".getBytes()), new IsNull<>());
        assertThat(db.getLongProperty(noTimestampColumnFamily, "rocksdb.estimate-num-keys"), is(0L));
        assertThat(db.get(withTimestampColumnFamily, "key".getBytes()).length, is(11));
        assertThat(db.getLongProperty(withTimestampColumnFamily, "rocksdb.estimate-num-keys"), is(1L));
    } finally {
        // Order of closing must follow: ColumnFamilyHandle > RocksDB > DBOptions > ColumnFamilyOptions
        if (noTimestampColumnFamily != null) {
            noTimestampColumnFamily.close();
        }
        if (withTimestampColumnFamily != null) {
            withTimestampColumnFamily.close();
        }
        if (db != null) {
            db.close();
        }
        dbOptions.close();
        columnFamilyOptions.close();
    }
}

Example 24 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project alluxio by Alluxio.

the class RocksStoreTest method backupRestore.

@Test
public void backupRestore() throws Exception {
    ColumnFamilyOptions cfOpts = new ColumnFamilyOptions().setMemTableConfig(new HashLinkedListMemTableConfig()).setCompressionType(CompressionType.NO_COMPRESSION).useFixedLengthPrefixExtractor(// We always search using the initial long key
    Longs.BYTES);
    List<ColumnFamilyDescriptor> columnDescriptors = Arrays.asList(new ColumnFamilyDescriptor("test".getBytes(), cfOpts));
    String dbDir = mFolder.newFolder("rocks").getAbsolutePath();
    String backupsDir = mFolder.newFolder("rocks-backups").getAbsolutePath();
    AtomicReference<ColumnFamilyHandle> testColumn = new AtomicReference<>();
    RocksStore store = new RocksStore("test", dbDir, backupsDir, columnDescriptors, Arrays.asList(testColumn));
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    RocksDB db = store.getDb();
    int count = 10;
    for (int i = 0; i < count; i++) {
        db.put(testColumn.get(), new WriteOptions().setDisableWAL(true), ("a" + i).getBytes(), "b".getBytes());
    }
    store.writeToCheckpoint(baos);
    String newBbDir = mFolder.newFolder("rocks-new").getAbsolutePath();
    store = new RocksStore("test-new", newBbDir, backupsDir, columnDescriptors, Arrays.asList(testColumn));
    store.restoreFromCheckpoint(new CheckpointInputStream(new ByteArrayInputStream(baos.toByteArray())));
    db = store.getDb();
    for (int i = 0; i < count; i++) {
        assertArrayEquals("b".getBytes(), db.get(testColumn.get(), ("a" + i).getBytes()));
    }
}

Example 25 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project flink by apache.

the class RocksDBOperationUtils method createColumnFamilyDescriptor.

/**
 * Creates a column descriptor for a state column family.
 *
 * <p>Sets TTL compaction filter if {@code ttlCompactFiltersManager} is not {@code null}.
 */
public static ColumnFamilyDescriptor createColumnFamilyDescriptor(RegisteredStateMetaInfoBase metaInfoBase, Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory, @Nullable RocksDbTtlCompactFiltersManager ttlCompactFiltersManager, @Nullable Long writeBufferManagerCapacity) {
    ColumnFamilyOptions options = createColumnFamilyOptions(columnFamilyOptionsFactory, metaInfoBase.getName());
    if (ttlCompactFiltersManager != null) {
        ttlCompactFiltersManager.setAndRegisterCompactFilterIfStateTtl(metaInfoBase, options);
    }
    byte[] nameBytes = metaInfoBase.getName().getBytes(ConfigConstants.DEFAULT_CHARSET);
    Preconditions.checkState(!Arrays.equals(RocksDB.DEFAULT_COLUMN_FAMILY, nameBytes), "The chosen state name 'default' collides with the name of the default column family!");
    if (writeBufferManagerCapacity != null) {
        // It'd be great to perform the check earlier, e.g. when creating write buffer manager.
        // Unfortunately the check needs write buffer size that was just calculated.
        sanityCheckArenaBlockSize(options.writeBufferSize(), options.arenaBlockSize(), writeBufferManagerCapacity);
    }
    return new ColumnFamilyDescriptor(nameBytes, options);
}