Examples with ColumnFamilyDescriptor org.rocksdb.ColumnFamilyDescriptor used on opensource projects

Example 26 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project flink by apache.

the class RocksDBWriteBatchWrapperTest method testWriteBatchWrapperFlushAfterCountExceed.

/**
 * Tests that {@link RocksDBWriteBatchWrapper} flushes after the kv count exceeds the
 * preconfigured value.
 */
@Test
public void testWriteBatchWrapperFlushAfterCountExceed() throws Exception {
    try (RocksDB db = RocksDB.open(folder.newFolder().getAbsolutePath());
        WriteOptions options = new WriteOptions().setDisableWAL(true);
        ColumnFamilyHandle handle = db.createColumnFamily(new ColumnFamilyDescriptor("test".getBytes()));
        RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(db, options, 100, 50000)) {
        long initBatchSize = writeBatchWrapper.getDataSize();
        byte[] dummy = new byte[2];
        ThreadLocalRandom.current().nextBytes(dummy);
        for (int i = 1; i < 100; ++i) {
            writeBatchWrapper.put(handle, dummy, dummy);
            // each kv consumes 8 bytes
            assertEquals(initBatchSize + 8 * i, writeBatchWrapper.getDataSize());
        }
        writeBatchWrapper.put(handle, dummy, dummy);
        assertEquals(initBatchSize, writeBatchWrapper.getDataSize());
    }
}

Example 27 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project flink by apache.

the class RocksDBIncrementalCheckpointUtilsTest method testClipDBWithKeyGroupRangeHelper.

private void testClipDBWithKeyGroupRangeHelper(KeyGroupRange targetGroupRange, KeyGroupRange currentGroupRange, int keyGroupPrefixBytes) throws RocksDBException, IOException {
    try (RocksDB rocksDB = RocksDB.open(tmp.newFolder().getAbsolutePath());
        ColumnFamilyHandle columnFamilyHandle = rocksDB.createColumnFamily(new ColumnFamilyDescriptor("test".getBytes()))) {
        int currentGroupRangeStart = currentGroupRange.getStartKeyGroup();
        int currentGroupRangeEnd = currentGroupRange.getEndKeyGroup();
        DataOutputSerializer outputView = new DataOutputSerializer(32);
        for (int i = currentGroupRangeStart; i <= currentGroupRangeEnd; ++i) {
            for (int j = 0; j < 100; ++j) {
                outputView.clear();
                CompositeKeySerializationUtils.writeKeyGroup(i, keyGroupPrefixBytes, outputView);
                CompositeKeySerializationUtils.writeKey(j, IntSerializer.INSTANCE, outputView, false);
                rocksDB.put(columnFamilyHandle, outputView.getCopyOfBuffer(), String.valueOf(j).getBytes());
            }
        }
        for (int i = currentGroupRangeStart; i <= currentGroupRangeEnd; ++i) {
            for (int j = 0; j < 100; ++j) {
                outputView.clear();
                CompositeKeySerializationUtils.writeKeyGroup(i, keyGroupPrefixBytes, outputView);
                CompositeKeySerializationUtils.writeKey(j, IntSerializer.INSTANCE, outputView, false);
                byte[] value = rocksDB.get(columnFamilyHandle, outputView.getCopyOfBuffer());
                Assert.assertEquals(String.valueOf(j), new String(value));
            }
        }
        RocksDBIncrementalCheckpointUtils.clipDBWithKeyGroupRange(rocksDB, Collections.singletonList(columnFamilyHandle), targetGroupRange, currentGroupRange, keyGroupPrefixBytes, RocksDBConfigurableOptions.WRITE_BATCH_SIZE.defaultValue().getBytes());
        for (int i = currentGroupRangeStart; i <= currentGroupRangeEnd; ++i) {
            for (int j = 0; j < 100; ++j) {
                outputView.clear();
                CompositeKeySerializationUtils.writeKeyGroup(i, keyGroupPrefixBytes, outputView);
                CompositeKeySerializationUtils.writeKey(j, IntSerializer.INSTANCE, outputView, false);
                byte[] value = rocksDB.get(columnFamilyHandle, outputView.getCopyOfBuffer());
                if (targetGroupRange.contains(i)) {
                    Assert.assertEquals(String.valueOf(j), new String(value));
                } else {
                    Assert.assertNull(value);
                }
            }
        }
    }
}

Example 28 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project flink by apache.

the class RocksDBResource method createNewColumnFamily.

/**
 * Creates and returns a new column family with the given name.
 */
public ColumnFamilyHandle createNewColumnFamily(String name) {
    try {
        final ColumnFamilyHandle columnFamily = rocksDB.createColumnFamily(new ColumnFamilyDescriptor(name.getBytes(), columnFamilyOptions));
        columnFamilyHandles.add(columnFamily);
        return columnFamily;
    } catch (Exception ex) {
        throw new FlinkRuntimeException("Could not create column family.", ex);
    }
}

Example 29 with ColumnFamilyDescriptor

use of org.rocksdb.ColumnFamilyDescriptor in project flink by apache.

the class RocksDBIncrementalRestoreOperation method restoreDBInstanceFromStateHandle.

private RestoredDBInstance restoreDBInstanceFromStateHandle(IncrementalRemoteKeyedStateHandle restoreStateHandle, Path temporaryRestoreInstancePath) throws Exception {
    try (RocksDBStateDownloader rocksDBStateDownloader = new RocksDBStateDownloader(numberOfTransferringThreads)) {
        rocksDBStateDownloader.transferAllStateDataToDirectory(restoreStateHandle, temporaryRestoreInstancePath, cancelStreamRegistry);
    }
    KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(restoreStateHandle.getMetaStateHandle());
    // read meta data
    List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = serializationProxy.getStateMetaInfoSnapshots();
    List<ColumnFamilyDescriptor> columnFamilyDescriptors = createColumnFamilyDescriptors(stateMetaInfoSnapshots, false);
    List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(stateMetaInfoSnapshots.size() + 1);
    RocksDB restoreDb = RocksDBOperationUtils.openDB(temporaryRestoreInstancePath.toString(), columnFamilyDescriptors, columnFamilyHandles, RocksDBOperationUtils.createColumnFamilyOptions(this.rocksHandle.getColumnFamilyOptionsFactory(), "default"), this.rocksHandle.getDbOptions());
    return new RestoredDBInstance(restoreDb, columnFamilyHandles, columnFamilyDescriptors, stateMetaInfoSnapshots);
}