Examples with DBOptions org.rocksdb.DBOptions used on opensource projects

Example 21 with DBOptions

use of org.rocksdb.DBOptions in project jstorm by alibaba.

the class RocksDBTest method visitorAccross.

public void visitorAccross() throws RocksDBException, InterruptedException {
    DBOptions dbOptions = null;
    TtlDB ttlDB = null;
    List<ColumnFamilyDescriptor> cfNames = new ArrayList<ColumnFamilyDescriptor>();
    List<ColumnFamilyHandle> columnFamilyHandleList = new ArrayList<ColumnFamilyHandle>();
    cfNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
    cfNames.add(new ColumnFamilyDescriptor("new_cf".getBytes()));
    List<Integer> ttlValues = new ArrayList<Integer>();
    // new column family with 1 second ttl
    ttlValues.add(1);
    // Default column family with infinite lifetime
    ttlValues.add(0);
    try {
        System.out.println("Begin to open db");
        dbOptions = new DBOptions().setCreateMissingColumnFamilies(true).setCreateIfMissing(true);
        ttlDB = TtlDB.open(dbOptions, rootDir, cfNames, columnFamilyHandleList, ttlValues, false);
        System.out.println("Successfully open db " + rootDir);
        List<String> keys = new ArrayList<String>();
        keys.add("key");
        ttlDB.put("key".getBytes(), "key".getBytes());
        for (int i = 0; i < 2; i++) {
            String key = "key" + i;
            keys.add(key);
            ttlDB.put(columnFamilyHandleList.get(i), key.getBytes(), key.getBytes());
        }
        try {
            byte[] value = ttlDB.get("others".getBytes());
            if (value != null) {
                System.out.println("Raw get :" + new String(value));
            } else {
                System.out.println("No value of other");
            }
        } catch (Exception e) {
            System.out.println("Occur exception other");
        }
        for (String key : keys) {
            try {
                byte[] value = ttlDB.get(key.getBytes());
                if (value != null) {
                    System.out.println("Raw get :" + new String(value));
                } else {
                    System.out.println("No value of " + key);
                }
            } catch (Exception e) {
                System.out.println("Occur exception " + key + ", Raw");
            }
            for (int i = 0; i < 2; i++) {
                try {
                    byte[] value = ttlDB.get(columnFamilyHandleList.get(i), key.getBytes());
                    if (value != null) {
                        System.out.println("handler index" + i + " get :" + new String(value));
                    } else {
                        System.out.println("No value of index" + i + " get :" + key);
                    }
                } catch (Exception e) {
                    System.out.println("Occur exception " + key + ", handler index:" + i);
                }
            }
        }
    } finally {
        for (ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) {
            columnFamilyHandle.dispose();
        }
        if (ttlDB != null) {
            ttlDB.close();
        }
        if (dbOptions != null) {
            dbOptions.dispose();
        }
    }
}

Example 22 with DBOptions

use of org.rocksdb.DBOptions in project kafka by apache.

the class RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapterTest method shouldLogWarningWhenSettingWalOptions.

@Test
public void shouldLogWarningWhenSettingWalOptions() throws Exception {
    try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.class)) {
        final RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter adapter = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(new DBOptions(), new ColumnFamilyOptions());
        for (final Method method : RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter.class.getDeclaredMethods()) {
            if (walRelatedMethods.contains(method.getName())) {
                method.invoke(adapter, getDBOptionsParameters(method.getParameterTypes()));
            }
        }
        final List<String> walOptions = Arrays.asList("walDir", "walFilter", "walRecoveryMode", "walBytesPerSync", "walSizeLimitMB", "manualWalFlush", "maxTotalWalSize", "walTtlSeconds");
        final Set<String> logMessages = appender.getEvents().stream().filter(e -> e.getLevel().equals("WARN")).map(LogCaptureAppender.Event::getMessage).collect(Collectors.toSet());
        walOptions.forEach(option -> assertThat(logMessages, hasItem(String.format("WAL is explicitly disabled by Streams in RocksDB. Setting option '%s' will be ignored", option))));
    }
}

Example 23 with DBOptions

use of org.rocksdb.DBOptions in project kafka by apache.

the class RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapterTest method verifyColumnFamilyOptionsMethodCall.

private void verifyColumnFamilyOptionsMethodCall(final Method method) throws Exception {
    final ColumnFamilyOptions mockedColumnFamilyOptions = mock(ColumnFamilyOptions.class);
    final RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter optionsFacadeColumnFamilyOptions = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(new DBOptions(), mockedColumnFamilyOptions);
    final Object[] parameters = getColumnFamilyOptionsParameters(method.getParameterTypes());
    try {
        reset(mockedColumnFamilyOptions);
        replay(mockedColumnFamilyOptions);
        method.invoke(optionsFacadeColumnFamilyOptions, parameters);
        verify();
        fail("Should have called ColumnFamilyOptions." + method.getName() + "()");
    } catch (final InvocationTargetException undeclaredMockMethodCall) {
        assertThat(undeclaredMockMethodCall.getCause(), instanceOf(AssertionError.class));
        assertThat(undeclaredMockMethodCall.getCause().getMessage().trim(), matchesPattern("Unexpected method call ColumnFamilyOptions\\." + method.getName() + "(.*)"));
    }
}

Example 24 with DBOptions

use of org.rocksdb.DBOptions in project kafka by apache.

the class RocksDBTimestampedStoreTest method shouldOpenExistingStoreInRegularMode.

@Test
public void shouldOpenExistingStoreInRegularMode() throws Exception {
    // prepare store
    rocksDBStore.init((StateStoreContext) context, rocksDBStore);
    rocksDBStore.put(new Bytes("key".getBytes()), "timestamped".getBytes());
    rocksDBStore.close();
    // re-open store
    try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(RocksDBTimestampedStore.class)) {
        rocksDBStore.init((StateStoreContext) context, rocksDBStore);
        assertThat(appender.getMessages(), hasItem("Opening store " + DB_NAME + " in regular mode"));
    } finally {
        rocksDBStore.close();
    }
    // verify store
    final DBOptions dbOptions = new DBOptions();
    final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions();
    final List<ColumnFamilyDescriptor> columnFamilyDescriptors = asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnFamilyOptions), new ColumnFamilyDescriptor("keyValueWithTimestamp".getBytes(StandardCharsets.UTF_8), columnFamilyOptions));
    final List<ColumnFamilyHandle> columnFamilies = new ArrayList<>(columnFamilyDescriptors.size());
    RocksDB db = null;
    ColumnFamilyHandle noTimestampColumnFamily = null, withTimestampColumnFamily = null;
    try {
        db = RocksDB.open(dbOptions, new File(new File(context.stateDir(), "rocksdb"), DB_NAME).getAbsolutePath(), columnFamilyDescriptors, columnFamilies);
        noTimestampColumnFamily = columnFamilies.get(0);
        withTimestampColumnFamily = columnFamilies.get(1);
        assertThat(db.get(noTimestampColumnFamily, "key".getBytes()), new IsNull<>());
        assertThat(db.getLongProperty(noTimestampColumnFamily, "rocksdb.estimate-num-keys"), is(0L));
        assertThat(db.get(withTimestampColumnFamily, "key".getBytes()).length, is(11));
        assertThat(db.getLongProperty(withTimestampColumnFamily, "rocksdb.estimate-num-keys"), is(1L));
    } finally {
        // Order of closing must follow: ColumnFamilyHandle > RocksDB > DBOptions > ColumnFamilyOptions
        if (noTimestampColumnFamily != null) {
            noTimestampColumnFamily.close();
        }
        if (withTimestampColumnFamily != null) {
            withTimestampColumnFamily.close();
        }
        if (db != null) {
            db.close();
        }
        dbOptions.close();
        columnFamilyOptions.close();
    }
}

Example 25 with DBOptions

use of org.rocksdb.DBOptions in project flink by apache.

the class RocksDBResourceContainer method getDbOptions.

/**
 * Gets the RocksDB {@link DBOptions} to be used for RocksDB instances.
 */
public DBOptions getDbOptions() {
    // initial options from common profile
    DBOptions opt = createBaseCommonDBOptions();
    handlesToClose.add(opt);
    // load configurable options on top of pre-defined profile
    setDBOptionsFromConfigurableOptions(opt);
    // add user-defined options factory, if specified
    if (optionsFactory != null) {
        opt = optionsFactory.createDBOptions(opt, handlesToClose);
    }
    // add necessary default options
    opt = opt.setCreateIfMissing(true);
    // if sharedResources is non-null, use the write buffer manager from it.
    if (sharedResources != null) {
        opt.setWriteBufferManager(sharedResources.getResourceHandle().getWriteBufferManager());
    }
    return opt;
}