Examples with RocksIterator org.rocksdb.RocksIterator used on opensource projects

Example 1 with RocksIterator

use of org.rocksdb.RocksIterator in project kafka by apache.

the class RocksDBStore method all.

@Override
public synchronized KeyValueIterator<K, V> all() {
    validateStoreOpen();
    // query rocksdb
    RocksIterator innerIter = db.newIterator();
    innerIter.seekToFirst();
    final RocksDbIterator rocksDbIterator = new RocksDbIterator(name, innerIter, serdes);
    openIterators.add(rocksDbIterator);
    return rocksDbIterator;
}

Example 2 with RocksIterator

use of org.rocksdb.RocksIterator in project flink by apache.

the class RocksDBStateBackendTest method setupRocksKeyedStateBackend.

public void setupRocksKeyedStateBackend() throws Exception {
    blocker = new OneShotLatch();
    waiter = new OneShotLatch();
    testStreamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
    testStreamFactory.setBlockerLatch(blocker);
    testStreamFactory.setWaiterLatch(waiter);
    testStreamFactory.setAfterNumberInvocations(100);
    RocksDBStateBackend backend = getStateBackend();
    Environment env = new DummyEnvironment("TestTask", 1, 0);
    keyedStateBackend = (RocksDBKeyedStateBackend<Integer>) backend.createKeyedStateBackend(env, new JobID(), "Test", IntSerializer.INSTANCE, 2, new KeyGroupRange(0, 1), mock(TaskKvStateRegistry.class));
    testState1 = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("TestState-1", Integer.class, 0));
    testState2 = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new ValueStateDescriptor<>("TestState-2", String.class, ""));
    allCreatedCloseables = new ArrayList<>();
    keyedStateBackend.db = spy(keyedStateBackend.db);
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            RocksIterator rocksIterator = spy((RocksIterator) invocationOnMock.callRealMethod());
            allCreatedCloseables.add(rocksIterator);
            return rocksIterator;
        }
    }).when(keyedStateBackend.db).newIterator(any(ColumnFamilyHandle.class), any(ReadOptions.class));
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            Snapshot snapshot = spy((Snapshot) invocationOnMock.callRealMethod());
            allCreatedCloseables.add(snapshot);
            return snapshot;
        }
    }).when(keyedStateBackend.db).getSnapshot();
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ColumnFamilyHandle snapshot = spy((ColumnFamilyHandle) invocationOnMock.callRealMethod());
            allCreatedCloseables.add(snapshot);
            return snapshot;
        }
    }).when(keyedStateBackend.db).createColumnFamily(any(ColumnFamilyDescriptor.class));
    for (int i = 0; i < 100; ++i) {
        keyedStateBackend.setCurrentKey(i);
        testState1.update(4200 + i);
        testState2.update("S-" + (4200 + i));
    }
}

Example 3 with RocksIterator

use of org.rocksdb.RocksIterator in project flink by apache.

the class ListViaRangeSpeedMiniBenchmark method main.

public static void main(String[] args) throws Exception {
    final File rocksDir = new File("/tmp/rdb");
    FileUtils.deleteDirectory(rocksDir);
    final Options options = new Options().setCompactionStyle(CompactionStyle.LEVEL).setLevelCompactionDynamicLevelBytes(true).setIncreaseParallelism(4).setUseFsync(false).setMaxOpenFiles(-1).setDisableDataSync(true).setCreateIfMissing(true).setMergeOperator(new StringAppendOperator());
    final WriteOptions write_options = new WriteOptions().setSync(false).setDisableWAL(true);
    final RocksDB rocksDB = RocksDB.open(options, rocksDir.getAbsolutePath());
    final String key = "key";
    final String value = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ7890654321";
    final byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
    final byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
    final byte[] keyTemplate = Arrays.copyOf(keyBytes, keyBytes.length + 4);
    final Unsafe unsafe = MemoryUtils.UNSAFE;
    final long offset = unsafe.arrayBaseOffset(byte[].class) + keyTemplate.length - 4;
    final int num = 50000;
    System.out.println("begin insert");
    final long beginInsert = System.nanoTime();
    for (int i = 0; i < num; i++) {
        unsafe.putInt(keyTemplate, offset, i);
        rocksDB.put(write_options, keyTemplate, valueBytes);
    }
    final long endInsert = System.nanoTime();
    System.out.println("end insert - duration: " + ((endInsert - beginInsert) / 1_000_000) + " ms");
    final byte[] resultHolder = new byte[num * valueBytes.length];
    final long beginGet = System.nanoTime();
    final RocksIterator iterator = rocksDB.newIterator();
    int pos = 0;
    // seek to start
    unsafe.putInt(keyTemplate, offset, 0);
    iterator.seek(keyTemplate);
    // mark end
    unsafe.putInt(keyTemplate, offset, -1);
    // iterate
    while (iterator.isValid()) {
        byte[] currKey = iterator.key();
        if (samePrefix(keyBytes, currKey)) {
            byte[] currValue = iterator.value();
            System.arraycopy(currValue, 0, resultHolder, pos, currValue.length);
            pos += currValue.length;
            iterator.next();
        } else {
            break;
        }
    }
    final long endGet = System.nanoTime();
    System.out.println("end get - duration: " + ((endGet - beginGet) / 1_000_000) + " ms");
}