Examples with LRUCache org.rocksdb.LRUCache used on opensource projects

Example 1 with LRUCache

use of org.rocksdb.LRUCache in project flink by apache.

the class RocksDBResourceContainerTest method testFreeSharedResourcesAfterClose.

@Test
public void testFreeSharedResourcesAfterClose() throws Exception {
    LRUCache cache = new LRUCache(1024L);
    WriteBufferManager wbm = new WriteBufferManager(1024L, cache);
    RocksDBSharedResources sharedResources = new RocksDBSharedResources(cache, wbm, 1024L, false);
    final ThrowingRunnable<Exception> disposer = sharedResources::close;
    OpaqueMemoryResource<RocksDBSharedResources> opaqueResource = new OpaqueMemoryResource<>(sharedResources, 1024L, disposer);
    RocksDBResourceContainer container = new RocksDBResourceContainer(PredefinedOptions.DEFAULT, null, opaqueResource);
    container.close();
    assertThat(cache.isOwningHandle(), is(false));
    assertThat(wbm.isOwningHandle(), is(false));
}

Example 2 with LRUCache

use of org.rocksdb.LRUCache in project flink by apache.

the class RocksDBResourceContainerTest method testGetColumnFamilyOptionsWithPartitionedIndex.

@Test
public void testGetColumnFamilyOptionsWithPartitionedIndex() throws Exception {
    LRUCache cache = new LRUCache(1024L);
    WriteBufferManager wbm = new WriteBufferManager(1024L, cache);
    RocksDBSharedResources sharedResources = new RocksDBSharedResources(cache, wbm, 1024L, true);
    final ThrowingRunnable<Exception> disposer = sharedResources::close;
    OpaqueMemoryResource<RocksDBSharedResources> opaqueResource = new OpaqueMemoryResource<>(sharedResources, 1024L, disposer);
    BloomFilter blockBasedFilter = new BloomFilter();
    RocksDBOptionsFactory blockBasedBloomFilterOptionFactory = new RocksDBOptionsFactory() {

        @Override
        public DBOptions createDBOptions(DBOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
            return currentOptions;
        }

        @Override
        public ColumnFamilyOptions createColumnOptions(ColumnFamilyOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
            TableFormatConfig tableFormatConfig = currentOptions.tableFormatConfig();
            BlockBasedTableConfig blockBasedTableConfig = tableFormatConfig == null ? new BlockBasedTableConfig() : (BlockBasedTableConfig) tableFormatConfig;
            blockBasedTableConfig.setFilter(blockBasedFilter);
            handlesToClose.add(blockBasedFilter);
            currentOptions.setTableFormatConfig(blockBasedTableConfig);
            return currentOptions;
        }
    };
    try (RocksDBResourceContainer container = new RocksDBResourceContainer(PredefinedOptions.DEFAULT, blockBasedBloomFilterOptionFactory, opaqueResource)) {
        ColumnFamilyOptions columnOptions = container.getColumnOptions();
        BlockBasedTableConfig actual = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
        assertThat(actual.indexType(), is(IndexType.kTwoLevelIndexSearch));
        assertThat(actual.partitionFilters(), is(true));
        assertThat(actual.pinTopLevelIndexAndFilter(), is(true));
        assertThat(actual.filterPolicy(), not(blockBasedFilter));
    }
    assertFalse("Block based filter is left unclosed.", blockBasedFilter.isOwningHandle());
}

Example 3 with LRUCache

use of org.rocksdb.LRUCache in project kafka by apache.

the class RocksDBStore method openDB.

@SuppressWarnings("unchecked")
void openDB(final Map<String, Object> configs, final File stateDir) {
    // initialize the default rocksdb options
    final DBOptions dbOptions = new DBOptions();
    final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions();
    userSpecifiedOptions = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(dbOptions, columnFamilyOptions);
    final BlockBasedTableConfigWithAccessibleCache tableConfig = new BlockBasedTableConfigWithAccessibleCache();
    cache = new LRUCache(BLOCK_CACHE_SIZE);
    tableConfig.setBlockCache(cache);
    tableConfig.setBlockSize(BLOCK_SIZE);
    filter = new BloomFilter();
    tableConfig.setFilterPolicy(filter);
    userSpecifiedOptions.optimizeFiltersForHits();
    userSpecifiedOptions.setTableFormatConfig(tableConfig);
    userSpecifiedOptions.setWriteBufferSize(WRITE_BUFFER_SIZE);
    userSpecifiedOptions.setCompressionType(COMPRESSION_TYPE);
    userSpecifiedOptions.setCompactionStyle(COMPACTION_STYLE);
    userSpecifiedOptions.setMaxWriteBufferNumber(MAX_WRITE_BUFFERS);
    userSpecifiedOptions.setCreateIfMissing(true);
    userSpecifiedOptions.setErrorIfExists(false);
    userSpecifiedOptions.setInfoLogLevel(InfoLogLevel.ERROR_LEVEL);
    // this is the recommended way to increase parallelism in RocksDb
    // note that the current implementation of setIncreaseParallelism affects the number
    // of compaction threads but not flush threads (the latter remains one). Also,
    // the parallelism value needs to be at least two because of the code in
    // https://github.com/facebook/rocksdb/blob/62ad0a9b19f0be4cefa70b6b32876e764b7f3c11/util/options.cc#L580
    // subtracts one from the value passed to determine the number of compaction threads
    // (this could be a bug in the RocksDB code and their devs have been contacted).
    userSpecifiedOptions.setIncreaseParallelism(Math.max(Runtime.getRuntime().availableProcessors(), 2));
    wOptions = new WriteOptions();
    wOptions.setDisableWAL(true);
    fOptions = new FlushOptions();
    fOptions.setWaitForFlush(true);
    final Class<RocksDBConfigSetter> configSetterClass = (Class<RocksDBConfigSetter>) configs.get(StreamsConfig.ROCKSDB_CONFIG_SETTER_CLASS_CONFIG);
    if (configSetterClass != null) {
        configSetter = Utils.newInstance(configSetterClass);
        configSetter.setConfig(name, userSpecifiedOptions, configs);
    }
    dbDir = new File(new File(stateDir, parentDir), name);
    try {
        Files.createDirectories(dbDir.getParentFile().toPath());
        Files.createDirectories(dbDir.getAbsoluteFile().toPath());
    } catch (final IOException fatal) {
        throw new ProcessorStateException(fatal);
    }
    // Setup statistics before the database is opened, otherwise the statistics are not updated
    // with the measurements from Rocks DB
    maybeSetUpStatistics(configs);
    openRocksDB(dbOptions, columnFamilyOptions);
    open = true;
    addValueProvidersToMetricsRecorder();
}

Example 4 with LRUCache

use of org.rocksdb.LRUCache in project kafka by apache.

the class BlockBasedTableConfigWithAccessibleCacheTest method shouldSetBlockCacheAndMakeItAccessible.

@Test
public void shouldSetBlockCacheAndMakeItAccessible() {
    final BlockBasedTableConfigWithAccessibleCache configWithAccessibleCache = new BlockBasedTableConfigWithAccessibleCache();
    final Cache blockCache = new LRUCache(1024);
    final BlockBasedTableConfig updatedConfig = configWithAccessibleCache.setBlockCache(blockCache);
    assertThat(updatedConfig, sameInstance(configWithAccessibleCache));
    assertThat(configWithAccessibleCache.blockCache(), sameInstance(blockCache));
}

Example 5 with LRUCache

use of org.rocksdb.LRUCache in project flink by apache.

the class RocksDBMemoryControllerUtilsTest method testCreateSharedResourcesWithExpectedCapacity.

@Test
public void testCreateSharedResourcesWithExpectedCapacity() {
    PowerMockito.mockStatic(RocksDBMemoryControllerUtils.class);
    final AtomicLong actualCacheCapacity = new AtomicLong(0L);
    final AtomicLong actualWbmCapacity = new AtomicLong(0L);
    when(RocksDBMemoryControllerUtils.allocateRocksDBSharedResources(anyLong(), anyDouble(), anyDouble(), anyBoolean())).thenCallRealMethod();
    when(RocksDBMemoryControllerUtils.calculateActualCacheCapacity(anyLong(), anyDouble())).thenCallRealMethod();
    when(RocksDBMemoryControllerUtils.calculateWriteBufferManagerCapacity(anyLong(), anyDouble())).thenCallRealMethod();
    // because PowerMockito cannot mock on native static method easily,
    // we introduce `createCache` and `createWriteBufferManager` wrappers here.
    when(RocksDBMemoryControllerUtils.createCache(anyLong(), anyDouble())).thenAnswer((Answer<LRUCache>) invocation -> {
        Object[] arguments = invocation.getArguments();
        actualCacheCapacity.set((long) arguments[0]);
        return (LRUCache) invocation.callRealMethod();
    });
    when(RocksDBMemoryControllerUtils.createWriteBufferManager(anyLong(), any(Cache.class))).thenAnswer((Answer<WriteBufferManager>) invocation -> {
        Object[] arguments = invocation.getArguments();
        actualWbmCapacity.set((long) arguments[0]);
        return (WriteBufferManager) invocation.callRealMethod();
    });
    long totalMemorySize = 2048L;
    double writeBufferRatio = 0.5;
    double highPriPoolRatio = 0.1;
    RocksDBSharedResources rocksDBSharedResources = RocksDBMemoryControllerUtils.allocateRocksDBSharedResources(totalMemorySize, writeBufferRatio, highPriPoolRatio, false);
    long expectedCacheCapacity = RocksDBMemoryControllerUtils.calculateActualCacheCapacity(totalMemorySize, writeBufferRatio);
    long expectedWbmCapacity = RocksDBMemoryControllerUtils.calculateWriteBufferManagerCapacity(totalMemorySize, writeBufferRatio);
    assertThat(actualCacheCapacity.get(), is(expectedCacheCapacity));
    assertThat(actualWbmCapacity.get(), is(expectedWbmCapacity));
    assertThat(rocksDBSharedResources.getWriteBufferManagerCapacity(), is(expectedWbmCapacity));
}