Examples with Cache org.rocksdb.Cache used on opensource projects

Example 1 with Cache

use of org.rocksdb.Cache in project flink by apache.

the class RocksDBMemoryControllerUtils method allocateRocksDBSharedResources.

/**
 * Allocate memory controllable RocksDB shared resources.
 *
 * @param totalMemorySize The total memory limit size.
 * @param writeBufferRatio The ratio of total memory which is occupied by write buffer manager.
 * @param highPriorityPoolRatio The high priority pool ratio of cache.
 * @return memory controllable RocksDB shared resources.
 */
public static RocksDBSharedResources allocateRocksDBSharedResources(long totalMemorySize, double writeBufferRatio, double highPriorityPoolRatio, boolean usingPartitionedIndexFilters) {
    long calculatedCacheCapacity = RocksDBMemoryControllerUtils.calculateActualCacheCapacity(totalMemorySize, writeBufferRatio);
    final Cache cache = RocksDBMemoryControllerUtils.createCache(calculatedCacheCapacity, highPriorityPoolRatio);
    long writeBufferManagerCapacity = RocksDBMemoryControllerUtils.calculateWriteBufferManagerCapacity(totalMemorySize, writeBufferRatio);
    final WriteBufferManager wbm = RocksDBMemoryControllerUtils.createWriteBufferManager(writeBufferManagerCapacity, cache);
    return new RocksDBSharedResources(cache, wbm, writeBufferManagerCapacity, usingPartitionedIndexFilters);
}

Example 2 with Cache

use of org.rocksdb.Cache in project flink by apache.

the class RocksDBResourceContainerTest method testGetColumnFamilyOptionsWithSharedResources.

/**
 * Guard that {@link RocksDBResourceContainer#getColumnOptions()} shares the same {@link Cache}
 * instance if the {@link RocksDBResourceContainer} instance is initiated with {@link
 * OpaqueMemoryResource}.
 *
 * @throws Exception if unexpected error happened.
 */
@Test
public void testGetColumnFamilyOptionsWithSharedResources() throws Exception {
    final int optionNumber = 20;
    OpaqueMemoryResource<RocksDBSharedResources> sharedResources = getSharedResources();
    RocksDBResourceContainer container = new RocksDBResourceContainer(PredefinedOptions.DEFAULT, null, sharedResources);
    HashSet<Cache> caches = new HashSet<>();
    for (int i = 0; i < optionNumber; i++) {
        ColumnFamilyOptions columnOptions = container.getColumnOptions();
        Cache cache = getBlockCache(columnOptions);
        caches.add(cache);
    }
    assertThat(caches.size(), is(1));
    assertThat(caches.iterator().next(), is(sharedResources.getResourceHandle().getCache()));
    container.close();
}

Example 3 with Cache

use of org.rocksdb.Cache in project kafka by apache.

the class BlockBasedTableConfigWithAccessibleCacheTest method shouldSetBlockCacheAndMakeItAccessible.

@Test
public void shouldSetBlockCacheAndMakeItAccessible() {
    final BlockBasedTableConfigWithAccessibleCache configWithAccessibleCache = new BlockBasedTableConfigWithAccessibleCache();
    final Cache blockCache = new LRUCache(1024);
    final BlockBasedTableConfig updatedConfig = configWithAccessibleCache.setBlockCache(blockCache);
    assertThat(updatedConfig, sameInstance(configWithAccessibleCache));
    assertThat(configWithAccessibleCache.blockCache(), sameInstance(blockCache));
}

Example 4 with Cache

use of org.rocksdb.Cache in project kafka by apache.

the class RocksDBStore method addValueProvidersToMetricsRecorder.

private void addValueProvidersToMetricsRecorder() {
    final TableFormatConfig tableFormatConfig = userSpecifiedOptions.tableFormatConfig();
    final Statistics statistics = userSpecifiedStatistics ? null : userSpecifiedOptions.statistics();
    if (tableFormatConfig instanceof BlockBasedTableConfigWithAccessibleCache) {
        final Cache cache = ((BlockBasedTableConfigWithAccessibleCache) tableFormatConfig).blockCache();
        metricsRecorder.addValueProviders(name, db, cache, statistics);
    } else if (tableFormatConfig instanceof BlockBasedTableConfig) {
        throw new ProcessorStateException("The used block-based table format configuration does not expose the " + "block cache. Use the BlockBasedTableConfig instance provided by Options#tableFormatConfig() to configure " + "the block-based table format of RocksDB. Do not provide a new instance of BlockBasedTableConfig to " + "the RocksDB options.");
    } else {
        metricsRecorder.addValueProviders(name, db, null, statistics);
    }
}

Example 5 with Cache

use of org.rocksdb.Cache in project flink by apache.

the class RocksDBResourceContainer method getColumnOptions.

/**
 * Gets the RocksDB {@link ColumnFamilyOptions} to be used for all RocksDB instances.
 */
public ColumnFamilyOptions getColumnOptions() {
    // initial options from common profile
    ColumnFamilyOptions opt = createBaseCommonColumnOptions();
    handlesToClose.add(opt);
    // load configurable options on top of pre-defined profile
    setColumnFamilyOptionsFromConfigurableOptions(opt, handlesToClose);
    // add user-defined options, if specified
    if (optionsFactory != null) {
        opt = optionsFactory.createColumnOptions(opt, handlesToClose);
    }
    // set necessary options for performance consideration with memory control
    if (sharedResources != null) {
        final RocksDBSharedResources rocksResources = sharedResources.getResourceHandle();
        final Cache blockCache = rocksResources.getCache();
        TableFormatConfig tableFormatConfig = opt.tableFormatConfig();
        BlockBasedTableConfig blockBasedTableConfig;
        if (tableFormatConfig == null) {
            blockBasedTableConfig = new BlockBasedTableConfig();
        } else {
            Preconditions.checkArgument(tableFormatConfig instanceof BlockBasedTableConfig, "We currently only support BlockBasedTableConfig When bounding total memory.");
            blockBasedTableConfig = (BlockBasedTableConfig) tableFormatConfig;
        }
        if (rocksResources.isUsingPartitionedIndexFilters() && overwriteFilterIfExist(blockBasedTableConfig)) {
            blockBasedTableConfig.setIndexType(IndexType.kTwoLevelIndexSearch);
            blockBasedTableConfig.setPartitionFilters(true);
            blockBasedTableConfig.setPinTopLevelIndexAndFilter(true);
        }
        blockBasedTableConfig.setBlockCache(blockCache);
        blockBasedTableConfig.setCacheIndexAndFilterBlocks(true);
        blockBasedTableConfig.setCacheIndexAndFilterBlocksWithHighPriority(true);
        blockBasedTableConfig.setPinL0FilterAndIndexBlocksInCache(true);
        opt.setTableFormatConfig(blockBasedTableConfig);
    }
    return opt;
}