Examples with WriteBufferManager org.rocksdb.WriteBufferManager used on opensource projects

Example 1 with WriteBufferManager

use of org.rocksdb.WriteBufferManager in project flink by apache.

the class RocksDBMemoryControllerUtils method allocateRocksDBSharedResources.

/**
 * Allocate memory controllable RocksDB shared resources.
 *
 * @param totalMemorySize The total memory limit size.
 * @param writeBufferRatio The ratio of total memory which is occupied by write buffer manager.
 * @param highPriorityPoolRatio The high priority pool ratio of cache.
 * @return memory controllable RocksDB shared resources.
 */
public static RocksDBSharedResources allocateRocksDBSharedResources(long totalMemorySize, double writeBufferRatio, double highPriorityPoolRatio, boolean usingPartitionedIndexFilters) {
    long calculatedCacheCapacity = RocksDBMemoryControllerUtils.calculateActualCacheCapacity(totalMemorySize, writeBufferRatio);
    final Cache cache = RocksDBMemoryControllerUtils.createCache(calculatedCacheCapacity, highPriorityPoolRatio);
    long writeBufferManagerCapacity = RocksDBMemoryControllerUtils.calculateWriteBufferManagerCapacity(totalMemorySize, writeBufferRatio);
    final WriteBufferManager wbm = RocksDBMemoryControllerUtils.createWriteBufferManager(writeBufferManagerCapacity, cache);
    return new RocksDBSharedResources(cache, wbm, writeBufferManagerCapacity, usingPartitionedIndexFilters);
}

Example 2 with WriteBufferManager

use of org.rocksdb.WriteBufferManager in project flink by apache.

the class RocksDBResourceContainerTest method testFreeSharedResourcesAfterClose.

@Test
public void testFreeSharedResourcesAfterClose() throws Exception {
    LRUCache cache = new LRUCache(1024L);
    WriteBufferManager wbm = new WriteBufferManager(1024L, cache);
    RocksDBSharedResources sharedResources = new RocksDBSharedResources(cache, wbm, 1024L, false);
    final ThrowingRunnable<Exception> disposer = sharedResources::close;
    OpaqueMemoryResource<RocksDBSharedResources> opaqueResource = new OpaqueMemoryResource<>(sharedResources, 1024L, disposer);
    RocksDBResourceContainer container = new RocksDBResourceContainer(PredefinedOptions.DEFAULT, null, opaqueResource);
    container.close();
    assertThat(cache.isOwningHandle(), is(false));
    assertThat(wbm.isOwningHandle(), is(false));
}

Example 3 with WriteBufferManager

use of org.rocksdb.WriteBufferManager in project flink by apache.

the class RocksDBResourceContainerTest method testGetColumnFamilyOptionsWithPartitionedIndex.

@Test
public void testGetColumnFamilyOptionsWithPartitionedIndex() throws Exception {
    LRUCache cache = new LRUCache(1024L);
    WriteBufferManager wbm = new WriteBufferManager(1024L, cache);
    RocksDBSharedResources sharedResources = new RocksDBSharedResources(cache, wbm, 1024L, true);
    final ThrowingRunnable<Exception> disposer = sharedResources::close;
    OpaqueMemoryResource<RocksDBSharedResources> opaqueResource = new OpaqueMemoryResource<>(sharedResources, 1024L, disposer);
    BloomFilter blockBasedFilter = new BloomFilter();
    RocksDBOptionsFactory blockBasedBloomFilterOptionFactory = new RocksDBOptionsFactory() {

        @Override
        public DBOptions createDBOptions(DBOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
            return currentOptions;
        }

        @Override
        public ColumnFamilyOptions createColumnOptions(ColumnFamilyOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
            TableFormatConfig tableFormatConfig = currentOptions.tableFormatConfig();
            BlockBasedTableConfig blockBasedTableConfig = tableFormatConfig == null ? new BlockBasedTableConfig() : (BlockBasedTableConfig) tableFormatConfig;
            blockBasedTableConfig.setFilter(blockBasedFilter);
            handlesToClose.add(blockBasedFilter);
            currentOptions.setTableFormatConfig(blockBasedTableConfig);
            return currentOptions;
        }
    };
    try (RocksDBResourceContainer container = new RocksDBResourceContainer(PredefinedOptions.DEFAULT, blockBasedBloomFilterOptionFactory, opaqueResource)) {
        ColumnFamilyOptions columnOptions = container.getColumnOptions();
        BlockBasedTableConfig actual = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
        assertThat(actual.indexType(), is(IndexType.kTwoLevelIndexSearch));
        assertThat(actual.partitionFilters(), is(true));
        assertThat(actual.pinTopLevelIndexAndFilter(), is(true));
        assertThat(actual.filterPolicy(), not(blockBasedFilter));
    }
    assertFalse("Block based filter is left unclosed.", blockBasedFilter.isOwningHandle());
}

Example 4 with WriteBufferManager

use of org.rocksdb.WriteBufferManager in project flink by apache.

the class RocksDBMemoryControllerUtilsTest method testCreateSharedResourcesWithExpectedCapacity.

@Test
public void testCreateSharedResourcesWithExpectedCapacity() {
    PowerMockito.mockStatic(RocksDBMemoryControllerUtils.class);
    final AtomicLong actualCacheCapacity = new AtomicLong(0L);
    final AtomicLong actualWbmCapacity = new AtomicLong(0L);
    when(RocksDBMemoryControllerUtils.allocateRocksDBSharedResources(anyLong(), anyDouble(), anyDouble(), anyBoolean())).thenCallRealMethod();
    when(RocksDBMemoryControllerUtils.calculateActualCacheCapacity(anyLong(), anyDouble())).thenCallRealMethod();
    when(RocksDBMemoryControllerUtils.calculateWriteBufferManagerCapacity(anyLong(), anyDouble())).thenCallRealMethod();
    // because PowerMockito cannot mock on native static method easily,
    // we introduce `createCache` and `createWriteBufferManager` wrappers here.
    when(RocksDBMemoryControllerUtils.createCache(anyLong(), anyDouble())).thenAnswer((Answer<LRUCache>) invocation -> {
        Object[] arguments = invocation.getArguments();
        actualCacheCapacity.set((long) arguments[0]);
        return (LRUCache) invocation.callRealMethod();
    });
    when(RocksDBMemoryControllerUtils.createWriteBufferManager(anyLong(), any(Cache.class))).thenAnswer((Answer<WriteBufferManager>) invocation -> {
        Object[] arguments = invocation.getArguments();
        actualWbmCapacity.set((long) arguments[0]);
        return (WriteBufferManager) invocation.callRealMethod();
    });
    long totalMemorySize = 2048L;
    double writeBufferRatio = 0.5;
    double highPriPoolRatio = 0.1;
    RocksDBSharedResources rocksDBSharedResources = RocksDBMemoryControllerUtils.allocateRocksDBSharedResources(totalMemorySize, writeBufferRatio, highPriPoolRatio, false);
    long expectedCacheCapacity = RocksDBMemoryControllerUtils.calculateActualCacheCapacity(totalMemorySize, writeBufferRatio);
    long expectedWbmCapacity = RocksDBMemoryControllerUtils.calculateWriteBufferManagerCapacity(totalMemorySize, writeBufferRatio);
    assertThat(actualCacheCapacity.get(), is(expectedCacheCapacity));
    assertThat(actualWbmCapacity.get(), is(expectedWbmCapacity));
    assertThat(rocksDBSharedResources.getWriteBufferManagerCapacity(), is(expectedWbmCapacity));
}

Example 5 with WriteBufferManager

use of org.rocksdb.WriteBufferManager in project flink by apache.

the class RocksDBResourceContainerTest method getSharedResources.

private OpaqueMemoryResource<RocksDBSharedResources> getSharedResources() {
    final long cacheSize = 1024L, writeBufferSize = 512L;
    final LRUCache cache = new LRUCache(cacheSize, -1, false, 0.1);
    final WriteBufferManager wbm = new WriteBufferManager(writeBufferSize, cache);
    RocksDBSharedResources rocksDBSharedResources = new RocksDBSharedResources(cache, wbm, writeBufferSize, false);
    return new OpaqueMemoryResource<>(rocksDBSharedResources, cacheSize, rocksDBSharedResources::close);
}