Examples with OpaqueMemoryResource org.apache.flink.runtime.memory.OpaqueMemoryResource used on opensource projects

Example 1 with OpaqueMemoryResource

use of org.apache.flink.runtime.memory.OpaqueMemoryResource in project flink by apache.

the class RocksDBOperationUtils method allocateSharedCachesIfConfigured.

@Nullable
public static OpaqueMemoryResource<RocksDBSharedResources> allocateSharedCachesIfConfigured(RocksDBMemoryConfiguration memoryConfig, MemoryManager memoryManager, double memoryFraction, Logger logger) throws IOException {
    if (!memoryConfig.isUsingFixedMemoryPerSlot() && !memoryConfig.isUsingManagedMemory()) {
        return null;
    }
    final double highPriorityPoolRatio = memoryConfig.getHighPriorityPoolRatio();
    final double writeBufferRatio = memoryConfig.getWriteBufferRatio();
    final boolean usingPartitionedIndexFilters = memoryConfig.isUsingPartitionedIndexFilters();
    final LongFunctionWithException<RocksDBSharedResources, Exception> allocator = (size) -> RocksDBMemoryControllerUtils.allocateRocksDBSharedResources(size, writeBufferRatio, highPriorityPoolRatio, usingPartitionedIndexFilters);
    try {
        if (memoryConfig.isUsingFixedMemoryPerSlot()) {
            assert memoryConfig.getFixedMemoryPerSlot() != null;
            logger.info("Getting fixed-size shared cache for RocksDB.");
            return memoryManager.getExternalSharedMemoryResource(FIXED_SLOT_MEMORY_RESOURCE_ID, allocator, memoryConfig.getFixedMemoryPerSlot().getBytes());
        } else {
            logger.info("Getting managed memory shared cache for RocksDB.");
            return memoryManager.getSharedMemoryResourceForManagedMemory(MANAGED_MEMORY_RESOURCE_ID, allocator, memoryFraction);
        }
    } catch (Exception e) {
        throw new IOException("Failed to acquire shared cache resource for RocksDB", e);
    }
}

Example 2 with OpaqueMemoryResource

use of org.apache.flink.runtime.memory.OpaqueMemoryResource in project flink by apache.

the class RocksDBResourceContainerTest method testFreeSharedResourcesAfterClose.

@Test
public void testFreeSharedResourcesAfterClose() throws Exception {
    LRUCache cache = new LRUCache(1024L);
    WriteBufferManager wbm = new WriteBufferManager(1024L, cache);
    RocksDBSharedResources sharedResources = new RocksDBSharedResources(cache, wbm, 1024L, false);
    final ThrowingRunnable<Exception> disposer = sharedResources::close;
    OpaqueMemoryResource<RocksDBSharedResources> opaqueResource = new OpaqueMemoryResource<>(sharedResources, 1024L, disposer);
    RocksDBResourceContainer container = new RocksDBResourceContainer(PredefinedOptions.DEFAULT, null, opaqueResource);
    container.close();
    assertThat(cache.isOwningHandle(), is(false));
    assertThat(wbm.isOwningHandle(), is(false));
}

Example 3 with OpaqueMemoryResource

use of org.apache.flink.runtime.memory.OpaqueMemoryResource in project flink by apache.

the class RocksDBResourceContainerTest method testGetColumnFamilyOptionsWithPartitionedIndex.

@Test
public void testGetColumnFamilyOptionsWithPartitionedIndex() throws Exception {
    LRUCache cache = new LRUCache(1024L);
    WriteBufferManager wbm = new WriteBufferManager(1024L, cache);
    RocksDBSharedResources sharedResources = new RocksDBSharedResources(cache, wbm, 1024L, true);
    final ThrowingRunnable<Exception> disposer = sharedResources::close;
    OpaqueMemoryResource<RocksDBSharedResources> opaqueResource = new OpaqueMemoryResource<>(sharedResources, 1024L, disposer);
    BloomFilter blockBasedFilter = new BloomFilter();
    RocksDBOptionsFactory blockBasedBloomFilterOptionFactory = new RocksDBOptionsFactory() {

        @Override
        public DBOptions createDBOptions(DBOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
            return currentOptions;
        }

        @Override
        public ColumnFamilyOptions createColumnOptions(ColumnFamilyOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
            TableFormatConfig tableFormatConfig = currentOptions.tableFormatConfig();
            BlockBasedTableConfig blockBasedTableConfig = tableFormatConfig == null ? new BlockBasedTableConfig() : (BlockBasedTableConfig) tableFormatConfig;
            blockBasedTableConfig.setFilter(blockBasedFilter);
            handlesToClose.add(blockBasedFilter);
            currentOptions.setTableFormatConfig(blockBasedTableConfig);
            return currentOptions;
        }
    };
    try (RocksDBResourceContainer container = new RocksDBResourceContainer(PredefinedOptions.DEFAULT, blockBasedBloomFilterOptionFactory, opaqueResource)) {
        ColumnFamilyOptions columnOptions = container.getColumnOptions();
        BlockBasedTableConfig actual = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
        assertThat(actual.indexType(), is(IndexType.kTwoLevelIndexSearch));
        assertThat(actual.partitionFilters(), is(true));
        assertThat(actual.pinTopLevelIndexAndFilter(), is(true));
        assertThat(actual.filterPolicy(), not(blockBasedFilter));
    }
    assertFalse("Block based filter is left unclosed.", blockBasedFilter.isOwningHandle());
}

Example 4 with OpaqueMemoryResource

use of org.apache.flink.runtime.memory.OpaqueMemoryResource in project flink by apache.

the class RocksDBResourceContainerTest method getSharedResources.

private OpaqueMemoryResource<RocksDBSharedResources> getSharedResources() {
    final long cacheSize = 1024L, writeBufferSize = 512L;
    final LRUCache cache = new LRUCache(cacheSize, -1, false, 0.1);
    final WriteBufferManager wbm = new WriteBufferManager(writeBufferSize, cache);
    RocksDBSharedResources rocksDBSharedResources = new RocksDBSharedResources(cache, wbm, writeBufferSize, false);
    return new OpaqueMemoryResource<>(rocksDBSharedResources, cacheSize, rocksDBSharedResources::close);
}