Examples with TableFormatConfig org.rocksdb.TableFormatConfig used on opensource projects

Example 1 with TableFormatConfig

use of org.rocksdb.TableFormatConfig in project flink by apache.

the class RocksDBResourceContainerTest method testGetColumnFamilyOptionsWithPartitionedIndex.

@Test
public void testGetColumnFamilyOptionsWithPartitionedIndex() throws Exception {
    LRUCache cache = new LRUCache(1024L);
    WriteBufferManager wbm = new WriteBufferManager(1024L, cache);
    RocksDBSharedResources sharedResources = new RocksDBSharedResources(cache, wbm, 1024L, true);
    final ThrowingRunnable<Exception> disposer = sharedResources::close;
    OpaqueMemoryResource<RocksDBSharedResources> opaqueResource = new OpaqueMemoryResource<>(sharedResources, 1024L, disposer);
    BloomFilter blockBasedFilter = new BloomFilter();
    RocksDBOptionsFactory blockBasedBloomFilterOptionFactory = new RocksDBOptionsFactory() {

        @Override
        public DBOptions createDBOptions(DBOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
            return currentOptions;
        }

        @Override
        public ColumnFamilyOptions createColumnOptions(ColumnFamilyOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
            TableFormatConfig tableFormatConfig = currentOptions.tableFormatConfig();
            BlockBasedTableConfig blockBasedTableConfig = tableFormatConfig == null ? new BlockBasedTableConfig() : (BlockBasedTableConfig) tableFormatConfig;
            blockBasedTableConfig.setFilter(blockBasedFilter);
            handlesToClose.add(blockBasedFilter);
            currentOptions.setTableFormatConfig(blockBasedTableConfig);
            return currentOptions;
        }
    };
    try (RocksDBResourceContainer container = new RocksDBResourceContainer(PredefinedOptions.DEFAULT, blockBasedBloomFilterOptionFactory, opaqueResource)) {
        ColumnFamilyOptions columnOptions = container.getColumnOptions();
        BlockBasedTableConfig actual = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
        assertThat(actual.indexType(), is(IndexType.kTwoLevelIndexSearch));
        assertThat(actual.partitionFilters(), is(true));
        assertThat(actual.pinTopLevelIndexAndFilter(), is(true));
        assertThat(actual.filterPolicy(), not(blockBasedFilter));
    }
    assertFalse("Block based filter is left unclosed.", blockBasedFilter.isOwningHandle());
}

Example 2 with TableFormatConfig

use of org.rocksdb.TableFormatConfig in project kafka by apache.

the class RocksDBStore method addValueProvidersToMetricsRecorder.

private void addValueProvidersToMetricsRecorder() {
    final TableFormatConfig tableFormatConfig = userSpecifiedOptions.tableFormatConfig();
    final Statistics statistics = userSpecifiedStatistics ? null : userSpecifiedOptions.statistics();
    if (tableFormatConfig instanceof BlockBasedTableConfigWithAccessibleCache) {
        final Cache cache = ((BlockBasedTableConfigWithAccessibleCache) tableFormatConfig).blockCache();
        metricsRecorder.addValueProviders(name, db, cache, statistics);
    } else if (tableFormatConfig instanceof BlockBasedTableConfig) {
        throw new ProcessorStateException("The used block-based table format configuration does not expose the " + "block cache. Use the BlockBasedTableConfig instance provided by Options#tableFormatConfig() to configure " + "the block-based table format of RocksDB. Do not provide a new instance of BlockBasedTableConfig to " + "the RocksDB options.");
    } else {
        metricsRecorder.addValueProviders(name, db, null, statistics);
    }
}

Example 3 with TableFormatConfig

use of org.rocksdb.TableFormatConfig in project flink by apache.

the class DefaultConfigurableOptionsFactory method createColumnOptions.

@Override
public ColumnFamilyOptions createColumnOptions(ColumnFamilyOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
    if (isOptionConfigured(COMPACTION_STYLE)) {
        currentOptions.setCompactionStyle(getCompactionStyle());
    }
    if (isOptionConfigured(USE_DYNAMIC_LEVEL_SIZE)) {
        currentOptions.setLevelCompactionDynamicLevelBytes(getUseDynamicLevelSize());
    }
    if (isOptionConfigured(TARGET_FILE_SIZE_BASE)) {
        currentOptions.setTargetFileSizeBase(getTargetFileSizeBase());
    }
    if (isOptionConfigured(MAX_SIZE_LEVEL_BASE)) {
        currentOptions.setMaxBytesForLevelBase(getMaxSizeLevelBase());
    }
    if (isOptionConfigured(WRITE_BUFFER_SIZE)) {
        currentOptions.setWriteBufferSize(getWriteBufferSize());
    }
    if (isOptionConfigured(MAX_WRITE_BUFFER_NUMBER)) {
        currentOptions.setMaxWriteBufferNumber(getMaxWriteBufferNumber());
    }
    if (isOptionConfigured(MIN_WRITE_BUFFER_NUMBER_TO_MERGE)) {
        currentOptions.setMinWriteBufferNumberToMerge(getMinWriteBufferNumberToMerge());
    }
    TableFormatConfig tableFormatConfig = currentOptions.tableFormatConfig();
    BlockBasedTableConfig blockBasedTableConfig;
    if (tableFormatConfig == null) {
        blockBasedTableConfig = new BlockBasedTableConfig();
    } else {
        if (tableFormatConfig instanceof PlainTableConfig) {
            // column-family options
            return currentOptions;
        } else {
            blockBasedTableConfig = (BlockBasedTableConfig) tableFormatConfig;
        }
    }
    if (isOptionConfigured(BLOCK_SIZE)) {
        blockBasedTableConfig.setBlockSize(getBlockSize());
    }
    if (isOptionConfigured(METADATA_BLOCK_SIZE)) {
        blockBasedTableConfig.setMetadataBlockSize(getMetadataBlockSize());
    }
    if (isOptionConfigured(BLOCK_CACHE_SIZE)) {
        blockBasedTableConfig.setBlockCacheSize(getBlockCacheSize());
    }
    if (isOptionConfigured(USE_BLOOM_FILTER)) {
        final boolean enabled = Boolean.parseBoolean(getInternal(USE_BLOOM_FILTER.key()));
        if (enabled) {
            final double bitsPerKey = isOptionConfigured(BLOOM_FILTER_BITS_PER_KEY) ? Double.parseDouble(getInternal(BLOOM_FILTER_BITS_PER_KEY.key())) : BLOOM_FILTER_BITS_PER_KEY.defaultValue();
            final boolean blockBasedMode = isOptionConfigured(BLOOM_FILTER_BLOCK_BASED_MODE) ? Boolean.parseBoolean(getInternal(BLOOM_FILTER_BLOCK_BASED_MODE.key())) : BLOOM_FILTER_BLOCK_BASED_MODE.defaultValue();
            BloomFilter bloomFilter = new BloomFilter(bitsPerKey, blockBasedMode);
            handlesToClose.add(bloomFilter);
            blockBasedTableConfig.setFilterPolicy(bloomFilter);
        }
    }
    return currentOptions.setTableFormatConfig(blockBasedTableConfig);
}

Example 4 with TableFormatConfig

use of org.rocksdb.TableFormatConfig in project flink by apache.

the class RocksDBResourceContainer method getColumnOptions.

/**
 * Gets the RocksDB {@link ColumnFamilyOptions} to be used for all RocksDB instances.
 */
public ColumnFamilyOptions getColumnOptions() {
    // initial options from common profile
    ColumnFamilyOptions opt = createBaseCommonColumnOptions();
    handlesToClose.add(opt);
    // load configurable options on top of pre-defined profile
    setColumnFamilyOptionsFromConfigurableOptions(opt, handlesToClose);
    // add user-defined options, if specified
    if (optionsFactory != null) {
        opt = optionsFactory.createColumnOptions(opt, handlesToClose);
    }
    // set necessary options for performance consideration with memory control
    if (sharedResources != null) {
        final RocksDBSharedResources rocksResources = sharedResources.getResourceHandle();
        final Cache blockCache = rocksResources.getCache();
        TableFormatConfig tableFormatConfig = opt.tableFormatConfig();
        BlockBasedTableConfig blockBasedTableConfig;
        if (tableFormatConfig == null) {
            blockBasedTableConfig = new BlockBasedTableConfig();
        } else {
            Preconditions.checkArgument(tableFormatConfig instanceof BlockBasedTableConfig, "We currently only support BlockBasedTableConfig When bounding total memory.");
            blockBasedTableConfig = (BlockBasedTableConfig) tableFormatConfig;
        }
        if (rocksResources.isUsingPartitionedIndexFilters() && overwriteFilterIfExist(blockBasedTableConfig)) {
            blockBasedTableConfig.setIndexType(IndexType.kTwoLevelIndexSearch);
            blockBasedTableConfig.setPartitionFilters(true);
            blockBasedTableConfig.setPinTopLevelIndexAndFilter(true);
        }
        blockBasedTableConfig.setBlockCache(blockCache);
        blockBasedTableConfig.setCacheIndexAndFilterBlocks(true);
        blockBasedTableConfig.setCacheIndexAndFilterBlocksWithHighPriority(true);
        blockBasedTableConfig.setPinL0FilterAndIndexBlocksInCache(true);
        opt.setTableFormatConfig(blockBasedTableConfig);
    }
    return opt;
}

Example 5 with TableFormatConfig

use of org.rocksdb.TableFormatConfig in project flink by apache.

the class RocksDBResourceContainer method setColumnFamilyOptionsFromConfigurableOptions.

@SuppressWarnings("ConstantConditions")
private ColumnFamilyOptions setColumnFamilyOptionsFromConfigurableOptions(ColumnFamilyOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
    currentOptions.setCompactionStyle(internalGetOption(RocksDBConfigurableOptions.COMPACTION_STYLE));
    currentOptions.setLevelCompactionDynamicLevelBytes(internalGetOption(RocksDBConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE));
    currentOptions.setTargetFileSizeBase(internalGetOption(RocksDBConfigurableOptions.TARGET_FILE_SIZE_BASE).getBytes());
    currentOptions.setMaxBytesForLevelBase(internalGetOption(RocksDBConfigurableOptions.MAX_SIZE_LEVEL_BASE).getBytes());
    currentOptions.setWriteBufferSize(internalGetOption(RocksDBConfigurableOptions.WRITE_BUFFER_SIZE).getBytes());
    currentOptions.setMaxWriteBufferNumber(internalGetOption(RocksDBConfigurableOptions.MAX_WRITE_BUFFER_NUMBER));
    currentOptions.setMinWriteBufferNumberToMerge(internalGetOption(RocksDBConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE));
    TableFormatConfig tableFormatConfig = currentOptions.tableFormatConfig();
    BlockBasedTableConfig blockBasedTableConfig;
    if (tableFormatConfig == null) {
        blockBasedTableConfig = new BlockBasedTableConfig();
    } else {
        if (tableFormatConfig instanceof PlainTableConfig) {
            // column-family options
            return currentOptions;
        } else {
            blockBasedTableConfig = (BlockBasedTableConfig) tableFormatConfig;
        }
    }
    blockBasedTableConfig.setBlockSize(internalGetOption(RocksDBConfigurableOptions.BLOCK_SIZE).getBytes());
    blockBasedTableConfig.setMetadataBlockSize(internalGetOption(RocksDBConfigurableOptions.METADATA_BLOCK_SIZE).getBytes());
    blockBasedTableConfig.setBlockCacheSize(internalGetOption(RocksDBConfigurableOptions.BLOCK_CACHE_SIZE).getBytes());
    if (internalGetOption(RocksDBConfigurableOptions.USE_BLOOM_FILTER)) {
        final double bitsPerKey = internalGetOption(RocksDBConfigurableOptions.BLOOM_FILTER_BITS_PER_KEY);
        final boolean blockBasedMode = internalGetOption(RocksDBConfigurableOptions.BLOOM_FILTER_BLOCK_BASED_MODE);
        BloomFilter bloomFilter = new BloomFilter(bitsPerKey, blockBasedMode);
        handlesToClose.add(bloomFilter);
        blockBasedTableConfig.setFilterPolicy(bloomFilter);
    }
    return currentOptions.setTableFormatConfig(blockBasedTableConfig);
}