Examples with BloomFilter org.rocksdb.BloomFilter used on opensource projects

Example 1 with BloomFilter

use of org.rocksdb.BloomFilter in project flink by apache.

the class RocksDBResourceContainerTest method testGetColumnFamilyOptionsWithPartitionedIndex.

@Test
public void testGetColumnFamilyOptionsWithPartitionedIndex() throws Exception {
    LRUCache cache = new LRUCache(1024L);
    WriteBufferManager wbm = new WriteBufferManager(1024L, cache);
    RocksDBSharedResources sharedResources = new RocksDBSharedResources(cache, wbm, 1024L, true);
    final ThrowingRunnable<Exception> disposer = sharedResources::close;
    OpaqueMemoryResource<RocksDBSharedResources> opaqueResource = new OpaqueMemoryResource<>(sharedResources, 1024L, disposer);
    BloomFilter blockBasedFilter = new BloomFilter();
    RocksDBOptionsFactory blockBasedBloomFilterOptionFactory = new RocksDBOptionsFactory() {

        @Override
        public DBOptions createDBOptions(DBOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
            return currentOptions;
        }

        @Override
        public ColumnFamilyOptions createColumnOptions(ColumnFamilyOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
            TableFormatConfig tableFormatConfig = currentOptions.tableFormatConfig();
            BlockBasedTableConfig blockBasedTableConfig = tableFormatConfig == null ? new BlockBasedTableConfig() : (BlockBasedTableConfig) tableFormatConfig;
            blockBasedTableConfig.setFilter(blockBasedFilter);
            handlesToClose.add(blockBasedFilter);
            currentOptions.setTableFormatConfig(blockBasedTableConfig);
            return currentOptions;
        }
    };
    try (RocksDBResourceContainer container = new RocksDBResourceContainer(PredefinedOptions.DEFAULT, blockBasedBloomFilterOptionFactory, opaqueResource)) {
        ColumnFamilyOptions columnOptions = container.getColumnOptions();
        BlockBasedTableConfig actual = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
        assertThat(actual.indexType(), is(IndexType.kTwoLevelIndexSearch));
        assertThat(actual.partitionFilters(), is(true));
        assertThat(actual.pinTopLevelIndexAndFilter(), is(true));
        assertThat(actual.filterPolicy(), not(blockBasedFilter));
    }
    assertFalse("Block based filter is left unclosed.", blockBasedFilter.isOwningHandle());
}

Example 2 with BloomFilter

use of org.rocksdb.BloomFilter in project kafka by apache.

the class RocksDBStore method openDB.

@SuppressWarnings("unchecked")
void openDB(final Map<String, Object> configs, final File stateDir) {
    // initialize the default rocksdb options
    final DBOptions dbOptions = new DBOptions();
    final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions();
    userSpecifiedOptions = new RocksDBGenericOptionsToDbOptionsColumnFamilyOptionsAdapter(dbOptions, columnFamilyOptions);
    final BlockBasedTableConfigWithAccessibleCache tableConfig = new BlockBasedTableConfigWithAccessibleCache();
    cache = new LRUCache(BLOCK_CACHE_SIZE);
    tableConfig.setBlockCache(cache);
    tableConfig.setBlockSize(BLOCK_SIZE);
    filter = new BloomFilter();
    tableConfig.setFilterPolicy(filter);
    userSpecifiedOptions.optimizeFiltersForHits();
    userSpecifiedOptions.setTableFormatConfig(tableConfig);
    userSpecifiedOptions.setWriteBufferSize(WRITE_BUFFER_SIZE);
    userSpecifiedOptions.setCompressionType(COMPRESSION_TYPE);
    userSpecifiedOptions.setCompactionStyle(COMPACTION_STYLE);
    userSpecifiedOptions.setMaxWriteBufferNumber(MAX_WRITE_BUFFERS);
    userSpecifiedOptions.setCreateIfMissing(true);
    userSpecifiedOptions.setErrorIfExists(false);
    userSpecifiedOptions.setInfoLogLevel(InfoLogLevel.ERROR_LEVEL);
    // this is the recommended way to increase parallelism in RocksDb
    // note that the current implementation of setIncreaseParallelism affects the number
    // of compaction threads but not flush threads (the latter remains one). Also,
    // the parallelism value needs to be at least two because of the code in
    // https://github.com/facebook/rocksdb/blob/62ad0a9b19f0be4cefa70b6b32876e764b7f3c11/util/options.cc#L580
    // subtracts one from the value passed to determine the number of compaction threads
    // (this could be a bug in the RocksDB code and their devs have been contacted).
    userSpecifiedOptions.setIncreaseParallelism(Math.max(Runtime.getRuntime().availableProcessors(), 2));
    wOptions = new WriteOptions();
    wOptions.setDisableWAL(true);
    fOptions = new FlushOptions();
    fOptions.setWaitForFlush(true);
    final Class<RocksDBConfigSetter> configSetterClass = (Class<RocksDBConfigSetter>) configs.get(StreamsConfig.ROCKSDB_CONFIG_SETTER_CLASS_CONFIG);
    if (configSetterClass != null) {
        configSetter = Utils.newInstance(configSetterClass);
        configSetter.setConfig(name, userSpecifiedOptions, configs);
    }
    dbDir = new File(new File(stateDir, parentDir), name);
    try {
        Files.createDirectories(dbDir.getParentFile().toPath());
        Files.createDirectories(dbDir.getAbsoluteFile().toPath());
    } catch (final IOException fatal) {
        throw new ProcessorStateException(fatal);
    }
    // Setup statistics before the database is opened, otherwise the statistics are not updated
    // with the measurements from Rocks DB
    maybeSetUpStatistics(configs);
    openRocksDB(dbOptions, columnFamilyOptions);
    open = true;
    addValueProvidersToMetricsRecorder();
}

Example 3 with BloomFilter

use of org.rocksdb.BloomFilter in project aion by aionnetwork.

the class RocksDBWrapper method setupBlockBasedTableConfig.

private BlockBasedTableConfig setupBlockBasedTableConfig() {
    BlockBasedTableConfig bbtc = new BlockBasedTableConfig();
    bbtc.setBlockSize(BLOCK_SIZE);
    bbtc.setCacheIndexAndFilterBlocks(true);
    bbtc.setPinL0FilterAndIndexBlocksInCache(true);
    bbtc.setFilterPolicy(new BloomFilter(BLOOMFILTER_BITS_PER_KEY, false));
    return bbtc;
}

Example 4 with BloomFilter

use of org.rocksdb.BloomFilter in project flink by apache.

the class DefaultConfigurableOptionsFactory method createColumnOptions.

@Override
public ColumnFamilyOptions createColumnOptions(ColumnFamilyOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
    if (isOptionConfigured(COMPACTION_STYLE)) {
        currentOptions.setCompactionStyle(getCompactionStyle());
    }
    if (isOptionConfigured(USE_DYNAMIC_LEVEL_SIZE)) {
        currentOptions.setLevelCompactionDynamicLevelBytes(getUseDynamicLevelSize());
    }
    if (isOptionConfigured(TARGET_FILE_SIZE_BASE)) {
        currentOptions.setTargetFileSizeBase(getTargetFileSizeBase());
    }
    if (isOptionConfigured(MAX_SIZE_LEVEL_BASE)) {
        currentOptions.setMaxBytesForLevelBase(getMaxSizeLevelBase());
    }
    if (isOptionConfigured(WRITE_BUFFER_SIZE)) {
        currentOptions.setWriteBufferSize(getWriteBufferSize());
    }
    if (isOptionConfigured(MAX_WRITE_BUFFER_NUMBER)) {
        currentOptions.setMaxWriteBufferNumber(getMaxWriteBufferNumber());
    }
    if (isOptionConfigured(MIN_WRITE_BUFFER_NUMBER_TO_MERGE)) {
        currentOptions.setMinWriteBufferNumberToMerge(getMinWriteBufferNumberToMerge());
    }
    TableFormatConfig tableFormatConfig = currentOptions.tableFormatConfig();
    BlockBasedTableConfig blockBasedTableConfig;
    if (tableFormatConfig == null) {
        blockBasedTableConfig = new BlockBasedTableConfig();
    } else {
        if (tableFormatConfig instanceof PlainTableConfig) {
            // column-family options
            return currentOptions;
        } else {
            blockBasedTableConfig = (BlockBasedTableConfig) tableFormatConfig;
        }
    }
    if (isOptionConfigured(BLOCK_SIZE)) {
        blockBasedTableConfig.setBlockSize(getBlockSize());
    }
    if (isOptionConfigured(METADATA_BLOCK_SIZE)) {
        blockBasedTableConfig.setMetadataBlockSize(getMetadataBlockSize());
    }
    if (isOptionConfigured(BLOCK_CACHE_SIZE)) {
        blockBasedTableConfig.setBlockCacheSize(getBlockCacheSize());
    }
    if (isOptionConfigured(USE_BLOOM_FILTER)) {
        final boolean enabled = Boolean.parseBoolean(getInternal(USE_BLOOM_FILTER.key()));
        if (enabled) {
            final double bitsPerKey = isOptionConfigured(BLOOM_FILTER_BITS_PER_KEY) ? Double.parseDouble(getInternal(BLOOM_FILTER_BITS_PER_KEY.key())) : BLOOM_FILTER_BITS_PER_KEY.defaultValue();
            final boolean blockBasedMode = isOptionConfigured(BLOOM_FILTER_BLOCK_BASED_MODE) ? Boolean.parseBoolean(getInternal(BLOOM_FILTER_BLOCK_BASED_MODE.key())) : BLOOM_FILTER_BLOCK_BASED_MODE.defaultValue();
            BloomFilter bloomFilter = new BloomFilter(bitsPerKey, blockBasedMode);
            handlesToClose.add(bloomFilter);
            blockBasedTableConfig.setFilterPolicy(bloomFilter);
        }
    }
    return currentOptions.setTableFormatConfig(blockBasedTableConfig);
}

Example 5 with BloomFilter

use of org.rocksdb.BloomFilter in project flink by apache.

the class RocksDBResourceContainer method overwriteFilterIfExist.

/**
 * Overwrite configured {@link Filter} if enable partitioned filter. Partitioned filter only
 * worked in full bloom filter, not blocked based.
 */
private boolean overwriteFilterIfExist(BlockBasedTableConfig blockBasedTableConfig) {
    if (blockBasedTableConfig.filterPolicy() != null) {
        // TODO Can get filter's config in the future RocksDB version, and build new filter use
        // existing config.
        BloomFilter newFilter = new BloomFilter(10, false);
        LOG.info("Existing filter has been overwritten to full filters since partitioned index filters is enabled.");
        blockBasedTableConfig.setFilterPolicy(newFilter);
        handlesToClose.add(newFilter);
    }
    return true;
}