Examples with BlockBasedTableConfig org.rocksdb.BlockBasedTableConfig used on opensource projects

Example 11 with BlockBasedTableConfig

use of org.rocksdb.BlockBasedTableConfig in project flink by apache.

the class RocksDBResourceContainer method getColumnOptions.

/**
 * Gets the RocksDB {@link ColumnFamilyOptions} to be used for all RocksDB instances.
 */
public ColumnFamilyOptions getColumnOptions() {
    // initial options from common profile
    ColumnFamilyOptions opt = createBaseCommonColumnOptions();
    handlesToClose.add(opt);
    // load configurable options on top of pre-defined profile
    setColumnFamilyOptionsFromConfigurableOptions(opt, handlesToClose);
    // add user-defined options, if specified
    if (optionsFactory != null) {
        opt = optionsFactory.createColumnOptions(opt, handlesToClose);
    }
    // set necessary options for performance consideration with memory control
    if (sharedResources != null) {
        final RocksDBSharedResources rocksResources = sharedResources.getResourceHandle();
        final Cache blockCache = rocksResources.getCache();
        TableFormatConfig tableFormatConfig = opt.tableFormatConfig();
        BlockBasedTableConfig blockBasedTableConfig;
        if (tableFormatConfig == null) {
            blockBasedTableConfig = new BlockBasedTableConfig();
        } else {
            Preconditions.checkArgument(tableFormatConfig instanceof BlockBasedTableConfig, "We currently only support BlockBasedTableConfig When bounding total memory.");
            blockBasedTableConfig = (BlockBasedTableConfig) tableFormatConfig;
        }
        if (rocksResources.isUsingPartitionedIndexFilters() && overwriteFilterIfExist(blockBasedTableConfig)) {
            blockBasedTableConfig.setIndexType(IndexType.kTwoLevelIndexSearch);
            blockBasedTableConfig.setPartitionFilters(true);
            blockBasedTableConfig.setPinTopLevelIndexAndFilter(true);
        }
        blockBasedTableConfig.setBlockCache(blockCache);
        blockBasedTableConfig.setCacheIndexAndFilterBlocks(true);
        blockBasedTableConfig.setCacheIndexAndFilterBlocksWithHighPriority(true);
        blockBasedTableConfig.setPinL0FilterAndIndexBlocksInCache(true);
        opt.setTableFormatConfig(blockBasedTableConfig);
    }
    return opt;
}

Example 12 with BlockBasedTableConfig

use of org.rocksdb.BlockBasedTableConfig in project flink by apache.

the class RocksDBResourceContainer method setColumnFamilyOptionsFromConfigurableOptions.

@SuppressWarnings("ConstantConditions")
private ColumnFamilyOptions setColumnFamilyOptionsFromConfigurableOptions(ColumnFamilyOptions currentOptions, Collection<AutoCloseable> handlesToClose) {
    currentOptions.setCompactionStyle(internalGetOption(RocksDBConfigurableOptions.COMPACTION_STYLE));
    currentOptions.setLevelCompactionDynamicLevelBytes(internalGetOption(RocksDBConfigurableOptions.USE_DYNAMIC_LEVEL_SIZE));
    currentOptions.setTargetFileSizeBase(internalGetOption(RocksDBConfigurableOptions.TARGET_FILE_SIZE_BASE).getBytes());
    currentOptions.setMaxBytesForLevelBase(internalGetOption(RocksDBConfigurableOptions.MAX_SIZE_LEVEL_BASE).getBytes());
    currentOptions.setWriteBufferSize(internalGetOption(RocksDBConfigurableOptions.WRITE_BUFFER_SIZE).getBytes());
    currentOptions.setMaxWriteBufferNumber(internalGetOption(RocksDBConfigurableOptions.MAX_WRITE_BUFFER_NUMBER));
    currentOptions.setMinWriteBufferNumberToMerge(internalGetOption(RocksDBConfigurableOptions.MIN_WRITE_BUFFER_NUMBER_TO_MERGE));
    TableFormatConfig tableFormatConfig = currentOptions.tableFormatConfig();
    BlockBasedTableConfig blockBasedTableConfig;
    if (tableFormatConfig == null) {
        blockBasedTableConfig = new BlockBasedTableConfig();
    } else {
        if (tableFormatConfig instanceof PlainTableConfig) {
            // column-family options
            return currentOptions;
        } else {
            blockBasedTableConfig = (BlockBasedTableConfig) tableFormatConfig;
        }
    }
    blockBasedTableConfig.setBlockSize(internalGetOption(RocksDBConfigurableOptions.BLOCK_SIZE).getBytes());
    blockBasedTableConfig.setMetadataBlockSize(internalGetOption(RocksDBConfigurableOptions.METADATA_BLOCK_SIZE).getBytes());
    blockBasedTableConfig.setBlockCacheSize(internalGetOption(RocksDBConfigurableOptions.BLOCK_CACHE_SIZE).getBytes());
    if (internalGetOption(RocksDBConfigurableOptions.USE_BLOOM_FILTER)) {
        final double bitsPerKey = internalGetOption(RocksDBConfigurableOptions.BLOOM_FILTER_BITS_PER_KEY);
        final boolean blockBasedMode = internalGetOption(RocksDBConfigurableOptions.BLOOM_FILTER_BLOCK_BASED_MODE);
        BloomFilter bloomFilter = new BloomFilter(bitsPerKey, blockBasedMode);
        handlesToClose.add(bloomFilter);
        blockBasedTableConfig.setFilterPolicy(bloomFilter);
    }
    return currentOptions.setTableFormatConfig(blockBasedTableConfig);
}

Example 13 with BlockBasedTableConfig

use of org.rocksdb.BlockBasedTableConfig in project flink by apache.

the class RocksDBResourceContainerTest method getBlockCache.

private Cache getBlockCache(ColumnFamilyOptions columnOptions) {
    BlockBasedTableConfig blockBasedTableConfig = null;
    try {
        blockBasedTableConfig = (BlockBasedTableConfig) columnOptions.tableFormatConfig();
    } catch (ClassCastException e) {
        fail("Table config got from ColumnFamilyOptions is not BlockBasedTableConfig");
    }
    Field cacheField = null;
    try {
        cacheField = BlockBasedTableConfig.class.getDeclaredField("blockCache");
    } catch (NoSuchFieldException e) {
        fail("blockCache is not defined");
    }
    cacheField.setAccessible(true);
    try {
        return (Cache) cacheField.get(blockBasedTableConfig);
    } catch (IllegalAccessException e) {
        fail("Cannot access blockCache field.");
        return null;
    }
}