Examples with CompactionStyle org.rocksdb.CompactionStyle used on opensource projects

Example 1 with CompactionStyle

use of org.rocksdb.CompactionStyle in project samza by apache.

the class RocksDbOptionsHelper method options.

public static Options options(Config storeConfig, SamzaContainerContext containerContext) {
    Options options = new Options();
    Long writeBufSize = storeConfig.getLong("container.write.buffer.size.bytes", 32 * 1024 * 1024);
    // Cache size and write buffer size are specified on a per-container basis.
    int numTasks = containerContext.taskNames.size();
    options.setWriteBufferSize((int) (writeBufSize / numTasks));
    CompressionType compressionType = CompressionType.SNAPPY_COMPRESSION;
    String compressionInConfig = storeConfig.get(ROCKSDB_COMPRESSION, "snappy");
    switch(compressionInConfig) {
        case "snappy":
            compressionType = CompressionType.SNAPPY_COMPRESSION;
            break;
        case "bzip2":
            compressionType = CompressionType.BZLIB2_COMPRESSION;
            break;
        case "zlib":
            compressionType = CompressionType.ZLIB_COMPRESSION;
            break;
        case "lz4":
            compressionType = CompressionType.LZ4_COMPRESSION;
            break;
        case "lz4hc":
            compressionType = CompressionType.LZ4HC_COMPRESSION;
            break;
        case "none":
            compressionType = CompressionType.NO_COMPRESSION;
            break;
        default:
            log.warn("Unknown rocksdb.compression codec " + compressionInConfig + ", overwriting to " + compressionType.name());
    }
    options.setCompressionType(compressionType);
    Long cacheSize = storeConfig.getLong("container.cache.size.bytes", 100 * 1024 * 1024L);
    Long cacheSizePerContainer = cacheSize / numTasks;
    int blockSize = storeConfig.getInt(ROCKSDB_BLOCK_SIZE_BYTES, 4096);
    BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
    tableOptions.setBlockCacheSize(cacheSizePerContainer).setBlockSize(blockSize);
    options.setTableFormatConfig(tableOptions);
    CompactionStyle compactionStyle = CompactionStyle.UNIVERSAL;
    String compactionStyleInConfig = storeConfig.get(ROCKSDB_COMPACTION_STYLE, "universal");
    switch(compactionStyleInConfig) {
        case "universal":
            compactionStyle = CompactionStyle.UNIVERSAL;
            break;
        case "fifo":
            compactionStyle = CompactionStyle.FIFO;
            break;
        case "level":
            compactionStyle = CompactionStyle.LEVEL;
            break;
        default:
            log.warn("Unknown rocksdb.compaction.style " + compactionStyleInConfig + ", overwriting to " + compactionStyle.name());
    }
    options.setCompactionStyle(compactionStyle);
    options.setMaxWriteBufferNumber(storeConfig.getInt(ROCKSDB_NUM_WRITE_BUFFERS, 3));
    options.setCreateIfMissing(true);
    options.setErrorIfExists(false);
    options.setMaxLogFileSize(storeConfig.getLong(ROCKSDB_MAX_LOG_FILE_SIZE_BYTES, 64 * 1024 * 1024L));
    options.setKeepLogFileNum(storeConfig.getLong(ROCKSDB_KEEP_LOG_FILE_NUM, 2));
    return options;
}

Example 2 with CompactionStyle

use of org.rocksdb.CompactionStyle in project samza by apache.

the class RocksDbOptionsHelper method setCompactionOptions.

private static void setCompactionOptions(Config storeConfig, Options options) {
    if (storeConfig.containsKey(ROCKSDB_COMPACTION_NUM_LEVELS)) {
        options.setNumLevels(storeConfig.getInt(ROCKSDB_COMPACTION_NUM_LEVELS));
    }
    if (storeConfig.containsKey(ROCKSDB_COMPACTION_LEVEL0_FILE_NUM_COMPACTION_TRIGGER)) {
        int level0FileNumCompactionTrigger = storeConfig.getInt(ROCKSDB_COMPACTION_LEVEL0_FILE_NUM_COMPACTION_TRIGGER);
        options.setLevel0FileNumCompactionTrigger(level0FileNumCompactionTrigger);
    }
    if (storeConfig.containsKey(ROCKSDB_COMPACTION_MAX_BACKGROUND_COMPACTIONS)) {
        options.setMaxBackgroundCompactions(storeConfig.getInt(ROCKSDB_COMPACTION_MAX_BACKGROUND_COMPACTIONS, DEFAULT_ROCKSDB_COMPACTION_MAX_BACKGROUND_COMPACTIONS));
    }
    if (storeConfig.containsKey(ROCKSDB_COMPACTION_TARGET_FILE_SIZE_BASE)) {
        options.setTargetFileSizeBase(storeConfig.getLong(ROCKSDB_COMPACTION_TARGET_FILE_SIZE_BASE));
    }
    if (storeConfig.containsKey(ROCKSDB_COMPACTION_TARGET_FILE_SIZE_MULTIPLIER)) {
        options.setTargetFileSizeBase(storeConfig.getLong(ROCKSDB_COMPACTION_TARGET_FILE_SIZE_MULTIPLIER));
    }
    CompactionStyle compactionStyle = CompactionStyle.UNIVERSAL;
    String compactionStyleInConfig = storeConfig.get(ROCKSDB_COMPACTION_STYLE, "universal");
    switch(compactionStyleInConfig) {
        case "universal":
            compactionStyle = CompactionStyle.UNIVERSAL;
            break;
        case "fifo":
            compactionStyle = CompactionStyle.FIFO;
            break;
        case "level":
            compactionStyle = CompactionStyle.LEVEL;
            break;
        default:
            log.warn("Unknown rocksdb.compaction.style " + compactionStyleInConfig + ", overwriting to " + compactionStyle.name());
    }
    options.setCompactionStyle(compactionStyle);
    // Universal compaction options
    if (compactionStyle.equals(CompactionStyle.UNIVERSAL)) {
        CompactionOptionsUniversal compactionOptions = new CompactionOptionsUniversal();
        if (storeConfig.containsKey(ROCKSDB_COMPACTION_UNIVERSAL_MAX_SIZE_AMPLIFICATION_PERCENT)) {
            int val = storeConfig.getInt(ROCKSDB_COMPACTION_UNIVERSAL_MAX_SIZE_AMPLIFICATION_PERCENT);
            compactionOptions.setMaxSizeAmplificationPercent(val);
        }
        if (storeConfig.containsKey(ROCKSDB_COMPACTION_UNIVERSAL_SIZE_RATIO)) {
            int val = storeConfig.getInt(ROCKSDB_COMPACTION_UNIVERSAL_SIZE_RATIO);
            compactionOptions.setSizeRatio(val);
        }
        if (storeConfig.containsKey(ROCKSDB_COMPACTION_UNIVERSAL_MIN_MERGE_WIDTH)) {
            int val = storeConfig.getInt(ROCKSDB_COMPACTION_UNIVERSAL_MIN_MERGE_WIDTH);
            compactionOptions.setMinMergeWidth(val);
        }
        if (storeConfig.containsKey(ROCKSDB_COMPACTION_UNIVERSAL_MAX_MERGE_WIDTH)) {
            int val = storeConfig.getInt(ROCKSDB_COMPACTION_UNIVERSAL_MAX_MERGE_WIDTH);
            compactionOptions.setMaxMergeWidth(val);
        }
        if (storeConfig.containsKey(ROCKSDB_COMPACTION_UNIVERSAL_COMPACTION_STOP_STYLE)) {
            String stopStyle = storeConfig.get(ROCKSDB_COMPACTION_UNIVERSAL_COMPACTION_STOP_STYLE);
            if (StringUtils.isNotBlank(stopStyle)) {
                compactionOptions.setStopStyle(CompactionStopStyle.valueOf(stopStyle));
            }
        }
        options.setCompactionOptionsUniversal(compactionOptions);
    }
}