Examples with FileIntegerGenerator voldemort.performance.benchmark.generator.FileIntegerGenerator used on opensource projects

Example 1 with FileIntegerGenerator

use of voldemort.performance.benchmark.generator.FileIntegerGenerator in project voldemort by voldemort.

the class DefaultWorkload method init.

/**
     * Initialize the workload. Called once, in the main client thread, before
     * any operations are started.
     */
@Override
public void init(Props props) {
    int readPercent = props.getInt(Benchmark.READS, 0);
    int writePercent = props.getInt(Benchmark.WRITES, 0);
    int deletePercent = props.getInt(Benchmark.DELETES, 0);
    int mixedPercent = props.getInt(Benchmark.MIXED, 0);
    int valueSize = props.getInt(Benchmark.VALUE_SIZE, 1024);
    this.value = new String(TestUtils.randomBytes(valueSize));
    this.sampleSize = props.getInt(Benchmark.SAMPLE_SIZE, 0);
    int cachedPercent = props.getInt(Benchmark.PERCENT_CACHED, 0);
    String keyType = props.getString(Benchmark.KEY_TYPE, Benchmark.STRING_KEY_TYPE);
    String recordSelection = props.getString(Benchmark.RECORD_SELECTION, Benchmark.UNIFORM_RECORD_SELECTION);
    boolean hasTransforms = props.getString(Benchmark.HAS_TRANSFORMS, "false").compareTo("true") == 0;
    double readProportion = (double) readPercent / (double) 100;
    double writeProportion = (double) writePercent / (double) 100;
    double deleteProportion = (double) deletePercent / (double) 100;
    double mixedProportion = (double) mixedPercent / (double) 100;
    // Using default read only
    if (Math.abs(writeProportion + readProportion + mixedProportion + deleteProportion) != 1.0) {
        throw new VoldemortException("The sum of all workload percentage is NOT 100% \n" + " Read=" + (double) readPercent / (double) 100 + " Write=" + (double) writePercent / (double) 100 + " Delete=" + (double) deletePercent / (double) 100 + " Mixed=" + (double) mixedPercent / (double) 100);
    }
    List<Integer> keysFromFile = null;
    int recordCount = 0;
    if (props.containsKey(Benchmark.REQUEST_FILE)) {
        try {
            String fileRecordSelectionFile = props.getString(Benchmark.REQUEST_FILE);
            if (!new File(fileRecordSelectionFile).exists()) {
                throw new UndefinedPropertyException("File does not exist");
            }
            keysFromFile = loadKeys(new File(fileRecordSelectionFile));
            recordSelection = new String(Benchmark.FILE_RECORD_SELECTION);
        } catch (Exception e) {
            // Falling back to default uniform selection
            recordSelection = new String(Benchmark.UNIFORM_RECORD_SELECTION);
        }
    } else {
        recordCount = props.getInt(Benchmark.RECORD_COUNT, -1);
    }
    // 2. sample_size shall be no greater than # of keys in the request_file
    if (sampleSize > 0) {
        if (recordSelection.compareTo(Benchmark.FILE_RECORD_SELECTION) != 0) {
            sampleSize = 0;
            logger.warn(Benchmark.SAMPLE_SIZE + " will be ignored because " + Benchmark.REQUEST_FILE + " is not specified.");
        } else if (keysFromFile.size() < sampleSize) {
            sampleSize = keysFromFile.size();
            logger.warn(Benchmark.SAMPLE_SIZE + " is reduced to " + sampleSize + " because it was larger than number of keys in " + Benchmark.REQUEST_FILE);
        }
    }
    int opCount = props.getInt(Benchmark.OPS_COUNT);
    Class<?> keyTypeClass = getKeyTypeClass(keyType);
    int insertStart = props.getInt(Benchmark.START_KEY_INDEX, 0);
    IntegerGenerator warmUpKeySequence = new CounterGenerator(insertStart);
    this.warmUpKeyProvider = getKeyProvider(keyTypeClass, warmUpKeySequence, 0);
    this.transformsChooser = null;
    if (hasTransforms) {
        this.transformsChooser = new DiscreteGenerator();
        List<String> transforms = BenchmarkViews.getTransforms();
        for (String transform : transforms) {
            this.transformsChooser.addValue(1.0, transform);
        }
    }
    operationChooser = new DiscreteGenerator();
    if (readProportion > 0) {
        operationChooser.addValue(readProportion, Benchmark.READS);
    }
    if (mixedProportion > 0) {
        operationChooser.addValue(mixedProportion, Benchmark.MIXED);
    }
    if (writeProportion > 0) {
        operationChooser.addValue(writeProportion, Benchmark.WRITES);
    }
    if (deleteProportion > 0) {
        operationChooser.addValue(deleteProportion, Benchmark.DELETES);
    }
    CounterGenerator insertKeySequence = null;
    if (recordCount > 0) {
        insertKeySequence = new CounterGenerator(recordCount);
    } else {
        Random randomizer = new Random();
        insertKeySequence = new CounterGenerator(randomizer.nextInt(Integer.MAX_VALUE));
    }
    IntegerGenerator keyGenerator = null;
    if (recordSelection.compareTo(Benchmark.UNIFORM_RECORD_SELECTION) == 0) {
        int keySpace = (recordCount > 0) ? recordCount : Integer.MAX_VALUE;
        keyGenerator = new UniformIntegerGenerator(0, keySpace - 1);
    } else if (recordSelection.compareTo(Benchmark.ZIPFIAN_RECORD_SELECTION) == 0) {
        int expectedNewKeys = (int) (opCount * writeProportion * 2.0);
        keyGenerator = new ScrambledZipfianGenerator(recordCount + expectedNewKeys);
    } else if (recordSelection.compareTo(Benchmark.LATEST_RECORD_SELECTION) == 0) {
        keyGenerator = new SkewedLatestGenerator(insertKeySequence);
    } else if (recordSelection.compareTo(Benchmark.FILE_RECORD_SELECTION) == 0) {
        keyGenerator = new FileIntegerGenerator(0, keysFromFile);
    }
    this.keyProvider = getKeyProvider(keyTypeClass, keyGenerator, cachedPercent);
    this.randomSampler = new Random(System.currentTimeMillis());
}