Examples with Props voldemort.utils.Props used on opensource projects

Example 6 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class StorageEnginePerformanceTest method main.

public static void main(String[] args) throws Exception {
    try {
        OptionParser parser = new OptionParser();
        parser.accepts("help", "print usage information");
        parser.accepts("requests", "[REQUIRED] number of requests to execute").withRequiredArg().ofType(Integer.class);
        parser.accepts("num-values", "[REQUIRED] number of values in the store").withRequiredArg().ofType(Integer.class);
        parser.accepts("data-dir", "Data directory for storage data").withRequiredArg().describedAs("directory");
        parser.accepts("threads", "number of threads").withRequiredArg().ofType(Integer.class);
        parser.accepts("storage-configuration-class", "[REQUIRED] class of the storage engine configuration to use [e.g. voldemort.store.bdb.BdbStorageConfiguration]").withRequiredArg().describedAs("class_name");
        parser.accepts("props", "Properties file with configuration for the engine").withRequiredArg().describedAs("config.properties");
        parser.accepts("value-size", "The size of the values in the store").withRequiredArg().describedAs("size").ofType(Integer.class);
        parser.accepts("cache-width", "Percentage of requests to save as possible re-requests").withRequiredArg().describedAs("width").ofType(Integer.class);
        parser.accepts("cache-hit-ratio", "Percentage of requests coming from the last cache-width requests").withRequiredArg().describedAs("ratio").ofType(Double.class);
        parser.accepts("clean-up", "Delete data directory when done.");
        OptionSet options = parser.parse(args);
        if (options.has("help")) {
            parser.printHelpOn(System.out);
            System.exit(0);
        }
        CmdUtils.croakIfMissing(parser, options, "requests");
        final int numThreads = CmdUtils.valueOf(options, "threads", 10);
        final int numRequests = (Integer) options.valueOf("requests");
        final int numValues = (Integer) options.valueOf("num-values");
        final int valueSize = CmdUtils.valueOf(options, "value-size", 1024);
        final int cacheWidth = CmdUtils.valueOf(options, "cache-width", 100000);
        final double cacheHitRatio = CmdUtils.valueOf(options, "cache-hit-ratio", 0.5);
        final String propsFile = (String) options.valueOf("props");
        final boolean cleanUp = options.has("clean-up");
        final String storageEngineClass = CmdUtils.valueOf(options, "storage-configuration-class", BdbStorageConfiguration.class.getName()).trim();
        File dataDir = null;
        if (options.has("data-dir"))
            dataDir = new File((String) options.valueOf("data-dir"));
        else
            dataDir = TestUtils.createTempDir();
        System.out.println("Data dir: " + dataDir);
        // create the storage engine
        Props props = new Props();
        if (propsFile != null)
            props = new Props(new File(propsFile));
        props.put("node.id", 0);
        props.put("data.directory", dataDir.getAbsolutePath());
        props.put("voldemort.home", System.getProperty("user.dir"));
        VoldemortConfig config = new VoldemortConfig(props);
        StorageConfiguration storageConfig = (StorageConfiguration) ReflectUtils.callConstructor(ReflectUtils.loadClass(storageEngineClass), new Object[] { config });
        StorageEngine<ByteArray, byte[], byte[]> engine = storageConfig.getStore(TestUtils.makeStoreDefinition("test"), TestUtils.makeSingleNodeRoutingStrategy());
        @SuppressWarnings("unchecked") final Store<String, byte[], byte[]> store = new SerializingStore(engine, new StringSerializer(), new IdentitySerializer(), null);
        final byte[] value = new byte[valueSize];
        new Random().nextBytes(value);
        // initialize test data
        for (int i = 0; i < numValues; i++) store.put(Integer.toString(i), Versioned.value(value), null);
        // initialize cache lookback data
        int[] recents = new int[cacheWidth];
        System.out.println("Write test:");
        CachedPerformanceTest writeTest = new CachedPerformanceTest(new PerformanceTest() {

            @Override
            public void doOperation(int index) {
                try {
                    String key = Integer.toString(index);
                    List<Versioned<byte[]>> vs = store.get(key, null);
                    VectorClock version;
                    if (vs.size() == 0)
                        version = new VectorClock();
                    else
                        version = (VectorClock) vs.get(0).getVersion();
                    version.incrementVersion(0, 847584375);
                    store.put(key, Versioned.value(value, version), null);
                } catch (ObsoleteVersionException e) {
                // do nothing
                } catch (RuntimeException e) {
                    e.printStackTrace();
                    throw e;
                }
            }
        }, recents, numValues, cacheHitRatio);
        writeTest.run(numRequests, numThreads);
        writeTest.printStats();
        System.out.println();
        System.out.println("Read test:");
        CachedPerformanceTest readTest = new CachedPerformanceTest(new PerformanceTest() {

            @Override
            public void doOperation(int index) {
                store.get(Integer.toString(index), null);
            }
        }, recents, numValues, cacheHitRatio);
        readTest.run(numRequests, numThreads);
        readTest.printStats();
        if (cleanUp)
            Utils.rm(dataDir);
    } catch (Exception e) {
        e.printStackTrace();
        System.exit(1);
    }
}

Example 7 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class Benchmark method main.

public static void main(String[] args) throws IOException {
    // Logger.getRootLogger().removeAllAppenders();
    OptionParser parser = new OptionParser();
    parser.accepts(READS, "percentage of --ops-count to be reads; valid values [0-100]").withRequiredArg().describedAs("read-percent").ofType(Integer.class);
    parser.accepts(WRITES, "percentage of --ops-count to be writes; valid values [0-100]").withRequiredArg().describedAs("write-percent").ofType(Integer.class);
    parser.accepts(DELETES, "percentage of --ops-count to be deletes; valid values [0-100]").withRequiredArg().describedAs("delete-percent").ofType(Integer.class);
    parser.accepts(MIXED, "percentage of --ops-count to be updates; valid values [0-100]").withRequiredArg().describedAs("update-percent").ofType(Integer.class);
    parser.accepts(SAMPLE_SIZE, "number of value samples to be obtained from the store for replay based on keys from request-file; 0 means no sample value replay. Default = 0").withRequiredArg().describedAs("sample-size").ofType(Integer.class);
    parser.accepts(VERBOSE, "verbose");
    parser.accepts(THREADS, "max number concurrent worker threads; Default = " + MAX_WORKERS).withRequiredArg().describedAs("num-threads").ofType(Integer.class);
    parser.accepts(NUM_CONNECTIONS_PER_NODE, "max number of connections to any node; Default = " + MAX_CONNECTIONS_PER_NODE).withRequiredArg().describedAs("num-connections-per-node").ofType(Integer.class);
    parser.accepts(ITERATIONS, "number of times to repeat benchmark phase; Default = 1").withRequiredArg().describedAs("num-iter").ofType(Integer.class);
    parser.accepts(VERIFY, "verify values read; runs only if warm-up phase is included");
    parser.accepts(PERCENT_CACHED, "percentage of requests to come from previously requested keys; valid values are in range [0..100]; 0 means caching disabled. Default = 0").withRequiredArg().describedAs("percent").ofType(Integer.class);
    parser.accepts(START_KEY_INDEX, "key index to start warm-up phase from; Default = 0").withRequiredArg().describedAs("index").ofType(Integer.class);
    parser.accepts(INTERVAL, "print status at interval seconds; Default = 0").withRequiredArg().describedAs("sec").ofType(Integer.class);
    parser.accepts(IGNORE_NULLS, "ignore null values in results");
    parser.accepts(PROP_FILE, "file containing all the properties in key=value format; will override all other command line options specified").withRequiredArg().describedAs("prop-file");
    parser.accepts(STORAGE_CONFIGURATION_CLASS, "class of the storage engine configuration to use [e.g. voldemort.store.bdb.BdbStorageConfiguration]").withRequiredArg().describedAs("class-name");
    parser.accepts(KEY_TYPE, "for local tests; key type to support; [ " + IDENTITY_KEY_TYPE + " | " + JSONINT_KEY_TYPE + " | " + JSONSTRING_KEY_TYPE + "|" + STRING_KEY_TYPE + " <default> ]").withRequiredArg().describedAs("type");
    parser.accepts(REQUEST_FILE, "file with limited list of keys to be used during benchmark phase; Overrides " + RECORD_SELECTION).withRequiredArg();
    parser.accepts(VALUE_SIZE, "size in bytes for random value; used during warm-up phase and write operation of benchmark phase; Default = 1024").withRequiredArg().describedAs("bytes").ofType(Integer.class);
    parser.accepts(RECORD_SELECTION, "record selection distribution [ " + ZIPFIAN_RECORD_SELECTION + " | " + LATEST_RECORD_SELECTION + " | " + UNIFORM_RECORD_SELECTION + " <default> ]").withRequiredArg();
    parser.accepts(TARGET_THROUGHPUT, "fix throughput").withRequiredArg().describedAs("ops/sec").ofType(Integer.class);
    parser.accepts(RECORD_COUNT, "number of records inserted during warmup phase").withRequiredArg().describedAs("count").ofType(Integer.class);
    parser.accepts(OPS_COUNT, "number of operations to do during benchmark phase").withRequiredArg().describedAs("count").ofType(Integer.class);
    parser.accepts(URL, "for remote tests; url of remote server").withRequiredArg();
    parser.accepts(STORE_NAME, "for remote tests; store name on the remote " + URL).withRequiredArg().describedAs("name");
    parser.accepts(METRIC_TYPE, "type of result metric [ " + HISTOGRAM_METRIC_TYPE + " | " + SUMMARY_METRIC_TYPE + " <default> ]").withRequiredArg();
    parser.accepts(PLUGIN_CLASS, "classname of implementation of WorkloadPlugin; used to run customized operations ").withRequiredArg().describedAs("class-name");
    parser.accepts(CLIENT_ZONE_ID, "zone id for client; enables zone routing").withRequiredArg().describedAs("zone-id").ofType(Integer.class);
    parser.accepts(LOCAL_SERVER_PROPERTIES, "path to a server properties file (local mode only)").withRequiredArg().describedAs(LOCAL_SERVER_PROPERTIES).ofType(String.class);
    parser.accepts(WORKLOAD_TYPE, "workload type; type to support; [ " + TRACE_WORKLOAD_TYPE + " | " + DEFAULT_WORKLOAD_TYPE + " <default> ]").withRequiredArg().describedAs(WORKLOAD_TYPE).ofType(String.class);
    parser.accepts(KEY_VALUE_FILE, "path to a file with keys and value sizes").withRequiredArg().describedAs(KEY_VALUE_FILE).ofType(String.class);
    parser.accepts(KEY_SEQUENCE_FILE, "path to a file with key access sequence").withRequiredArg().describedAs(KEY_SEQUENCE_FILE).ofType(String.class);
    parser.accepts(HELP);
    OptionSet options = parser.parse(args);
    if (options.has(HELP)) {
        parser.printHelpOn(System.out);
        System.exit(0);
    }
    Props mainProps = null;
    if (options.has(PROP_FILE)) {
        String propFileDestination = (String) options.valueOf(PROP_FILE);
        File propertyFile = new File(propFileDestination);
        if (!propertyFile.exists()) {
            printUsage(parser, "Property file does not exist");
        }
        try {
            mainProps = new Props(propertyFile);
        } catch (Exception e) {
            printUsage(parser, "Unable to parse the property file");
        }
    } else {
        mainProps = new Props();
        if (options.has(REQUEST_FILE)) {
            mainProps.put(REQUEST_FILE, (String) options.valueOf(REQUEST_FILE));
            mainProps.put(RECORD_SELECTION, FILE_RECORD_SELECTION);
        } else {
            mainProps.put(RECORD_SELECTION, CmdUtils.valueOf(options, RECORD_SELECTION, UNIFORM_RECORD_SELECTION));
        }
        if (options.has(RECORD_COUNT)) {
            mainProps.put(RECORD_COUNT, (Integer) options.valueOf(RECORD_COUNT));
        } else {
            mainProps.put(RECORD_COUNT, 0);
        }
        if (!options.has(OPS_COUNT)) {
            printUsage(parser, "Missing " + OPS_COUNT);
        }
        mainProps.put(OPS_COUNT, (Integer) options.valueOf(OPS_COUNT));
        if (options.has(URL)) {
            mainProps.put(URL, (String) options.valueOf(URL));
            if (options.has(STORE_NAME)) {
                mainProps.put(STORE_NAME, (String) options.valueOf(STORE_NAME));
            } else {
                printUsage(parser, "Missing store name");
            }
        } else {
            mainProps.put(KEY_TYPE, CmdUtils.valueOf(options, KEY_TYPE, STRING_KEY_TYPE));
            mainProps.put(STORAGE_CONFIGURATION_CLASS, CmdUtils.valueOf(options, STORAGE_CONFIGURATION_CLASS, BdbStorageConfiguration.class.getName()));
        }
        if (options.has(LOCAL_SERVER_PROPERTIES)) {
            mainProps.put(LOCAL_SERVER_PROPERTIES, (String) options.valueOf(LOCAL_SERVER_PROPERTIES));
        }
        if (options.has(WORKLOAD_TYPE)) {
            mainProps.put(WORKLOAD_TYPE, (String) options.valueOf(WORKLOAD_TYPE));
            if (((String) options.valueOf(WORKLOAD_TYPE)).equals(TRACE_WORKLOAD_TYPE)) {
                CmdUtils.croakIfMissing(parser, options, KEY_VALUE_FILE, KEY_SEQUENCE_FILE);
                mainProps.put(KEY_VALUE_FILE, (String) options.valueOf(KEY_VALUE_FILE));
                mainProps.put(KEY_SEQUENCE_FILE, (String) options.valueOf(KEY_SEQUENCE_FILE));
            }
        }
        mainProps.put(VERBOSE, getCmdBoolean(options, VERBOSE));
        mainProps.put(VERIFY, getCmdBoolean(options, VERIFY));
        mainProps.put(IGNORE_NULLS, getCmdBoolean(options, IGNORE_NULLS));
        mainProps.put(CLIENT_ZONE_ID, CmdUtils.valueOf(options, CLIENT_ZONE_ID, -1));
        mainProps.put(START_KEY_INDEX, CmdUtils.valueOf(options, START_KEY_INDEX, 0));
        mainProps.put(VALUE_SIZE, CmdUtils.valueOf(options, VALUE_SIZE, 1024));
        mainProps.put(ITERATIONS, CmdUtils.valueOf(options, ITERATIONS, 1));
        mainProps.put(THREADS, CmdUtils.valueOf(options, THREADS, MAX_WORKERS));
        mainProps.put(NUM_CONNECTIONS_PER_NODE, CmdUtils.valueOf(options, NUM_CONNECTIONS_PER_NODE, MAX_CONNECTIONS_PER_NODE));
        mainProps.put(PERCENT_CACHED, CmdUtils.valueOf(options, PERCENT_CACHED, 0));
        mainProps.put(INTERVAL, CmdUtils.valueOf(options, INTERVAL, 0));
        mainProps.put(TARGET_THROUGHPUT, CmdUtils.valueOf(options, TARGET_THROUGHPUT, -1));
        mainProps.put(METRIC_TYPE, CmdUtils.valueOf(options, METRIC_TYPE, SUMMARY_METRIC_TYPE));
        mainProps.put(READS, CmdUtils.valueOf(options, READS, 0));
        mainProps.put(WRITES, CmdUtils.valueOf(options, WRITES, 0));
        mainProps.put(DELETES, CmdUtils.valueOf(options, DELETES, 0));
        mainProps.put(MIXED, CmdUtils.valueOf(options, MIXED, 0));
        mainProps.put(PLUGIN_CLASS, CmdUtils.valueOf(options, PLUGIN_CLASS, ""));
        mainProps.put(SAMPLE_SIZE, CmdUtils.valueOf(options, SAMPLE_SIZE, 0));
    }
    // Start the benchmark
    Benchmark benchmark = null;
    try {
        benchmark = new Benchmark();
        benchmark.initialize(mainProps);
        benchmark.warmUpAndRun();
        benchmark.close();
    } catch (Exception e) {
        if (options.has(VERBOSE)) {
            e.printStackTrace();
        }
        parser.printHelpOn(System.err);
        System.exit(-1);
    }
}

Example 8 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class SlopStreamingTestEnvironment method makeSlopStreamingClient.

public StreamingClient makeSlopStreamingClient(boolean wrapException) throws InterruptedException {
    startFinishLatch.await();
    if (throwIntermittentException)
        waitForFaultyNodeLatch.await();
    Props property = new Props();
    property.put("streaming.platform.bootstrapURL", bootstrapUrl);
    property.put("streaming.platform.max.failed.nodes", numFailedNodes);
    StreamingClientConfig config = new StreamingClientConfig(property);
    StreamingClient streamer = new StreamingClient(config);
    Callable<Integer> cpCallable = new CheckpointCallable();
    Callable<Integer> rbCallable = new RollbackCallable();
    List<String> stores = new ArrayList();
    stores.add(STORE_NAME);
    List<Integer> failedNodes = new ArrayList<Integer>();
    for (int i = 0; i < numFailedNodes; i++) failedNodes.add(i + 1);
    if (!wrapException) {
        streamer.initStreamingSessions(stores, cpCallable, rbCallable, true, failedNodes);
    } else {
        try {
            streamer.initStreamingSessions(stores, cpCallable, rbCallable, true, failedNodes);
        } catch (Exception e) {
            logger.error(e);
        }
    }
    return streamer;
}

Example 9 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class BenchmarkViews method main.

public static void main(String[] args) throws IOException {
    OptionParser parser = new OptionParser();
    parser.accepts(Benchmark.RECORD_COUNT, "number of records inserted during warmup phase").withRequiredArg().ofType(Integer.class);
    parser.accepts(Benchmark.OPS_COUNT, "number of operations to do").withRequiredArg().ofType(Integer.class);
    parser.accepts(Benchmark.HELP);
    OptionSet options = parser.parse(args);
    if (options.has(Benchmark.HELP)) {
        parser.printHelpOn(System.out);
        System.exit(0);
    }
    if (!options.has(Benchmark.RECORD_COUNT) || !options.has(Benchmark.OPS_COUNT)) {
        parser.printHelpOn(System.out);
        Utils.croak("Missing params");
        System.exit(0);
    }
    Props props = new Props();
    props.put(Benchmark.RECORD_COUNT, (Integer) options.valueOf(Benchmark.RECORD_COUNT));
    props.put(Benchmark.OPS_COUNT, (Integer) options.valueOf(Benchmark.OPS_COUNT));
    props.put(Benchmark.STORAGE_CONFIGURATION_CLASS, BdbStorageConfiguration.class.getCanonicalName());
    props.put(Benchmark.STORE_TYPE, "view");
    props.put(Benchmark.VIEW_CLASS, "voldemort.store.views.UpperCaseView");
    props.put(Benchmark.HAS_TRANSFORMS, "true");
    BenchmarkViews benchmark = null;
    try {
        benchmark = new BenchmarkViews();
        benchmark.runBenchmark(props);
    } catch (Exception e) {
        e.printStackTrace();
        System.exit(-1);
    }
}

Example 10 with Props

use of voldemort.utils.Props in project voldemort by voldemort.

the class CatBdbStore method main.

public static void main(String[] args) throws Exception {
    if (args.length != 2)
        Utils.croak("USAGE: java " + CatBdbStore.class.getName() + " bdb_dir" + " storeName" + " server.properties.path");
    String bdbDir = args[0];
    String storeName = args[1];
    String serverProperties = args[2];
    VoldemortConfig config = new VoldemortConfig(new Props(new File(serverProperties)));
    EnvironmentConfig environmentConfig = new EnvironmentConfig();
    environmentConfig.setDurability(Durability.COMMIT_NO_SYNC);
    environmentConfig.setAllowCreate(true);
    environmentConfig.setTransactional(config.isBdbWriteTransactionsEnabled());
    Environment environment = new Environment(new File(bdbDir), environmentConfig);
    DatabaseConfig databaseConfig = new DatabaseConfig();
    databaseConfig.setAllowCreate(true);
    databaseConfig.setTransactional(config.isBdbWriteTransactionsEnabled());
    databaseConfig.setSortedDuplicates(false);
    Database database = environment.openDatabase(null, storeName, databaseConfig);
    StorageEngine<ByteArray, byte[], byte[]> store = null;
    if (config.getBdbPrefixKeysWithPartitionId()) {
        store = new PartitionPrefixedBdbStorageEngine(storeName, environment, database, new BdbRuntimeConfig(), TestUtils.makeSingleNodeRoutingStrategy());
    } else {
        store = new BdbStorageEngine(storeName, environment, database, new BdbRuntimeConfig());
    }
    StorageEngine<String, String, String> stringStore = SerializingStorageEngine.wrap(store, new StringSerializer(), new StringSerializer(), new StringSerializer());
    Iterator<Pair<String, Versioned<String>>> iter = stringStore.entries();
    while (iter.hasNext()) {
        Pair<String, Versioned<String>> entry = iter.next();
        System.out.println(entry.getFirst() + " => " + entry.getSecond().getValue());
    }
}