Examples with OptionSet joptsimple.OptionSet used on opensource projects

Example 41 with OptionSet

use of joptsimple.OptionSet in project voldemort by voldemort.

the class RoutedStoreParallelismTest method main.

public static void main(String[] args) throws Throwable {
    OptionParser parser = new OptionParser();
    parser.accepts("num-keys", "The number of keys to submit for retrieval  Default = " + DEFAULT_NUM_KEYS).withRequiredArg().ofType(Integer.class);
    parser.accepts("max-connections", "The maximum number of connections (sockets) per node; same value as client configuration parameter \"" + ClientConfig.MAX_CONNECTIONS_PER_NODE_PROPERTY + "\"  Default = " + DEFAULT_MAX_CONNECTIONS).withRequiredArg().ofType(Integer.class);
    parser.accepts("max-threads", "The maximum number of threads used by the threaded RoutedStore implementation; same value as client configuration parameter \"" + ClientConfig.MAX_THREADS_PROPERTY + "\"  Default = " + DEFAULT_MAX_THREADS).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-nodes", "The number of nodes  Default = " + DEFAULT_NUM_NODES).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-slow-nodes", "The number of nodes that exhibit delay Default = " + DEFAULT_NUM_SLOW_NODES).withRequiredArg().ofType(Integer.class);
    parser.accepts("delay", "The millisecond delay shown by slow nodes Default = " + DEFAULT_DELAY).withRequiredArg().ofType(Integer.class);
    parser.accepts("num-clients", "The number of threads to make requests concurrently  Default = " + DEFAULT_NUM_CLIENTS).withRequiredArg().ofType(Integer.class);
    parser.accepts("routed-store-type", "Type of routed store, either \"" + THREAD_POOL_ROUTED_STORE + "\" or \"" + PIPELINE_ROUTED_STORE + "\"  Default = " + DEFAULT_ROUTED_STORE_TYPE).withRequiredArg();
    parser.accepts("help", "This help");
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        printUsage(System.out, parser);
    }
    final int numKeys = CmdUtils.valueOf(options, "num-keys", DEFAULT_NUM_KEYS);
    int maxConnectionsPerNode = CmdUtils.valueOf(options, "max-connections", DEFAULT_MAX_CONNECTIONS);
    int maxThreads = CmdUtils.valueOf(options, "max-threads", DEFAULT_MAX_THREADS);
    int numNodes = CmdUtils.valueOf(options, "num-nodes", DEFAULT_NUM_NODES);
    int numSlowNodes = CmdUtils.valueOf(options, "num-slow-nodes", DEFAULT_NUM_SLOW_NODES);
    int delay = CmdUtils.valueOf(options, "delay", DEFAULT_DELAY);
    int numClients = CmdUtils.valueOf(options, "num-clients", DEFAULT_NUM_CLIENTS);
    System.err.println("num-keys : " + numKeys);
    System.err.println("max-connections : " + maxConnectionsPerNode);
    System.err.println("max-threads : " + maxThreads);
    System.err.println("num-nodes : " + numNodes);
    System.err.println("num-slow-nodes : " + numSlowNodes);
    System.err.println("delay : " + delay);
    System.err.println("num-clients : " + numClients);
    ClientConfig clientConfig = new ClientConfig().setMaxConnectionsPerNode(maxConnectionsPerNode).setMaxThreads(maxThreads);
    String storeDefinitionFile = "test/common/voldemort/config/single-store.xml";
    StoreDefinition storeDefinition = new StoreDefinitionsMapper().readStoreList(new File(storeDefinitionFile)).get(0);
    SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(clientConfig.getSelectors(), clientConfig.getMaxConnectionsPerNode(), clientConfig.getConnectionTimeout(TimeUnit.MILLISECONDS), clientConfig.getSocketTimeout(TimeUnit.MILLISECONDS), clientConfig.getSocketBufferSize(), clientConfig.getSocketKeepAlive());
    VoldemortServer[] servers = new VoldemortServer[numNodes];
    int[][] partitionMap = new int[numNodes][1];
    for (int i = 0; i < numNodes; i++) {
        partitionMap[i][0] = i;
    }
    Cluster cluster = ServerTestUtils.startVoldemortCluster(numNodes, servers, partitionMap, socketStoreFactory, true, null, storeDefinitionFile, new Properties());
    Map<Integer, VoldemortServer> serverMap = new HashMap<Integer, VoldemortServer>();
    for (int i = 0; i < cluster.getNumberOfNodes(); i++) {
        serverMap.put(i, servers[i]);
        Store<ByteArray, byte[], byte[]> store = new InMemoryStorageEngine<ByteArray, byte[], byte[]>("test-sleepy");
        if (i < numSlowNodes)
            store = new SleepyStore<ByteArray, byte[], byte[]>(delay, store);
        StoreRepository storeRepository = servers[i].getStoreRepository();
        storeRepository.addLocalStore(store);
    }
    Map<Integer, Store<ByteArray, byte[], byte[]>> stores = new HashMap<Integer, Store<ByteArray, byte[], byte[]>>();
    for (Node node : cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> socketStore = ServerTestUtils.getSocketStore(socketStoreFactory, "test-sleepy", node.getSocketPort(), clientConfig.getRequestFormatType());
        stores.put(node.getId(), socketStore);
    }
    FailureDetectorConfig failureDetectorConfig = new FailureDetectorConfig().setImplementationClassName(BannagePeriodFailureDetector.class.getName()).setCluster(cluster).setConnectionVerifier(MutableStoreConnectionVerifier.create(stores));
    FailureDetector failureDetector = FailureDetectorUtils.create(failureDetectorConfig, false);
    ExecutorService routedStoreThreadPool = Executors.newFixedThreadPool(clientConfig.getMaxThreads());
    RoutedStoreFactory routedStoreFactory = new RoutedStoreFactory(routedStoreThreadPool);
    RoutedStoreConfig routedStoreConfig = new RoutedStoreConfig(clientConfig);
    final RoutedStore routedStore = routedStoreFactory.create(cluster, storeDefinition, stores, failureDetector, routedStoreConfig);
    ExecutorService runner = Executors.newFixedThreadPool(numClients);
    long start = System.nanoTime();
    try {
        for (int i = 0; i < numClients; i++) {
            runner.submit(new Runnable() {

                public void run() {
                    for (int i = 0; i < numKeys; i++) {
                        ByteArray key = new ByteArray(("test-key-" + i).getBytes());
                        try {
                            routedStore.get(key, null);
                        } catch (VoldemortException e) {
                        //
                        }
                    }
                }
            });
        }
        runner.shutdown();
        runner.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
        long time = (System.nanoTime() - start) / Time.NS_PER_MS;
        System.err.println("Time: " + time + " ms.");
    } finally {
        runner.shutdown();
    }
    if (failureDetector != null)
        failureDetector.destroy();
    for (VoldemortServer server : serverMap.values()) server.stop();
    if (routedStoreThreadPool != null)
        routedStoreThreadPool.shutdown();
    System.exit(0);
}

Example 42 with OptionSet

use of joptsimple.OptionSet in project voldemort by voldemort.

the class StorageEnginePerformanceTest method main.

public static void main(String[] args) throws Exception {
    try {
        OptionParser parser = new OptionParser();
        parser.accepts("help", "print usage information");
        parser.accepts("requests", "[REQUIRED] number of requests to execute").withRequiredArg().ofType(Integer.class);
        parser.accepts("num-values", "[REQUIRED] number of values in the store").withRequiredArg().ofType(Integer.class);
        parser.accepts("data-dir", "Data directory for storage data").withRequiredArg().describedAs("directory");
        parser.accepts("threads", "number of threads").withRequiredArg().ofType(Integer.class);
        parser.accepts("storage-configuration-class", "[REQUIRED] class of the storage engine configuration to use [e.g. voldemort.store.bdb.BdbStorageConfiguration]").withRequiredArg().describedAs("class_name");
        parser.accepts("props", "Properties file with configuration for the engine").withRequiredArg().describedAs("config.properties");
        parser.accepts("value-size", "The size of the values in the store").withRequiredArg().describedAs("size").ofType(Integer.class);
        parser.accepts("cache-width", "Percentage of requests to save as possible re-requests").withRequiredArg().describedAs("width").ofType(Integer.class);
        parser.accepts("cache-hit-ratio", "Percentage of requests coming from the last cache-width requests").withRequiredArg().describedAs("ratio").ofType(Double.class);
        parser.accepts("clean-up", "Delete data directory when done.");
        OptionSet options = parser.parse(args);
        if (options.has("help")) {
            parser.printHelpOn(System.out);
            System.exit(0);
        }
        CmdUtils.croakIfMissing(parser, options, "requests");
        final int numThreads = CmdUtils.valueOf(options, "threads", 10);
        final int numRequests = (Integer) options.valueOf("requests");
        final int numValues = (Integer) options.valueOf("num-values");
        final int valueSize = CmdUtils.valueOf(options, "value-size", 1024);
        final int cacheWidth = CmdUtils.valueOf(options, "cache-width", 100000);
        final double cacheHitRatio = CmdUtils.valueOf(options, "cache-hit-ratio", 0.5);
        final String propsFile = (String) options.valueOf("props");
        final boolean cleanUp = options.has("clean-up");
        final String storageEngineClass = CmdUtils.valueOf(options, "storage-configuration-class", BdbStorageConfiguration.class.getName()).trim();
        File dataDir = null;
        if (options.has("data-dir"))
            dataDir = new File((String) options.valueOf("data-dir"));
        else
            dataDir = TestUtils.createTempDir();
        System.out.println("Data dir: " + dataDir);
        // create the storage engine
        Props props = new Props();
        if (propsFile != null)
            props = new Props(new File(propsFile));
        props.put("node.id", 0);
        props.put("data.directory", dataDir.getAbsolutePath());
        props.put("voldemort.home", System.getProperty("user.dir"));
        VoldemortConfig config = new VoldemortConfig(props);
        StorageConfiguration storageConfig = (StorageConfiguration) ReflectUtils.callConstructor(ReflectUtils.loadClass(storageEngineClass), new Object[] { config });
        StorageEngine<ByteArray, byte[], byte[]> engine = storageConfig.getStore(TestUtils.makeStoreDefinition("test"), TestUtils.makeSingleNodeRoutingStrategy());
        @SuppressWarnings("unchecked") final Store<String, byte[], byte[]> store = new SerializingStore(engine, new StringSerializer(), new IdentitySerializer(), null);
        final byte[] value = new byte[valueSize];
        new Random().nextBytes(value);
        // initialize test data
        for (int i = 0; i < numValues; i++) store.put(Integer.toString(i), Versioned.value(value), null);
        // initialize cache lookback data
        int[] recents = new int[cacheWidth];
        System.out.println("Write test:");
        CachedPerformanceTest writeTest = new CachedPerformanceTest(new PerformanceTest() {

            @Override
            public void doOperation(int index) {
                try {
                    String key = Integer.toString(index);
                    List<Versioned<byte[]>> vs = store.get(key, null);
                    VectorClock version;
                    if (vs.size() == 0)
                        version = new VectorClock();
                    else
                        version = (VectorClock) vs.get(0).getVersion();
                    version.incrementVersion(0, 847584375);
                    store.put(key, Versioned.value(value, version), null);
                } catch (ObsoleteVersionException e) {
                // do nothing
                } catch (RuntimeException e) {
                    e.printStackTrace();
                    throw e;
                }
            }
        }, recents, numValues, cacheHitRatio);
        writeTest.run(numRequests, numThreads);
        writeTest.printStats();
        System.out.println();
        System.out.println("Read test:");
        CachedPerformanceTest readTest = new CachedPerformanceTest(new PerformanceTest() {

            @Override
            public void doOperation(int index) {
                store.get(Integer.toString(index), null);
            }
        }, recents, numValues, cacheHitRatio);
        readTest.run(numRequests, numThreads);
        readTest.printStats();
        if (cleanUp)
            Utils.rm(dataDir);
    } catch (Exception e) {
        e.printStackTrace();
        System.exit(1);
    }
}

Example 43 with OptionSet

use of joptsimple.OptionSet in project voldemort by voldemort.

the class Benchmark method main.

public static void main(String[] args) throws IOException {
    // Logger.getRootLogger().removeAllAppenders();
    OptionParser parser = new OptionParser();
    parser.accepts(READS, "percentage of --ops-count to be reads; valid values [0-100]").withRequiredArg().describedAs("read-percent").ofType(Integer.class);
    parser.accepts(WRITES, "percentage of --ops-count to be writes; valid values [0-100]").withRequiredArg().describedAs("write-percent").ofType(Integer.class);
    parser.accepts(DELETES, "percentage of --ops-count to be deletes; valid values [0-100]").withRequiredArg().describedAs("delete-percent").ofType(Integer.class);
    parser.accepts(MIXED, "percentage of --ops-count to be updates; valid values [0-100]").withRequiredArg().describedAs("update-percent").ofType(Integer.class);
    parser.accepts(SAMPLE_SIZE, "number of value samples to be obtained from the store for replay based on keys from request-file; 0 means no sample value replay. Default = 0").withRequiredArg().describedAs("sample-size").ofType(Integer.class);
    parser.accepts(VERBOSE, "verbose");
    parser.accepts(THREADS, "max number concurrent worker threads; Default = " + MAX_WORKERS).withRequiredArg().describedAs("num-threads").ofType(Integer.class);
    parser.accepts(NUM_CONNECTIONS_PER_NODE, "max number of connections to any node; Default = " + MAX_CONNECTIONS_PER_NODE).withRequiredArg().describedAs("num-connections-per-node").ofType(Integer.class);
    parser.accepts(ITERATIONS, "number of times to repeat benchmark phase; Default = 1").withRequiredArg().describedAs("num-iter").ofType(Integer.class);
    parser.accepts(VERIFY, "verify values read; runs only if warm-up phase is included");
    parser.accepts(PERCENT_CACHED, "percentage of requests to come from previously requested keys; valid values are in range [0..100]; 0 means caching disabled. Default = 0").withRequiredArg().describedAs("percent").ofType(Integer.class);
    parser.accepts(START_KEY_INDEX, "key index to start warm-up phase from; Default = 0").withRequiredArg().describedAs("index").ofType(Integer.class);
    parser.accepts(INTERVAL, "print status at interval seconds; Default = 0").withRequiredArg().describedAs("sec").ofType(Integer.class);
    parser.accepts(IGNORE_NULLS, "ignore null values in results");
    parser.accepts(PROP_FILE, "file containing all the properties in key=value format; will override all other command line options specified").withRequiredArg().describedAs("prop-file");
    parser.accepts(STORAGE_CONFIGURATION_CLASS, "class of the storage engine configuration to use [e.g. voldemort.store.bdb.BdbStorageConfiguration]").withRequiredArg().describedAs("class-name");
    parser.accepts(KEY_TYPE, "for local tests; key type to support; [ " + IDENTITY_KEY_TYPE + " | " + JSONINT_KEY_TYPE + " | " + JSONSTRING_KEY_TYPE + "|" + STRING_KEY_TYPE + " <default> ]").withRequiredArg().describedAs("type");
    parser.accepts(REQUEST_FILE, "file with limited list of keys to be used during benchmark phase; Overrides " + RECORD_SELECTION).withRequiredArg();
    parser.accepts(VALUE_SIZE, "size in bytes for random value; used during warm-up phase and write operation of benchmark phase; Default = 1024").withRequiredArg().describedAs("bytes").ofType(Integer.class);
    parser.accepts(RECORD_SELECTION, "record selection distribution [ " + ZIPFIAN_RECORD_SELECTION + " | " + LATEST_RECORD_SELECTION + " | " + UNIFORM_RECORD_SELECTION + " <default> ]").withRequiredArg();
    parser.accepts(TARGET_THROUGHPUT, "fix throughput").withRequiredArg().describedAs("ops/sec").ofType(Integer.class);
    parser.accepts(RECORD_COUNT, "number of records inserted during warmup phase").withRequiredArg().describedAs("count").ofType(Integer.class);
    parser.accepts(OPS_COUNT, "number of operations to do during benchmark phase").withRequiredArg().describedAs("count").ofType(Integer.class);
    parser.accepts(URL, "for remote tests; url of remote server").withRequiredArg();
    parser.accepts(STORE_NAME, "for remote tests; store name on the remote " + URL).withRequiredArg().describedAs("name");
    parser.accepts(METRIC_TYPE, "type of result metric [ " + HISTOGRAM_METRIC_TYPE + " | " + SUMMARY_METRIC_TYPE + " <default> ]").withRequiredArg();
    parser.accepts(PLUGIN_CLASS, "classname of implementation of WorkloadPlugin; used to run customized operations ").withRequiredArg().describedAs("class-name");
    parser.accepts(CLIENT_ZONE_ID, "zone id for client; enables zone routing").withRequiredArg().describedAs("zone-id").ofType(Integer.class);
    parser.accepts(LOCAL_SERVER_PROPERTIES, "path to a server properties file (local mode only)").withRequiredArg().describedAs(LOCAL_SERVER_PROPERTIES).ofType(String.class);
    parser.accepts(WORKLOAD_TYPE, "workload type; type to support; [ " + TRACE_WORKLOAD_TYPE + " | " + DEFAULT_WORKLOAD_TYPE + " <default> ]").withRequiredArg().describedAs(WORKLOAD_TYPE).ofType(String.class);
    parser.accepts(KEY_VALUE_FILE, "path to a file with keys and value sizes").withRequiredArg().describedAs(KEY_VALUE_FILE).ofType(String.class);
    parser.accepts(KEY_SEQUENCE_FILE, "path to a file with key access sequence").withRequiredArg().describedAs(KEY_SEQUENCE_FILE).ofType(String.class);
    parser.accepts(HELP);
    OptionSet options = parser.parse(args);
    if (options.has(HELP)) {
        parser.printHelpOn(System.out);
        System.exit(0);
    }
    Props mainProps = null;
    if (options.has(PROP_FILE)) {
        String propFileDestination = (String) options.valueOf(PROP_FILE);
        File propertyFile = new File(propFileDestination);
        if (!propertyFile.exists()) {
            printUsage(parser, "Property file does not exist");
        }
        try {
            mainProps = new Props(propertyFile);
        } catch (Exception e) {
            printUsage(parser, "Unable to parse the property file");
        }
    } else {
        mainProps = new Props();
        if (options.has(REQUEST_FILE)) {
            mainProps.put(REQUEST_FILE, (String) options.valueOf(REQUEST_FILE));
            mainProps.put(RECORD_SELECTION, FILE_RECORD_SELECTION);
        } else {
            mainProps.put(RECORD_SELECTION, CmdUtils.valueOf(options, RECORD_SELECTION, UNIFORM_RECORD_SELECTION));
        }
        if (options.has(RECORD_COUNT)) {
            mainProps.put(RECORD_COUNT, (Integer) options.valueOf(RECORD_COUNT));
        } else {
            mainProps.put(RECORD_COUNT, 0);
        }
        if (!options.has(OPS_COUNT)) {
            printUsage(parser, "Missing " + OPS_COUNT);
        }
        mainProps.put(OPS_COUNT, (Integer) options.valueOf(OPS_COUNT));
        if (options.has(URL)) {
            mainProps.put(URL, (String) options.valueOf(URL));
            if (options.has(STORE_NAME)) {
                mainProps.put(STORE_NAME, (String) options.valueOf(STORE_NAME));
            } else {
                printUsage(parser, "Missing store name");
            }
        } else {
            mainProps.put(KEY_TYPE, CmdUtils.valueOf(options, KEY_TYPE, STRING_KEY_TYPE));
            mainProps.put(STORAGE_CONFIGURATION_CLASS, CmdUtils.valueOf(options, STORAGE_CONFIGURATION_CLASS, BdbStorageConfiguration.class.getName()));
        }
        if (options.has(LOCAL_SERVER_PROPERTIES)) {
            mainProps.put(LOCAL_SERVER_PROPERTIES, (String) options.valueOf(LOCAL_SERVER_PROPERTIES));
        }
        if (options.has(WORKLOAD_TYPE)) {
            mainProps.put(WORKLOAD_TYPE, (String) options.valueOf(WORKLOAD_TYPE));
            if (((String) options.valueOf(WORKLOAD_TYPE)).equals(TRACE_WORKLOAD_TYPE)) {
                CmdUtils.croakIfMissing(parser, options, KEY_VALUE_FILE, KEY_SEQUENCE_FILE);
                mainProps.put(KEY_VALUE_FILE, (String) options.valueOf(KEY_VALUE_FILE));
                mainProps.put(KEY_SEQUENCE_FILE, (String) options.valueOf(KEY_SEQUENCE_FILE));
            }
        }
        mainProps.put(VERBOSE, getCmdBoolean(options, VERBOSE));
        mainProps.put(VERIFY, getCmdBoolean(options, VERIFY));
        mainProps.put(IGNORE_NULLS, getCmdBoolean(options, IGNORE_NULLS));
        mainProps.put(CLIENT_ZONE_ID, CmdUtils.valueOf(options, CLIENT_ZONE_ID, -1));
        mainProps.put(START_KEY_INDEX, CmdUtils.valueOf(options, START_KEY_INDEX, 0));
        mainProps.put(VALUE_SIZE, CmdUtils.valueOf(options, VALUE_SIZE, 1024));
        mainProps.put(ITERATIONS, CmdUtils.valueOf(options, ITERATIONS, 1));
        mainProps.put(THREADS, CmdUtils.valueOf(options, THREADS, MAX_WORKERS));
        mainProps.put(NUM_CONNECTIONS_PER_NODE, CmdUtils.valueOf(options, NUM_CONNECTIONS_PER_NODE, MAX_CONNECTIONS_PER_NODE));
        mainProps.put(PERCENT_CACHED, CmdUtils.valueOf(options, PERCENT_CACHED, 0));
        mainProps.put(INTERVAL, CmdUtils.valueOf(options, INTERVAL, 0));
        mainProps.put(TARGET_THROUGHPUT, CmdUtils.valueOf(options, TARGET_THROUGHPUT, -1));
        mainProps.put(METRIC_TYPE, CmdUtils.valueOf(options, METRIC_TYPE, SUMMARY_METRIC_TYPE));
        mainProps.put(READS, CmdUtils.valueOf(options, READS, 0));
        mainProps.put(WRITES, CmdUtils.valueOf(options, WRITES, 0));
        mainProps.put(DELETES, CmdUtils.valueOf(options, DELETES, 0));
        mainProps.put(MIXED, CmdUtils.valueOf(options, MIXED, 0));
        mainProps.put(PLUGIN_CLASS, CmdUtils.valueOf(options, PLUGIN_CLASS, ""));
        mainProps.put(SAMPLE_SIZE, CmdUtils.valueOf(options, SAMPLE_SIZE, 0));
    }
    // Start the benchmark
    Benchmark benchmark = null;
    try {
        benchmark = new Benchmark();
        benchmark.initialize(mainProps);
        benchmark.warmUpAndRun();
        benchmark.close();
    } catch (Exception e) {
        if (options.has(VERBOSE)) {
            e.printStackTrace();
        }
        parser.printHelpOn(System.err);
        System.exit(-1);
    }
}

Example 44 with OptionSet

use of joptsimple.OptionSet in project voldemort by voldemort.

the class MysqlBench method main.

public static void main(String[] args) throws Exception {
    OptionParser parser = new OptionParser();
    parser.accepts("help", "print usage information");
    parser.acceptsAll(asList("r", "reads"), "Enable reads.");
    parser.acceptsAll(asList("w", "writes"), "Enable writes.");
    parser.acceptsAll(asList("d", "deletes"), "Enable deletes.");
    parser.accepts("table", "Table name").withRequiredArg();
    parser.accepts("db", "Database name").withRequiredArg();
    parser.acceptsAll(asList("u", "user"), "DB username.").withRequiredArg();
    parser.acceptsAll(asList("P", "password"), "DB password").withRequiredArg();
    parser.acceptsAll(asList("p", "port"), "DB port").withRequiredArg();
    parser.acceptsAll(asList("h", "host"), "DB host").withRequiredArg();
    parser.accepts("requests").withRequiredArg().ofType(Integer.class);
    parser.accepts("request-file").withRequiredArg();
    parser.accepts("threads").withRequiredArg().ofType(Integer.class);
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        parser.printHelpOn(System.out);
        System.exit(0);
    }
    Set<String> missing = CmdUtils.missing(options, "table", "requests", "db");
    if (missing.size() > 0)
        Utils.croak("Missing required arguments: " + Joiner.on(", ").join(missing));
    String host = CmdUtils.valueOf(options, "host", "localhost");
    String table = (String) options.valueOf("table");
    int port = CmdUtils.valueOf(options, "port", 3306);
    String database = (String) options.valueOf("db");
    String jdbcUrl = "jdbc:mysql://" + host + ":" + port + "/" + database;
    String user = CmdUtils.valueOf(options, "user", "root");
    String password = CmdUtils.valueOf(options, "password", "");
    String requestFile = (String) options.valueOf("request-file");
    int numRequests = (Integer) options.valueOf("requests");
    int numThreads = CmdUtils.valueOf(options, "threads", 10);
    boolean doReads = false;
    boolean doWrites = false;
    if (options.has("reads") || options.has("writes")) {
        doReads = options.has("reads");
        doWrites = options.has("writes");
    } else {
        doReads = true;
        doWrites = true;
    }
    MysqlBench bench = new MysqlBench(table, numThreads, numRequests, jdbcUrl, user, password, requestFile, doReads, doWrites);
    bench.benchmark();
}

Example 45 with OptionSet

use of joptsimple.OptionSet in project voldemort by voldemort.

the class ReadOnlyStorePerformanceTest method main.

public static void main(String[] args) throws FileNotFoundException, IOException {
    OptionParser parser = new OptionParser();
    parser.accepts("help", "print usage information");
    parser.accepts("threads", "number of threads").withRequiredArg().ofType(Integer.class);
    parser.accepts("requests", "[REQUIRED] number of requests").withRequiredArg().ofType(Integer.class);
    parser.accepts("store-dir", "[REQUIRED] store directory").withRequiredArg().describedAs("directory");
    parser.accepts("cluster-xml", "Path to cluster.xml").withRequiredArg().describedAs("path");
    parser.accepts("node-id", "Id of node").withRequiredArg().ofType(Integer.class).describedAs("node-id");
    parser.accepts("search-strategy", "class of the search strategy to use").withRequiredArg().describedAs("class_name");
    parser.accepts("build", "If present, first build the data");
    parser.accepts("num-values", "The number of values in the store").withRequiredArg().describedAs("count").ofType(Integer.class);
    parser.accepts("num-chunks", "The number of chunks per partition").withRequiredArg().describedAs("chunks").ofType(Integer.class);
    parser.accepts("internal-sort-size", "The number of items to sort in memory at a time").withRequiredArg().describedAs("size").ofType(Integer.class);
    parser.accepts("value-size", "The size of the values in the store").withRequiredArg().describedAs("size").ofType(Integer.class);
    parser.accepts("working-dir", "The directory in which to store temporary data").withRequiredArg().describedAs("dir");
    parser.accepts("gzip", "Compress the intermediate temp files used in building the store");
    parser.accepts("request-file", "file get request ids from").withRequiredArg();
    parser.accepts("version", "Version of read-only store [" + ReadOnlyStorageFormat.READONLY_V0 + "," + ReadOnlyStorageFormat.READONLY_V1 + "," + ReadOnlyStorageFormat.READONLY_V2 + " (default)]").withRequiredArg().describedAs("version");
    parser.accepts("test-gz", "Path to gzip containing data. Works with --build only").withRequiredArg().describedAs("path");
    OptionSet options = parser.parse(args);
    if (options.has("help")) {
        parser.printHelpOn(System.out);
        System.exit(0);
    }
    CmdUtils.croakIfMissing(parser, options, "requests", "store-dir");
    final int numThreads = CmdUtils.valueOf(options, "threads", 10);
    final int numRequests = (Integer) options.valueOf("requests");
    final int internalSortSize = CmdUtils.valueOf(options, "internal-sort-size", 500000);
    int numValues = numRequests;
    final String inputFile = (String) options.valueOf("request-file");
    final String searcherClass = CmdUtils.valueOf(options, "search-strategy", BinarySearchStrategy.class.getName()).trim();
    final boolean gzipIntermediate = options.has("gzip");
    final SearchStrategy searcher = (SearchStrategy) ReflectUtils.callConstructor(ReflectUtils.loadClass(searcherClass));
    final File workingDir = new File(CmdUtils.valueOf(options, "working-dir", System.getProperty("java.io.tmpdir")));
    String storeDir = (String) options.valueOf("store-dir");
    ReadOnlyStorageFormat format = ReadOnlyStorageFormat.fromCode(CmdUtils.valueOf(options, "version", ReadOnlyStorageFormat.READONLY_V2.toString()));
    Cluster cluster = null;
    int nodeId = 0;
    SerializerDefinition sdef = new SerializerDefinition("json", "'string'");
    StoreDefinition storeDef = new StoreDefinitionBuilder().setName("test").setKeySerializer(sdef).setValueSerializer(sdef).setRequiredReads(1).setReplicationFactor(1).setRequiredWrites(1).setType("read-only").setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setRoutingPolicy(RoutingTier.CLIENT).build();
    if (options.has("build")) {
        CmdUtils.croakIfMissing(parser, options, "num-values", "value-size");
        numValues = (Integer) options.valueOf("num-values");
        int numChunks = 1;
        if (options.has("num-chunks"))
            numChunks = (Integer) options.valueOf("num-chunks");
        int valueSize = (Integer) options.valueOf("value-size");
        // generate test data
        File temp = null;
        if (options.has("test-gz")) {
            temp = new File((String) options.valueOf("test-gz"));
        } else {
            temp = File.createTempFile("json-data", ".txt.gz", workingDir);
            temp.deleteOnExit();
            System.out.println("Generating test data in " + temp);
            OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(temp));
            Writer writer = new BufferedWriter(new OutputStreamWriter(outputStream), 10 * 1024 * 1024);
            String value = TestUtils.randomLetters(valueSize);
            for (int i = 0; i < numValues; i++) {
                writer.write("\"");
                writer.write(Integer.toString(i));
                writer.write("\" \"");
                writer.write(value);
                writer.write("\"");
                writer.write("\n");
            }
            writer.close();
            writer = null;
        }
        System.out.println("Building store.");
        InputStream inputStream = new GZIPInputStream(new FileInputStream(temp));
        Reader r = new BufferedReader(new InputStreamReader(inputStream), 1 * 1024 * 1024);
        File output = TestUtils.createTempDir(workingDir);
        File tempDir = TestUtils.createTempDir(workingDir);
        cluster = ServerTestUtils.getLocalCluster(1);
        nodeId = 0;
        JsonStoreBuilder builder = new JsonStoreBuilder(new JsonReader(r), cluster, storeDef, new ConsistentRoutingStrategy(cluster, 1), output, tempDir, internalSortSize, 2, numChunks, 64 * 1024, gzipIntermediate);
        builder.build(format);
        // copy to store dir
        File dir = new File(storeDir);
        Utils.rm(dir);
        dir.mkdirs();
        System.out.println("Moving store data from " + output + " to " + dir);
        boolean copyWorked = new File(output, "node-0").renameTo(new File(dir, "version-0"));
        if (!copyWorked)
            Utils.croak("Copy of data from " + output + " to " + dir + " failed.");
    } else {
        CmdUtils.croakIfMissing(parser, options, "cluster-xml", "node-id");
        String clusterXmlPath = (String) options.valueOf("cluster-xml");
        nodeId = (Integer) options.valueOf("node-id");
        File clusterXml = new File(clusterXmlPath);
        if (!clusterXml.exists()) {
            Utils.croak("Cluster.xml does not exist");
        }
        cluster = new ClusterMapper().readCluster(clusterXml);
    }
    final Store<ByteArray, byte[], byte[]> store = new ReadOnlyStorageEngine("test", searcher, new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster), nodeId, new File(storeDir), 0);
    final AtomicInteger obsoletes = new AtomicInteger(0);
    final AtomicInteger nullResults = new AtomicInteger(0);
    final AtomicInteger totalResults = new AtomicInteger(0);
    final BlockingQueue<String> requestIds = new ArrayBlockingQueue<String>(20000);
    final Executor executor = Executors.newFixedThreadPool(1);
    // if they have given us a file make a request generator that reads from
    // it, otherwise just generate random values
    final int numVals = numValues;
    Runnable requestGenerator;
    if (inputFile == null) {
        requestGenerator = new Runnable() {

            public void run() {
                System.out.println("Generating random requests.");
                Random random = new Random();
                try {
                    while (true) requestIds.put(Integer.toString(random.nextInt(numRequests) % numVals));
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        };
    } else {
        requestGenerator = new Runnable() {

            public void run() {
                try {
                    System.out.println("Using request file to generate requests.");
                    BufferedReader reader = new BufferedReader(new FileReader(inputFile), 1000000);
                    while (true) {
                        String line = reader.readLine();
                        if (line == null)
                            return;
                        requestIds.put(line.trim());
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        };
    }
    executor.execute(requestGenerator);
    final Serializer<Object> keySerializer = new JsonTypeSerializer(JsonTypeDefinition.fromJson("'string'"), true);
    final AtomicInteger current = new AtomicInteger();
    final int progressIncrement = numRequests / 5;
    PerformanceTest readWriteTest = new PerformanceTest() {

        @Override
        public void doOperation(int index) throws Exception {
            try {
                totalResults.incrementAndGet();
                int curr = current.getAndIncrement();
                List<Versioned<byte[]>> results = store.get(new ByteArray(keySerializer.toBytes(requestIds.take())), null);
                if (curr % progressIncrement == 0)
                    System.out.println(curr);
                if (results.size() == 0)
                    nullResults.incrementAndGet();
            } catch (ObsoleteVersionException e) {
                obsoletes.incrementAndGet();
            }
        }
    };
    System.out.println("Running test...");
    readWriteTest.run(numRequests, numThreads);
    System.out.println("Random Access Read Only store Results:");
    System.out.println("Null reads ratio:" + (nullResults.doubleValue()) / totalResults.doubleValue());
    readWriteTest.printStats();
    System.exit(0);
}