Examples with OptionSet joptsimple.OptionSet used on opensource projects

Example 51 with OptionSet

use of joptsimple.OptionSet in project voldemort by voldemort.

the class ExportBDBToTextDump method main.

public static void main(String[] argv) throws Exception {
    OptionParser parser = getParser();
    OptionSet options = parser.parse(argv);
    validateOptions(options);
    // bdb_folder output_folder
    String storeBdbFolderPath = (String) options.valueOf("bdb");
    String outputFolderPath = (String) options.valueOf("output");
    File storeBdbFolder = new File(storeBdbFolderPath);
    File outputFolder = new File(outputFolderPath);
    final String storeName = storeBdbFolder.getName();
    Properties properties = new Properties();
    properties.put("node.id", "0");
    properties.put("voldemort.home", storeBdbFolder.getParent());
    VoldemortConfig voldemortConfig = new VoldemortConfig(properties);
    voldemortConfig.setBdbDataDirectory(storeBdbFolder.getParent());
    voldemortConfig.setEnableJmx(false);
    voldemortConfig.setBdbOneEnvPerStore(true);
    BdbStorageConfiguration bdbConfiguration = new BdbStorageConfiguration(voldemortConfig);
    class MockStoreDefinition extends StoreDefinition {

        public MockStoreDefinition() {
            super(storeName, null, null, null, null, null, null, null, 0, null, 0, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, 0);
        }

        @Override
        public boolean hasMemoryFootprint() {
            return false;
        }
    }
    StoreDefinition storeDef = new MockStoreDefinition();
    StorageEngine<ByteArray, byte[], byte[]> engine = bdbConfiguration.getStore(storeDef, null);
    long reportIntervalMs = 10000L;
    long lastCount = 0;
    Reporter<Boolean> rp = new Reporter<Boolean>(reportIntervalMs);
    long count = 0;
    BufferedWriter splitFileWriter = null;
    ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entries = engine.entries();
    while (entries.hasNext()) {
        if (splitFileWriter == null) {
            long splitId = count / SPLIT_SIZE;
            File splitFile = new File(outputFolder, makeSplitFileName(splitId));
            splitFileWriter = new BufferedWriter(new FileWriter(splitFile), WRITER_BUFFER_SIZE);
        }
        Pair<ByteArray, Versioned<byte[]>> pair = entries.next();
        String line = makeLine(pair);
        splitFileWriter.write(line);
        if ((count + 1) % SPLIT_SIZE == 0) {
            splitFileWriter.close();
            splitFileWriter = null;
        }
        count++;
        final Long countObject = count;
        Boolean reported = rp.tryReport(new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                System.out.print(String.format("Exported %15d entries", countObject));
                return true;
            }
        });
        if (reported != null) {
            System.out.println(String.format("; Speed: %8d/s", (count - lastCount) / (reportIntervalMs / 1000)));
            lastCount = count;
        }
    }
    entries.close();
    if (splitFileWriter != null) {
        splitFileWriter.close();
    }
    System.out.println(String.format("Finished exporting %d entries", count));
}

Example 52 with OptionSet

use of joptsimple.OptionSet in project voldemort by voldemort.

the class GenerateScriptCLI method main.

public static void main(String[] args) throws IOException {
    OptionParser parser = null;
    OptionSet options = null;
    try {
        parser = setupParser();
        options = parser.parse(args);
    } catch (OptionException oe) {
        parser.printHelpOn(System.out);
        printUsageAndDie("Exception when parsing arguments : " + oe.getMessage());
        return;
    }
    /* validate options */
    if (options.has("help")) {
        printUsage();
        return;
    }
    if (!options.hasArgument("url") || !options.hasArgument("script") || !options.hasArgument("output")) {
        printUsageAndDie("Missing a required argument.");
        return;
    }
    String url = (String) options.valueOf("url");
    String inputScriptPath = getFilePath(options, "script");
    String outputScriptPath = getFilePath(options, "output");
    String scpFilePath = getFilePath(options, "scp");
    AdminClient client = AdminToolUtils.getAdminClient(url);
    PrintWriter writer = new PrintWriter(outputScriptPath, "UTF-8");
    for (Node node : client.getAdminClientCluster().getNodes()) {
        FileInputStream fis = new FileInputStream(inputScriptPath);
        BufferedReader br = new BufferedReader(new InputStreamReader(fis));
        if (scpFilePath != null && scpFilePath.length() > 0) {
            writer.println("scp " + scpFilePath + "  " + node.getHost() + ":~");
        }
        int randomNumber = 1000 + new Random().nextInt(100000);
        String hereDocumentTag = "NODE_" + node.getId() + "_" + randomNumber;
        // Use SSH here script
        writer.println("ssh -T " + node.getHost() + "  << " + hereDocumentTag);
        String line = null;
        while ((line = br.readLine()) != null) {
            line = line.replace("@@ID@@", Integer.toString(node.getId()));
            line = line.replace("@@HOST@@", node.getHost());
            line = line.replace("@@URL@@", node.getSocketUrl().toString());
            writer.println(line);
        }
        writer.println(hereDocumentTag);
        writer.println("\n\n\n");
        br.close();
    }
    writer.close();
    System.out.println("Output script generated at " + outputScriptPath);
}

Example 53 with OptionSet

use of joptsimple.OptionSet in project voldemort by voldemort.

the class ImportTextDumpToBDB method main.

public static void main(String[] argv) throws Exception {
    OptionParser parser = getParser();
    OptionSet options = parser.parse(argv);
    validateOptions(options);
    String inputPath = (String) options.valueOf("input");
    String storeBdbFolderPath = (String) options.valueOf("bdb");
    String clusterXmlPath = (String) options.valueOf("cluster-xml");
    String storesXmlPath = (String) options.valueOf("stores-xml");
    Integer nodeId = (Integer) options.valueOf("node-id");
    File input = new File(inputPath);
    List<File> dataFiles = new ArrayList<File>();
    if (input.isDirectory()) {
        File[] files = input.listFiles();
        if (files != null)
            Collections.addAll(dataFiles, files);
    } else if (input.isFile()) {
        dataFiles.add(input);
    } else {
        System.err.println(inputPath + "is not file or directory");
    }
    File storeBdbFolder = new File(storeBdbFolderPath);
    final String storeName = storeBdbFolder.getName();
    Cluster cluster = new ClusterMapper().readCluster(new File(clusterXmlPath));
    List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new File(storesXmlPath));
    StoreDefinition storeDef = null;
    for (StoreDefinition sd : storeDefs) {
        if (sd.getName() != null && sd.getName().equals(storeName)) {
            storeDef = sd;
        }
    }
    if (storeDef == null) {
        throw new VoldemortException("StoreNotfound: " + storeName);
    }
    RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, cluster);
    Properties properties = new Properties();
    properties.put("node.id", "0");
    properties.put("voldemort.home", storeBdbFolder.getParent());
    VoldemortConfig voldemortConfig = new VoldemortConfig(properties);
    voldemortConfig.setBdbDataDirectory(storeBdbFolder.getParent());
    voldemortConfig.setEnableJmx(false);
    voldemortConfig.setBdbOneEnvPerStore(true);
    BdbStorageConfiguration bdbConfiguration = new BdbStorageConfiguration(voldemortConfig);
    class MockStoreDefinition extends StoreDefinition {

        public MockStoreDefinition() {
            super(storeName, null, null, null, null, null, null, null, 0, null, 0, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, 0);
        }

        @Override
        public boolean hasMemoryFootprint() {
            return false;
        }
    }
    StoreDefinition mockStoreDef = new MockStoreDefinition();
    StorageEngine<ByteArray, byte[], byte[]> engine = bdbConfiguration.getStore(mockStoreDef, routingStrategy);
    long reportIntervalMs = 10000L;
    long lastCount = 0;
    long lastInserted = 0;
    Reporter<Boolean> rp = new Reporter<Boolean>(reportIntervalMs);
    long count = 0;
    long inserted = 0;
    for (File f : dataFiles) {
        try {
            BufferedReader bufferedReader = new BufferedReader(new FileReader(f), READER_BUFFER_SIZE);
            engine.beginBatchModifications();
            while (true) {
                String line = bufferedReader.readLine();
                if (line == null) {
                    break;
                }
                Pair<ByteArray, Versioned<byte[]>> entry;
                try {
                    entry = lineToEntry(line);
                } catch (Exception e) {
                    System.err.println("Skipping line: " + line);
                    e.printStackTrace();
                    continue;
                }
                ByteArray key = entry.getFirst();
                List<Node> nodeList = routingStrategy.routeRequest(key.get());
                for (Node node : nodeList) {
                    if (nodeId == node.getId()) {
                        try {
                            engine.put(key, entry.getSecond(), null);
                            inserted++;
                        } catch (ObsoleteVersionException e) {
                            e.printStackTrace();
                        }
                        break;
                    }
                }
                count++;
                final Long countObject = count;
                final Long insertedObject = inserted;
                Boolean reported = rp.tryReport(new Callable<Boolean>() {

                    @Override
                    public Boolean call() throws Exception {
                        System.out.print(String.format("Imported %15d entries; Inserted %15d entries", countObject, insertedObject));
                        return true;
                    }
                });
                if (reported != null) {
                    long importSpeed = (count - lastCount) / (reportIntervalMs / 1000);
                    long insertSpeed = (inserted - lastInserted) / (reportIntervalMs / 1000);
                    System.out.println(String.format("; ImportSpeed: %8d/s; InsertSpeed: %8d/s ", importSpeed, insertSpeed));
                    lastCount = count;
                    lastInserted = inserted;
                }
            }
            bufferedReader.close();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            engine.endBatchModifications();
        }
    }
    engine.close();
    System.out.println(String.format("Finished importing %d entries (%d inserted, rest discarded)", count, inserted));
}

Example 54 with OptionSet

use of joptsimple.OptionSet in project voldemort by voldemort.

the class KeySamplerCLI method main.

public static void main(String[] args) throws Exception {
    OptionParser parser = null;
    OptionSet options = null;
    try {
        parser = getParser();
        options = parser.parse(args);
    } catch (OptionException oe) {
        parser.printHelpOn(System.out);
        printUsageAndDie("Exception when parsing arguments : " + oe.getMessage());
        return;
    }
    /* validate options */
    if (options.hasArgument("help")) {
        parser.printHelpOn(System.out);
        printUsage();
        return;
    }
    if (!options.hasArgument("url") || !options.hasArgument("out-dir")) {
        parser.printHelpOn(System.out);
        printUsageAndDie("Missing a required argument.");
        return;
    }
    String url = (String) options.valueOf("url");
    String outDir = (String) options.valueOf("out-dir");
    Utils.mkdirs(new File(outDir));
    List<String> storeNames = null;
    if (options.hasArgument("store-names")) {
        @SuppressWarnings("unchecked") List<String> list = (List<String>) options.valuesOf("store-names");
        storeNames = list;
    }
    List<Integer> partitionIds = null;
    if (options.hasArgument("partition-ids")) {
        @SuppressWarnings("unchecked") List<Integer> list = (List<Integer>) options.valuesOf("partition-ids");
        partitionIds = list;
    }
    Integer nodeParallelism = DEFAULT_NODE_PARALLELISM;
    if (options.hasArgument("parallelism")) {
        nodeParallelism = (Integer) options.valueOf("parallelism");
    }
    Integer recordsPerPartition = DEFAULT_RECORDS_PER_PARTITION;
    if (options.hasArgument("records-per-partition")) {
        recordsPerPartition = (Integer) options.valueOf("records-per-partition");
    }
    Integer keysPerSecondLimit = DEFAULT_KEYS_PER_SECOND_LIMIT;
    if (options.hasArgument("keys-per-second-limit")) {
        keysPerSecondLimit = (Integer) options.valueOf("keys-per-second-limit");
    }
    System.err.println("throttle: " + keysPerSecondLimit);
    Integer progressPeriodOps = DEFAULT_PROGRESS_PERIOD_OPS;
    if (options.hasArgument("progress-period-ops")) {
        progressPeriodOps = (Integer) options.valueOf("progress-period-ops");
    }
    KeySamplerCLI sampler = new KeySamplerCLI(url, outDir, storeNames, partitionIds, nodeParallelism, recordsPerPartition, keysPerSecondLimit, progressPeriodOps);
    try {
        if (!sampler.sampleStores()) {
            logger.error("Some stores were not successfully sampled.");
        }
    } finally {
        sampler.stop();
    }
}

Example 55 with OptionSet

use of joptsimple.OptionSet in project voldemort by voldemort.

the class KeyVersionFetcherCLI method main.

// In the future, this tool could be expanded with the following options:
// - fetch value in addition to version
// - choose between printing human readable data (.toString()) or computer
// readable data (ByteUtils.toHexString(byte[])).
public static void main(String[] args) throws Exception {
    OptionParser parser = null;
    OptionSet options = null;
    try {
        parser = getParser();
        options = parser.parse(args);
    } catch (OptionException oe) {
        parser.printHelpOn(System.out);
        printUsageAndDie("Exception when parsing arguments : " + oe.getMessage());
        return;
    }
    /* validate options */
    if (options.has("help")) {
        parser.printHelpOn(System.out);
        printUsage();
        return;
    }
    if (!options.hasArgument("url") || !options.hasArgument("in-dir") || !options.hasArgument("out-dir")) {
        parser.printHelpOn(System.out);
        printUsageAndDie("Missing a required argument.");
        return;
    }
    String url = (String) options.valueOf("url");
    String inDir = (String) options.valueOf("in-dir");
    Utils.mkdirs(new File(inDir));
    String outDir = (String) options.valueOf("out-dir");
    Utils.mkdirs(new File(outDir));
    List<String> storeNames = null;
    if (options.hasArgument("store-names")) {
        @SuppressWarnings("unchecked") List<String> list = (List<String>) options.valuesOf("store-names");
        storeNames = list;
    }
    Integer keyParallelism = DEFAULT_KEY_PARALLELISM;
    if (options.hasArgument("parallelism")) {
        keyParallelism = (Integer) options.valueOf("parallelism");
    }
    Integer progressPeriodOps = DEFAULT_PROGRESS_PERIOD_OPS;
    if (options.hasArgument("progress-period-ops")) {
        progressPeriodOps = (Integer) options.valueOf("progress-period-ops");
    }
    Integer outputBatchSize = DEFAULT_OUTPUT_BATCH_SIZE;
    if (options.hasArgument("output-batch-size")) {
        outputBatchSize = (Integer) options.valueOf("output-batch-size");
    }
    boolean details = options.has("details");
    try {
        KeyVersionFetcherCLI sampler = new KeyVersionFetcherCLI(url, inDir, outDir, storeNames, keyParallelism, progressPeriodOps, outputBatchSize, details);
        try {
            if (!sampler.sampleStores()) {
                logger.error("Key-versions were not successfully sampled from some stores.");
            }
        } finally {
            sampler.stop();
        }
    } catch (Exception e) {
        logger.error("Exception during key-version sampling: ", e);
    }
}