Examples with PosixParser org.apache.commons.cli.PosixParser used on opensource projects

Example 41 with PosixParser

use of org.apache.commons.cli.PosixParser in project metron by apache.

the class LoadGenerator method main.

public static void main(String[] args) throws Exception {
    CommandLine cli = LoadOptions.parse(new PosixParser(), args);
    EnumMap<LoadOptions, Optional<Object>> evaluatedArgs = LoadOptions.createConfig(cli);
    Map<String, Object> kafkaConfig = new HashMap<>();
    kafkaConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
    kafkaConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
    kafkaConfig.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    kafkaConfig.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    if (LoadOptions.ZK.has(cli)) {
        String zkQuorum = (String) evaluatedArgs.get(LoadOptions.ZK).get();
        kafkaConfig.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, Joiner.on(",").join(KafkaUtils.INSTANCE.getBrokersFromZookeeper(zkQuorum)));
    }
    String groupId = evaluatedArgs.get(LoadOptions.CONSUMER_GROUP).get().toString();
    System.out.println("Consumer Group: " + groupId);
    kafkaConfig.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
    if (LoadOptions.KAFKA_CONFIG.has(cli)) {
        kafkaConfig.putAll((Map<String, Object>) evaluatedArgs.get(LoadOptions.KAFKA_CONFIG).get());
    }
    kafkaProducer = ThreadLocal.withInitial(() -> new KafkaProducer<>(kafkaConfig));
    int numThreads = (int) evaluatedArgs.get(LoadOptions.NUM_THREADS).get();
    System.out.println("Thread pool size: " + numThreads);
    pool = Executors.newFixedThreadPool(numThreads);
    Optional<Object> eps = evaluatedArgs.get(LoadOptions.EPS);
    Optional<Object> outputTopic = evaluatedArgs.get(LoadOptions.OUTPUT_TOPIC);
    Optional<Object> monitorTopic = evaluatedArgs.get(LoadOptions.MONITOR_TOPIC);
    long sendDelta = (long) evaluatedArgs.get(LoadOptions.SEND_DELTA).get();
    long monitorDelta = (long) evaluatedArgs.get(LoadOptions.MONITOR_DELTA).get();
    if ((eps.isPresent() && outputTopic.isPresent()) || monitorTopic.isPresent()) {
        Timer timer = new Timer(false);
        long startTimeMs = System.currentTimeMillis();
        if (outputTopic.isPresent() && eps.isPresent()) {
            List<String> templates = (List<String>) evaluatedArgs.get(LoadOptions.TEMPLATE).get();
            if (templates.isEmpty()) {
                System.out.println("Empty templates, so nothing to do.");
                return;
            }
            Optional<Object> biases = evaluatedArgs.get(LoadOptions.BIASED_SAMPLE);
            Sampler sampler = new UnbiasedSampler();
            if (biases.isPresent()) {
                sampler = new BiasedSampler((List<Map.Entry<Integer, Integer>>) biases.get(), templates.size());
            }
            MessageGenerator generator = new MessageGenerator(templates, sampler);
            Long targetLoad = (Long) eps.get();
            int periodsPerSecond = (int) (1000 / sendDelta);
            long messagesPerPeriod = targetLoad / periodsPerSecond;
            String outputTopicStr = (String) outputTopic.get();
            System.out.println("Generating data to " + outputTopicStr + " at " + targetLoad + " events per second");
            System.out.println("Sending " + messagesPerPeriod + " messages to " + outputTopicStr + " every " + sendDelta + "ms");
            timer.scheduleAtFixedRate(new SendToKafka(outputTopicStr, messagesPerPeriod, numThreads, generator, pool, numSent, kafkaProducer), 0, sendDelta);
        }
        List<AbstractMonitor> monitors = new ArrayList<>();
        if (outputTopic.isPresent() && monitorTopic.isPresent()) {
            System.out.println("Monitoring " + monitorTopic.get() + " every " + monitorDelta + " ms");
            monitors.add(new EPSGeneratedMonitor(outputTopic, numSent));
            monitors.add(new EPSThroughputWrittenMonitor(monitorTopic, kafkaConfig));
        } else if (outputTopic.isPresent() && !monitorTopic.isPresent()) {
            System.out.println("Monitoring " + outputTopic.get() + " every " + monitorDelta + " ms");
            monitors.add(new EPSGeneratedMonitor(outputTopic, numSent));
            monitors.add(new EPSThroughputWrittenMonitor(outputTopic, kafkaConfig));
        } else if (!outputTopic.isPresent() && monitorTopic.isPresent()) {
            System.out.println("Monitoring " + monitorTopic.get() + " every " + monitorDelta + " ms");
            monitors.add(new EPSThroughputWrittenMonitor(monitorTopic, kafkaConfig));
        } else if (!outputTopic.isPresent() && !monitorTopic.isPresent()) {
            System.out.println("You have not specified an output topic or a monitoring topic, so I have nothing to do here.");
        }
        int lookback = (int) evaluatedArgs.get(LoadOptions.SUMMARY_LOOKBACK).get();
        if (lookback > 0) {
            System.out.println("Summarizing over the last " + lookback + " monitoring periods (" + lookback * monitorDelta + "ms)");
        } else {
            System.out.println("Turning off summarization.");
        }
        final CSVWriter csvWriter = new CSVWriter((File) evaluatedArgs.get(LoadOptions.CSV).orElse(null));
        Writer writer = new Writer(monitors, lookback, new ArrayList<Consumer<Writable>>() {

            {
                add(new ConsoleWriter());
                add(csvWriter);
            }
        });
        timer.scheduleAtFixedRate(new MonitorTask(writer), 0, monitorDelta);
        Optional<Object> timeLimit = evaluatedArgs.get(LoadOptions.TIME_LIMIT);
        if (timeLimit.isPresent()) {
            System.out.println("Ending in " + timeLimit.get() + " ms.");
            timer.schedule(new TimerTask() {

                @Override
                public void run() {
                    timer.cancel();
                    long durationS = (System.currentTimeMillis() - startTimeMs) / 1000;
                    System.out.println("\nGenerated " + numSent.get() + " in " + durationS + " seconds.");
                    csvWriter.close();
                    System.exit(0);
                }
            }, (Long) timeLimit.get());
        }
    }
}

Example 42 with PosixParser

use of org.apache.commons.cli.PosixParser in project metron by apache.

the class LoadOptionsTest method testCsvMissing.

@Test
public void testCsvMissing() throws Exception {
    CommandLine cli = LoadOptions.parse(new PosixParser(), new String[] {});
    EnumMap<LoadOptions, Optional<Object>> results = LoadOptions.createConfig(cli);
    Assert.assertFalse(results.get(LoadOptions.CSV).isPresent());
}

Example 43 with PosixParser

use of org.apache.commons.cli.PosixParser in project metron by apache.

the class LoadOptionsTest method testCsvPresent.

@Test
public void testCsvPresent() throws Exception {
    CommandLine cli = LoadOptions.parse(new PosixParser(), new String[] { "-c", "/tmp/blah" });
    EnumMap<LoadOptions, Optional<Object>> results = LoadOptions.createConfig(cli);
    Assert.assertEquals(new File("/tmp/blah"), results.get(LoadOptions.CSV).get());
}

Example 44 with PosixParser

use of org.apache.commons.cli.PosixParser in project metron by apache.

the class LoadOptionsTest method testTemplatePresent.

@Test
public void testTemplatePresent() throws Exception {
    File templateFile = new File("target/template");
    String template = "test template1";
    try (BufferedWriter w = new BufferedWriter(new FileWriter(templateFile))) {
        IOUtils.write(template, w);
    }
    templateFile.deleteOnExit();
    CommandLine cli = LoadOptions.parse(new PosixParser(), new String[] { "-t", templateFile.getPath() });
    EnumMap<LoadOptions, Optional<Object>> results = LoadOptions.createConfig(cli);
    List<String> templates = (List<String>) results.get(LoadOptions.TEMPLATE).get();
    Assert.assertEquals(1, templates.size());
    Assert.assertEquals(template, templates.get(0));
}

Example 45 with PosixParser

use of org.apache.commons.cli.PosixParser in project metron by apache.

the class HDFSDataPruner method main.

public static void main(String... argv) throws IOException, java.text.ParseException, ClassNotFoundException, InterruptedException {
    /**
     * Example
     * start=$(date -d '30 days ago' +%m/%d/%Y)
     * yarn jar Metron-DataLoads-0.1BETA.jar org.apache.metron.dataloads.bulk.HDFSDataPruner -f hdfs://ec2-52-36-25-217.us-west-2.compute.amazonaws.com:8020 -g '/apps/metron/enrichment/indexed/bro_doc/*enrichment-*' -s $(date -d '30 days ago' +%m/%d/%Y) -n 1;
     * echo ${start}
     */
    Options options = new Options();
    Options help = new Options();
    {
        Option o = new Option("h", "help", false, "This screen");
        o.setRequired(false);
        help.addOption(o);
    }
    {
        Option o = new Option("s", "start-date", true, "Starting Date (MM/DD/YYYY)");
        o.setArgName("START_DATE");
        o.setRequired(true);
        options.addOption(o);
    }
    {
        Option o = new Option("f", "filesystem", true, "Filesystem uri - e.g. hdfs://host:8020 or file:///");
        o.setArgName("FILESYSTEM");
        o.setRequired(true);
        options.addOption(o);
    }
    {
        Option o = new Option("n", "numdays", true, "Number of days back to purge");
        o.setArgName("NUMDAYS");
        o.setRequired(true);
        options.addOption(o);
    }
    {
        Option o = new Option("g", "glob-string", true, "Glob filemask for files to delete - e.g. /apps/metron/enrichment/bro_doc/file-*");
        o.setArgName("GLOBSTRING");
        o.setRequired(true);
        options.addOption(o);
    }
    try {
        CommandLineParser parser = new PosixParser();
        CommandLine cmd = null;
        try {
            cmd = parser.parse(help, argv, true);
            if (cmd.getOptions().length > 0) {
                final HelpFormatter usageFormatter = new HelpFormatter();
                usageFormatter.printHelp("HDFSDataPruner", null, options, null, true);
                System.exit(0);
            }
            cmd = parser.parse(options, argv);
        } catch (ParseException pe) {
            final HelpFormatter usageFormatter = new HelpFormatter();
            usageFormatter.printHelp("HDFSDataPruner", null, options, null, true);
            System.exit(-1);
        }
        String start = cmd.getOptionValue("s");
        Date startDate = new SimpleDateFormat("MM/dd/yyyy").parse(start);
        String fileSystemUri = cmd.getOptionValue("f");
        Integer numDays = Integer.parseInt(cmd.getOptionValue("n"));
        String globString = cmd.getOptionValue("g");
        LOG.debug("Running prune with args: {} {} {} {}", startDate, numDays, fileSystemUri, globString);
        DataPruner pruner = new HDFSDataPruner(startDate, numDays, fileSystemUri, globString);
        LOG.info("Pruned {} files from {}{}", pruner.prune(), fileSystemUri, globString);
    } catch (Exception e) {
        e.printStackTrace();
        System.exit(-1);
    }
}