Examples with GenericOptionsParser org.apache.hadoop.util.GenericOptionsParser used on opensource projects

Example 81 with GenericOptionsParser

use of org.apache.hadoop.util.GenericOptionsParser in project metron by apache.

the class MaxmindDbEnrichmentLoaderTest method testCommandLineShortOpts.

@Test
public void testCommandLineShortOpts() throws Exception {
    String[] argv = { "-g testGeoUrl", "-a testAsnUrl", "-r /test/remoteDirGeo", "-ra", "/test/remoteDirAsn", "-t /test/tmpDir", "-z test:2181" };
    String[] otherArgs = new GenericOptionsParser(argv).getRemainingArgs();
    CommandLine cli = MaxmindDbEnrichmentLoader.GeoEnrichmentOptions.parse(new PosixParser(), otherArgs);
    assertEquals("testGeoUrl", MaxmindDbEnrichmentLoader.GeoEnrichmentOptions.GEO_URL.get(cli).trim());
    assertEquals("testAsnUrl", MaxmindDbEnrichmentLoader.GeoEnrichmentOptions.ASN_URL.get(cli).trim());
    assertEquals("/test/remoteDirGeo", MaxmindDbEnrichmentLoader.GeoEnrichmentOptions.REMOTE_GEO_DIR.get(cli).trim());
    assertEquals("/test/remoteDirAsn", MaxmindDbEnrichmentLoader.GeoEnrichmentOptions.REMOTE_ASN_DIR.get(cli).trim());
    assertEquals("/test/tmpDir", MaxmindDbEnrichmentLoader.GeoEnrichmentOptions.TMP_DIR.get(cli).trim());
    assertEquals("test:2181", MaxmindDbEnrichmentLoader.GeoEnrichmentOptions.ZK_QUORUM.get(cli).trim());
}

Example 82 with GenericOptionsParser

use of org.apache.hadoop.util.GenericOptionsParser in project metron by apache.

the class SimpleFlatFileSummarizerTest method testArgs.

@Test
public void testArgs() throws Exception {
    String[] argv = { "-e extractor.json", "-o out.ser", "-l log4j", "-i input.csv", "-p 2", "-b 128", "-q" };
    Configuration config = new Configuration();
    String[] otherArgs = new GenericOptionsParser(config, argv).getRemainingArgs();
    CommandLine cli = SummarizeOptions.parse(new PosixParser(), otherArgs);
    assertEquals("extractor.json", SummarizeOptions.EXTRACTOR_CONFIG.get(cli).trim());
    assertEquals("input.csv", SummarizeOptions.INPUT.get(cli).trim());
    assertEquals("log4j", SummarizeOptions.LOG4J_PROPERTIES.get(cli).trim());
    assertEquals("2", SummarizeOptions.NUM_THREADS.get(cli).trim());
    assertEquals("128", SummarizeOptions.BATCH_SIZE.get(cli).trim());
}

Example 83 with GenericOptionsParser

use of org.apache.hadoop.util.GenericOptionsParser in project metron by apache.

the class SimpleFlatFileSummarizer method main.

public static void main(String... argv) throws Exception {
    Configuration hadoopConfig = HBaseConfiguration.create();
    String[] otherArgs = new GenericOptionsParser(hadoopConfig, argv).getRemainingArgs();
    main(hadoopConfig, otherArgs);
}

Example 84 with GenericOptionsParser

use of org.apache.hadoop.util.GenericOptionsParser in project metron by apache.

the class SimpleEnrichmentFlatFileLoader method main.

public static void main(String... argv) throws Exception {
    Configuration hadoopConfig = HBaseConfiguration.create();
    String[] otherArgs = new GenericOptionsParser(hadoopConfig, argv).getRemainingArgs();
    main(hadoopConfig, otherArgs);
}

Example 85 with GenericOptionsParser

use of org.apache.hadoop.util.GenericOptionsParser in project metron by apache.

the class PcapCli method run.

public int run(String[] args) {
    if (args.length < 1) {
        printBasicHelp();
        return -1;
    }
    String jobType = args[0];
    String[] commandArgs = Arrays.copyOfRange(args, 1, args.length);
    Configuration hadoopConf = new Configuration();
    String[] otherArgs = null;
    try {
        otherArgs = new GenericOptionsParser(hadoopConf, commandArgs).getRemainingArgs();
    } catch (IOException e) {
        LOGGER.error("Failed to configure hadoop with provided options: {}", e.getMessage(), e);
        return -1;
    }
    PcapConfig commonConfig = null;
    Pageable<Path> results;
    // write to local FS in the executing directory
    String execDir = System.getProperty("user.dir");
    if ("fixed".equals(jobType)) {
        FixedCliParser fixedParser = new FixedCliParser(prefixStrategy);
        FixedPcapConfig config = null;
        try {
            config = fixedParser.parse(otherArgs);
            commonConfig = config;
            PcapOptions.FINAL_OUTPUT_PATH.put(commonConfig, new Path(execDir));
        } catch (ParseException | java.text.ParseException e) {
            System.err.println(e.getMessage());
            System.err.flush();
            fixedParser.printHelp();
            return -1;
        }
        if (config.showHelp()) {
            fixedParser.printHelp();
            return 0;
        }
        PcapOptions.FILTER_IMPL.put(commonConfig, new FixedPcapFilter.Configurator());
        config.getYarnQueue().ifPresent(s -> hadoopConf.set(MRJobConfig.QUEUE_NAME, s));
        PcapOptions.HADOOP_CONF.put(commonConfig, hadoopConf);
        try {
            PcapOptions.FILESYSTEM.put(commonConfig, FileSystem.get(hadoopConf));
            results = jobRunner.submit(PcapFinalizerStrategies.CLI, commonConfig).get();
        } catch (IOException | InterruptedException | JobException e) {
            LOGGER.error("Failed to execute fixed filter job: {}", e.getMessage(), e);
            return -1;
        }
    } else if ("query".equals(jobType)) {
        QueryCliParser queryParser = new QueryCliParser(prefixStrategy);
        QueryPcapConfig config = null;
        try {
            config = queryParser.parse(otherArgs);
            commonConfig = config;
            PcapOptions.FINAL_OUTPUT_PATH.put(commonConfig, new Path(execDir));
        } catch (ParseException | java.text.ParseException e) {
            System.err.println(e.getMessage());
            queryParser.printHelp();
            return -1;
        }
        if (config.showHelp()) {
            queryParser.printHelp();
            return 0;
        }
        PcapOptions.FILTER_IMPL.put(commonConfig, new FixedPcapFilter.Configurator());
        config.getYarnQueue().ifPresent(s -> hadoopConf.set(MRJobConfig.QUEUE_NAME, s));
        PcapOptions.HADOOP_CONF.put(commonConfig, hadoopConf);
        try {
            PcapOptions.FILESYSTEM.put(commonConfig, FileSystem.get(hadoopConf));
            results = jobRunner.submit(PcapFinalizerStrategies.CLI, commonConfig).get();
        } catch (IOException | InterruptedException | JobException e) {
            LOGGER.error("Failed to execute fixed filter job: {}", e.getMessage(), e);
            return -1;
        }
    } else {
        printBasicHelp();
        return -1;
    }
    return 0;
}