Examples with GenericOptionsParser org.apache.hadoop.util.GenericOptionsParser used on opensource projects

Example 11 with GenericOptionsParser

use of org.apache.hadoop.util.GenericOptionsParser in project incubator-systemml by apache.

the class DMLAppMaster method runApplicationMaster.

public void runApplicationMaster(String[] args) throws YarnException, IOException {
    _conf = new YarnConfiguration();
    // obtain application ID
    String containerIdString = System.getenv(Environment.CONTAINER_ID.name());
    ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
    _appId = containerId.getApplicationAttemptId().getApplicationId();
    LOG.info("SystemML appplication master (applicationID: " + _appId + ")");
    // initialize clients to ResourceManager
    AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(_conf);
    rmClient.start();
    // register with ResourceManager
    // host, port for rm communication
    rmClient.registerApplicationMaster("", 0, "");
    LOG.debug("Registered the SystemML application master with resource manager");
    // start status reporter to ResourceManager
    DMLAppMasterStatusReporter reporter = new DMLAppMasterStatusReporter(rmClient, 10000);
    reporter.start();
    LOG.debug("Started status reporter (heartbeat to resource manager)");
    // set DMLscript app master context
    DMLScript.setActiveAM();
    // parse input arguments
    String[] otherArgs = new GenericOptionsParser(_conf, args).getRemainingArgs();
    // run SystemML CP
    FinalApplicationStatus status = null;
    try {
        // core dml script execution (equivalent to non-AM runtime)
        boolean success = DMLScript.executeScript(_conf, otherArgs);
        if (success)
            status = FinalApplicationStatus.SUCCEEDED;
        else
            status = FinalApplicationStatus.FAILED;
    } catch (DMLScriptException ex) {
        LOG.error(DMLYarnClient.APPMASTER_NAME + ": Failed to executed DML script due to stop call:\n\t" + ex.getMessage());
        status = FinalApplicationStatus.FAILED;
        writeMessageToHDFSWorkingDir(ex.getMessage());
    } catch (Exception ex) {
        LOG.error(DMLYarnClient.APPMASTER_NAME + ": Failed to executed DML script.", ex);
        status = FinalApplicationStatus.FAILED;
    } finally {
        // stop periodic status reports
        reporter.stopStatusReporter();
        LOG.debug("Stopped status reporter");
        // unregister resource manager client
        rmClient.unregisterApplicationMaster(status, "", "");
        LOG.debug("Unregistered the SystemML application master");
    }
}

Example 12 with GenericOptionsParser

use of org.apache.hadoop.util.GenericOptionsParser in project metron by apache.

the class GeoEnrichmentLoader method main.

public static void main(String... argv) throws IOException {
    String[] otherArgs = new GenericOptionsParser(argv).getRemainingArgs();
    CommandLine cli = GeoEnrichmentOptions.parse(new PosixParser(), otherArgs);
    GeoEnrichmentLoader loader = new GeoEnrichmentLoader();
    loader.loadGeoIpDatabase(cli);
}

Example 13 with GenericOptionsParser

use of org.apache.hadoop.util.GenericOptionsParser in project metron by apache.

the class GeoEnrichmentLoaderTest method testCommandLineShortOpts.

@Test
public void testCommandLineShortOpts() throws Exception {
    String[] argv = { "-g testGeoUrl", "-r /test/remoteDir", "-t /test/tmpDir", "-z test:2181" };
    String[] otherArgs = new GenericOptionsParser(argv).getRemainingArgs();
    CommandLine cli = GeoEnrichmentLoader.GeoEnrichmentOptions.parse(new PosixParser(), otherArgs);
    Assert.assertEquals("testGeoUrl", GeoEnrichmentLoader.GeoEnrichmentOptions.GEO_URL.get(cli).trim());
    Assert.assertEquals("/test/remoteDir", GeoEnrichmentLoader.GeoEnrichmentOptions.REMOTE_DIR.get(cli).trim());
    Assert.assertEquals("/test/tmpDir", GeoEnrichmentLoader.GeoEnrichmentOptions.TMP_DIR.get(cli).trim());
    Assert.assertEquals("test:2181", GeoEnrichmentLoader.GeoEnrichmentOptions.ZK_QUORUM.get(cli).trim());
}

Example 14 with GenericOptionsParser

use of org.apache.hadoop.util.GenericOptionsParser in project nutch by apache.

the class Content method main.

public static void main(String[] argv) throws Exception {
    String usage = "Content (-local | -dfs <namenode:port>) recno segment";
    if (argv.length < 3) {
        System.out.println("usage:" + usage);
        return;
    }
    Options opts = new Options();
    Configuration conf = NutchConfiguration.create();
    GenericOptionsParser parser = new GenericOptionsParser(conf, opts, argv);
    String[] remainingArgs = parser.getRemainingArgs();
    try (FileSystem fs = FileSystem.get(conf)) {
        int recno = Integer.parseInt(remainingArgs[0]);
        String segment = remainingArgs[1];
        Path file = new Path(segment, DIR_NAME);
        System.out.println("Reading from file: " + file);
        ArrayFile.Reader contents = new ArrayFile.Reader(fs, file.toString(), conf);
        Content content = new Content();
        contents.get(recno, content);
        System.out.println("Retrieved " + recno + " from file " + file);
        System.out.println(content);
        contents.close();
    }
}

Example 15 with GenericOptionsParser

use of org.apache.hadoop.util.GenericOptionsParser in project incubator-gobblin by apache.

the class CliOptions method parseArgs.

/**
 * Parse command line arguments and return a {@link java.util.Properties} object for the Gobblin job found.
 * @param caller Class of the calling main method. Used for error logs.
 * @param args Command line arguments.
 * @param conf Hadoop configuration object
 * @return Instance of {@link Properties} for the Gobblin job to run.
 * @throws IOException
 */
public static Properties parseArgs(Class<?> caller, String[] args, Configuration conf) throws IOException {
    try {
        // Parse command-line options
        if (conf != null) {
            args = new GenericOptionsParser(conf, args).getCommandLine().getArgs();
        }
        CommandLine cmd = new DefaultParser().parse(options(), args);
        if (cmd.hasOption(HELP_OPTION.getOpt())) {
            printUsage(caller);
            System.exit(0);
        }
        String jobConfigLocation = JOB_CONFIG_OPTION.getLongOpt();
        if (!cmd.hasOption(jobConfigLocation)) {
            printUsage(caller);
            System.exit(1);
        }
        // Load job configuration properties
        Properties jobConfig;
        if (conf == null) {
            jobConfig = JobConfigurationUtils.fileToProperties(cmd.getOptionValue(jobConfigLocation));
        } else {
            jobConfig = JobConfigurationUtils.fileToProperties(cmd.getOptionValue(jobConfigLocation), conf);
            JobConfigurationUtils.putConfigurationIntoProperties(conf, jobConfig);
        }
        return jobConfig;
    } catch (ParseException | ConfigurationException e) {
        throw new IOException(e);
    }
}