Examples with CommandLineParser org.apache.flink.runtime.entrypoint.parser.CommandLineParser used on opensource projects

Example 1 with CommandLineParser

use of org.apache.flink.runtime.entrypoint.parser.CommandLineParser in project flink by apache.

the class ConfigurationParserUtils method loadCommonConfiguration.

/**
 * Generate configuration from only the config file and dynamic properties.
 *
 * @param args the commandline arguments
 * @param cmdLineSyntax the syntax for this application
 * @return generated configuration
 * @throws FlinkParseException if the configuration cannot be generated
 */
public static Configuration loadCommonConfiguration(String[] args, String cmdLineSyntax) throws FlinkParseException {
    final CommandLineParser<ClusterConfiguration> commandLineParser = new CommandLineParser<>(new ClusterConfigurationParserFactory());
    final ClusterConfiguration clusterConfiguration;
    try {
        clusterConfiguration = commandLineParser.parse(args);
    } catch (FlinkParseException e) {
        LOG.error("Could not parse the command line options.", e);
        commandLineParser.printHelp(cmdLineSyntax);
        throw e;
    }
    final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(clusterConfiguration.getDynamicProperties());
    return GlobalConfiguration.loadConfiguration(clusterConfiguration.getConfigDir(), dynamicProperties);
}

Example 2 with CommandLineParser

use of org.apache.flink.runtime.entrypoint.parser.CommandLineParser in project flink by apache.

the class PythonDriver method main.

public static void main(String[] args) throws Throwable {
    // e.g. pym a.b [user args]
    if (args.length < 2) {
        LOG.error("Required at least two arguments, only python file or python module is available.");
        System.exit(1);
    }
    // parse args
    final CommandLineParser<PythonDriverOptions> commandLineParser = new CommandLineParser<>(new PythonDriverOptionsParserFactory());
    PythonDriverOptions pythonDriverOptions = null;
    try {
        pythonDriverOptions = commandLineParser.parse(args);
    } catch (Exception e) {
        LOG.error("Could not parse command line arguments {}.", args, e);
        commandLineParser.printHelp(PythonDriver.class.getSimpleName());
        System.exit(1);
    }
    // Get configuration from ContextEnvironment/OptimizerPlanEnvironment. As the configurations
    // of
    // streaming and batch environments are always set at the same time, for streaming jobs we
    // can
    // also get its configuration from batch environments.
    Configuration config = ExecutionEnvironment.getExecutionEnvironment().getConfiguration();
    // start gateway server
    GatewayServer gatewayServer = PythonEnvUtils.startGatewayServer();
    PythonEnvUtils.setGatewayServer(gatewayServer);
    PythonEnvUtils.PythonProcessShutdownHook shutdownHook = null;
    // commands which will be exec in python progress.
    final List<String> commands = constructPythonCommands(pythonDriverOptions);
    try {
        // prepare the exec environment of python progress.
        String tmpDir = System.getProperty("java.io.tmpdir") + File.separator + "pyflink" + File.separator + UUID.randomUUID();
        // start the python process.
        Process pythonProcess = PythonEnvUtils.launchPy4jPythonClient(gatewayServer, config, commands, pythonDriverOptions.getEntryPointScript().orElse(null), tmpDir, true);
        shutdownHook = new PythonEnvUtils.PythonProcessShutdownHook(pythonProcess, gatewayServer, tmpDir);
        Runtime.getRuntime().addShutdownHook(shutdownHook);
        BufferedReader in = new BufferedReader(new InputStreamReader(pythonProcess.getInputStream(), StandardCharsets.UTF_8));
        LOG.info("--------------------------- Python Process Started --------------------------");
        // print the python process output to stdout and log file
        final StringBuilder sb = new StringBuilder();
        try {
            while (true) {
                String line = in.readLine();
                if (line == null) {
                    break;
                } else {
                    System.out.println(line);
                    sb.append(line);
                    sb.append("\n");
                }
            }
        } finally {
            LOG.info(sb.toString());
        }
        int exitCode = pythonProcess.waitFor();
        LOG.info("--------------------------- Python Process Exited ---------------------------");
        if (exitCode != 0) {
            throw new RuntimeException("Python process exits with code: " + exitCode);
        }
    } catch (Throwable e) {
        LOG.error("Run python process failed", e);
        if (PythonEnvUtils.capturedJavaException != null) {
            throw PythonEnvUtils.capturedJavaException;
        } else {
            // there is no harm to throw ProgramAbortException even if it is not the case.
            throw new ProgramAbortException(e);
        }
    } finally {
        PythonEnvUtils.setGatewayServer(null);
        if (shutdownHook != null && Runtime.getRuntime().removeShutdownHook(shutdownHook)) {
            shutdownHook.run();
        }
    }
}