Examples with AbstractYarnClusterDescriptor org.apache.flink.yarn.AbstractYarnClusterDescriptor used on opensource projects

Example 1 with AbstractYarnClusterDescriptor

use of org.apache.flink.yarn.AbstractYarnClusterDescriptor in project flink by apache.

the class FlinkYarnSessionCli method run.

public int run(String[] args) {
    //
    //	Command Line Options
    //
    Options options = new Options();
    addGeneralOptions(options);
    addRunOptions(options);
    CommandLineParser parser = new PosixParser();
    CommandLine cmd;
    try {
        cmd = parser.parse(options, args);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        printUsage();
        return 1;
    }
    // Query cluster for metrics
    if (cmd.hasOption(QUERY.getOpt())) {
        AbstractYarnClusterDescriptor yarnDescriptor = getClusterDescriptor();
        String description;
        try {
            description = yarnDescriptor.getClusterDescription();
        } catch (Exception e) {
            System.err.println("Error while querying the YARN cluster for available resources: " + e.getMessage());
            e.printStackTrace(System.err);
            return 1;
        }
        System.out.println(description);
        return 0;
    } else if (cmd.hasOption(APPLICATION_ID.getOpt())) {
        AbstractYarnClusterDescriptor yarnDescriptor = getClusterDescriptor();
        //configure ZK namespace depending on the value passed
        String zkNamespace = cmd.hasOption(ZOOKEEPER_NAMESPACE.getOpt()) ? cmd.getOptionValue(ZOOKEEPER_NAMESPACE.getOpt()) : yarnDescriptor.getFlinkConfiguration().getString(HA_ZOOKEEPER_NAMESPACE_KEY, cmd.getOptionValue(APPLICATION_ID.getOpt()));
        LOG.info("Going to use the ZK namespace: {}", zkNamespace);
        yarnDescriptor.getFlinkConfiguration().setString(HA_ZOOKEEPER_NAMESPACE_KEY, zkNamespace);
        try {
            yarnCluster = yarnDescriptor.retrieve(cmd.getOptionValue(APPLICATION_ID.getOpt()));
        } catch (Exception e) {
            throw new RuntimeException("Could not retrieve existing Yarn application", e);
        }
        if (detachedMode) {
            LOG.info("The Flink YARN client has been started in detached mode. In order to stop " + "Flink on YARN, use the following command or a YARN web interface to stop it:\n" + "yarn application -kill " + APPLICATION_ID.getOpt());
            yarnCluster.disconnect();
        } else {
            runInteractiveCli(yarnCluster, true);
        }
    } else {
        AbstractYarnClusterDescriptor yarnDescriptor;
        try {
            yarnDescriptor = createDescriptor(null, cmd);
        } catch (Exception e) {
            System.err.println("Error while starting the YARN Client: " + e.getMessage());
            e.printStackTrace(System.err);
            return 1;
        }
        try {
            yarnCluster = yarnDescriptor.deploy();
        } catch (Exception e) {
            System.err.println("Error while deploying YARN cluster: " + e.getMessage());
            e.printStackTrace(System.err);
            return 1;
        }
        //------------------ ClusterClient deployed, handle connection details
        String jobManagerAddress = yarnCluster.getJobManagerAddress().getAddress().getHostName() + ":" + yarnCluster.getJobManagerAddress().getPort();
        System.out.println("Flink JobManager is now running on " + jobManagerAddress);
        System.out.println("JobManager Web Interface: " + yarnCluster.getWebInterfaceURL());
        // file that we write into the conf/ dir containing the jobManager address and the dop.
        File yarnPropertiesFile = getYarnPropertiesLocation(yarnCluster.getFlinkConfiguration());
        Properties yarnProps = new Properties();
        yarnProps.setProperty(YARN_APPLICATION_ID_KEY, yarnCluster.getApplicationId().toString());
        if (yarnDescriptor.getTaskManagerSlots() != -1) {
            String parallelism = Integer.toString(yarnDescriptor.getTaskManagerSlots() * yarnDescriptor.getTaskManagerCount());
            yarnProps.setProperty(YARN_PROPERTIES_PARALLELISM, parallelism);
        }
        // add dynamic properties
        if (yarnDescriptor.getDynamicPropertiesEncoded() != null) {
            yarnProps.setProperty(YARN_PROPERTIES_DYNAMIC_PROPERTIES_STRING, yarnDescriptor.getDynamicPropertiesEncoded());
        }
        writeYarnProperties(yarnProps, yarnPropertiesFile);
        if (detachedMode) {
            // print info and quit:
            LOG.info("The Flink YARN client has been started in detached mode. In order to stop " + "Flink on YARN, use the following command or a YARN web interface to stop it:\n" + "yarn application -kill " + yarnCluster.getApplicationId() + System.lineSeparator() + "Please also note that the temporary files of the YARN session in {} will not be removed.", yarnDescriptor.getSessionFilesDir());
            yarnCluster.waitForClusterToBeReady();
            yarnCluster.disconnect();
        } else {
            runInteractiveCli(yarnCluster, acceptInteractiveInput);
        }
    }
    return 0;
}

Example 2 with AbstractYarnClusterDescriptor

use of org.apache.flink.yarn.AbstractYarnClusterDescriptor in project flink by apache.

the class FlinkYarnSessionCli method retrieveCluster.

@Override
public YarnClusterClient retrieveCluster(CommandLine cmdLine, Configuration config) throws UnsupportedOperationException {
    // first check for an application id, then try to load from yarn properties
    String applicationID = cmdLine.hasOption(APPLICATION_ID.getOpt()) ? cmdLine.getOptionValue(APPLICATION_ID.getOpt()) : loadYarnPropertiesFile(cmdLine, config);
    if (null != applicationID) {
        String zkNamespace = cmdLine.hasOption(ZOOKEEPER_NAMESPACE.getOpt()) ? cmdLine.getOptionValue(ZOOKEEPER_NAMESPACE.getOpt()) : config.getString(HighAvailabilityOptions.HA_CLUSTER_ID, applicationID);
        config.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zkNamespace);
        AbstractYarnClusterDescriptor yarnDescriptor = getClusterDescriptor();
        yarnDescriptor.setFlinkConfiguration(config);
        return yarnDescriptor.retrieve(applicationID);
    } else {
        throw new UnsupportedOperationException("Could not resume a Yarn cluster.");
    }
}

Example 3 with AbstractYarnClusterDescriptor

use of org.apache.flink.yarn.AbstractYarnClusterDescriptor in project flink by apache.

the class FlinkYarnSessionCli method createCluster.

@Override
public YarnClusterClient createCluster(String applicationName, CommandLine cmdLine, Configuration config, List<URL> userJarFiles) {
    Preconditions.checkNotNull(userJarFiles, "User jar files should not be null.");
    AbstractYarnClusterDescriptor yarnClusterDescriptor = createDescriptor(applicationName, cmdLine);
    yarnClusterDescriptor.setFlinkConfiguration(config);
    yarnClusterDescriptor.setProvidedUserJarFiles(userJarFiles);
    try {
        return yarnClusterDescriptor.deploy();
    } catch (Exception e) {
        throw new RuntimeException("Error deploying the YARN cluster", e);
    }
}

Example 4 with AbstractYarnClusterDescriptor

use of org.apache.flink.yarn.AbstractYarnClusterDescriptor in project flink by apache.

the class FlinkYarnSessionCli method createDescriptor.

public AbstractYarnClusterDescriptor createDescriptor(String defaultApplicationName, CommandLine cmd) {
    AbstractYarnClusterDescriptor yarnClusterDescriptor = getClusterDescriptor();
    if (!cmd.hasOption(CONTAINER.getOpt())) {
        // number of containers is required option!
        LOG.error("Missing required argument {}", CONTAINER.getOpt());
        printUsage();
        throw new IllegalArgumentException("Missing required argument " + CONTAINER.getOpt());
    }
    yarnClusterDescriptor.setTaskManagerCount(Integer.valueOf(cmd.getOptionValue(CONTAINER.getOpt())));
    // Jar Path
    Path localJarPath;
    if (cmd.hasOption(FLINK_JAR.getOpt())) {
        String userPath = cmd.getOptionValue(FLINK_JAR.getOpt());
        if (!userPath.startsWith("file://")) {
            userPath = "file://" + userPath;
        }
        localJarPath = new Path(userPath);
    } else {
        LOG.info("No path for the flink jar passed. Using the location of " + yarnClusterDescriptor.getClass() + " to locate the jar");
        String encodedJarPath = yarnClusterDescriptor.getClass().getProtectionDomain().getCodeSource().getLocation().getPath();
        try {
            // we have to decode the url encoded parts of the path
            String decodedPath = URLDecoder.decode(encodedJarPath, Charset.defaultCharset().name());
            localJarPath = new Path(new File(decodedPath).toURI());
        } catch (UnsupportedEncodingException e) {
            throw new RuntimeException("Couldn't decode the encoded Flink dist jar path: " + encodedJarPath + " Please supply a path manually via the -" + FLINK_JAR.getOpt() + " option.");
        }
    }
    yarnClusterDescriptor.setLocalJarPath(localJarPath);
    List<File> shipFiles = new ArrayList<>();
    // path to directory to ship
    if (cmd.hasOption(SHIP_PATH.getOpt())) {
        String shipPath = cmd.getOptionValue(SHIP_PATH.getOpt());
        File shipDir = new File(shipPath);
        if (shipDir.isDirectory()) {
            shipFiles.add(shipDir);
        } else {
            LOG.warn("Ship directory is not a directory. Ignoring it.");
        }
    }
    yarnClusterDescriptor.addShipFiles(shipFiles);
    // queue
    if (cmd.hasOption(QUEUE.getOpt())) {
        yarnClusterDescriptor.setQueue(cmd.getOptionValue(QUEUE.getOpt()));
    }
    // JobManager Memory
    if (cmd.hasOption(JM_MEMORY.getOpt())) {
        int jmMemory = Integer.valueOf(cmd.getOptionValue(JM_MEMORY.getOpt()));
        yarnClusterDescriptor.setJobManagerMemory(jmMemory);
    }
    // Task Managers memory
    if (cmd.hasOption(TM_MEMORY.getOpt())) {
        int tmMemory = Integer.valueOf(cmd.getOptionValue(TM_MEMORY.getOpt()));
        yarnClusterDescriptor.setTaskManagerMemory(tmMemory);
    }
    if (cmd.hasOption(SLOTS.getOpt())) {
        int slots = Integer.valueOf(cmd.getOptionValue(SLOTS.getOpt()));
        yarnClusterDescriptor.setTaskManagerSlots(slots);
    }
    String[] dynamicProperties = null;
    if (cmd.hasOption(DYNAMIC_PROPERTIES.getOpt())) {
        dynamicProperties = cmd.getOptionValues(DYNAMIC_PROPERTIES.getOpt());
    }
    String dynamicPropertiesEncoded = StringUtils.join(dynamicProperties, YARN_DYNAMIC_PROPERTIES_SEPARATOR);
    yarnClusterDescriptor.setDynamicPropertiesEncoded(dynamicPropertiesEncoded);
    if (cmd.hasOption(DETACHED.getOpt()) || cmd.hasOption(CliFrontendParser.DETACHED_OPTION.getOpt())) {
        this.detachedMode = true;
        yarnClusterDescriptor.setDetachedMode(true);
    }
    if (cmd.hasOption(NAME.getOpt())) {
        yarnClusterDescriptor.setName(cmd.getOptionValue(NAME.getOpt()));
    } else {
        // set the default application name, if none is specified
        if (defaultApplicationName != null) {
            yarnClusterDescriptor.setName(defaultApplicationName);
        }
    }
    if (cmd.hasOption(ZOOKEEPER_NAMESPACE.getOpt())) {
        String zookeeperNamespace = cmd.getOptionValue(ZOOKEEPER_NAMESPACE.getOpt());
        yarnClusterDescriptor.setZookeeperNamespace(zookeeperNamespace);
    }
    // ----- Convenience -----
    // the number of slots available from YARN:
    int yarnTmSlots = yarnClusterDescriptor.getTaskManagerSlots();
    if (yarnTmSlots == -1) {
        yarnTmSlots = 1;
        yarnClusterDescriptor.setTaskManagerSlots(yarnTmSlots);
    }
    int maxSlots = yarnTmSlots * yarnClusterDescriptor.getTaskManagerCount();
    int userParallelism = Integer.valueOf(cmd.getOptionValue(CliFrontendParser.PARALLELISM_OPTION.getOpt(), "-1"));
    if (userParallelism != -1) {
        int slotsPerTM = (int) Math.ceil((double) userParallelism / yarnClusterDescriptor.getTaskManagerCount());
        String message = "The YARN cluster has " + maxSlots + " slots available, " + "but the user requested a parallelism of " + userParallelism + " on YARN. " + "Each of the " + yarnClusterDescriptor.getTaskManagerCount() + " TaskManagers " + "will get " + slotsPerTM + " slots.";
        logAndSysout(message);
        yarnClusterDescriptor.setTaskManagerSlots(slotsPerTM);
    }
    return yarnClusterDescriptor;
}