Examples with YarnRunner io.hops.hopsworks.common.jobs.yarn.YarnRunner used on opensource projects

Example 1 with YarnRunner

use of io.hops.hopsworks.common.jobs.yarn.YarnRunner in project hopsworks by logicalclocks.

the class FlinkYarnRunnerBuilder method getYarnRunner.

YarnRunner getYarnRunner(Project project, String jobUser, Users hopsworksUser, DistributedFileSystemOps dfsClient, YarnClient yarnClient, AsynchronousJobExecutor services, Settings settings, String kafkaBrokersString, String hopsworksRestEndpoint, ServingConfig servingConfig, ServiceDiscoveryController serviceDiscoveryController) throws IOException, ServiceDiscoveryException {
    String stagingPath = File.separator + "Projects" + File.separator + project.getName() + File.separator + Settings.PROJECT_STAGING_DIR;
    Configuration conf = services.getSettings().getConfiguration();
    // Create the YarnRunner builder for Flink, proceed with setting values
    YarnRunner.Builder builder = new YarnRunner.Builder(Settings.FLINK_AM_MAIN);
    org.apache.flink.configuration.Configuration flinkConf = org.apache.flink.configuration.GlobalConfiguration.loadConfiguration(settings.getFlinkConfDir());
    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    try {
        yarnConf.addResource(new File(settings.getHadoopConfDir() + "/" + Settings.DEFAULT_YARN_CONFFILE_NAME).toURI().toURL());
    } catch (MalformedURLException t) {
        throw new RuntimeException("Error", t);
    }
    Map<String, String> extraJavaOptions = new HashMap<>();
    extraJavaOptions.put(Settings.LOGSTASH_JOB_INFO, project.getName().toLowerCase() + "," + job.getName() + "," + job.getId() + "," + YarnRunner.APPID_PLACEHOLDER);
    Map<String, String> finalJobProps = flinkConfigurationUtil.setFrameworkProperties(project, job.getJobConfig(), settings, jobUser, hopsworksUser, extraJavaOptions, kafkaBrokersString, hopsworksRestEndpoint, servingConfig, serviceDiscoveryController);
    // Parse properties from Spark config file
    Yaml yaml = new Yaml();
    try (InputStream in = new FileInputStream(new File(settings.getFlinkConfFile()))) {
        Map<String, String> flinkConfProps = (Map<String, String>) yaml.load(in);
        for (String key : flinkConfProps.keySet()) {
            finalJobProps.putIfAbsent(key, String.valueOf(flinkConfProps.get(key)));
        }
    }
    // Create dynamicProperties from finalJobProps
    YarnClusterDescriptor cluster = new YarnClusterDescriptor(flinkConf, yarnConf, settings.getFlinkConfDir(), yarnClient, true);
    ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder().setMasterMemoryMB(flinkJobConfiguration.getJobManagerMemory()).setTaskManagerMemoryMB(flinkJobConfiguration.getTaskManagerMemory()).setSlotsPerTaskManager(flinkJobConfiguration.getNumberOfTaskSlots()).setNumberTaskManagers(flinkJobConfiguration.getNumberOfTaskManagers()).createClusterSpecification();
    cluster.setLocalJarPath(new Path(settings.getLocalFlinkJarPath()));
    cluster.setDocker(ProjectUtils.getFullDockerImageName(project, settings, serviceDiscoveryController, true), settings.getDockerMounts());
    builder.setYarnClient(yarnClient);
    builder.setDfsClient(dfsClient);
    builder.setFlinkCluster(cluster);
    builder.setFlinkClusterSpecification(clusterSpecification);
    builder.localResourcesBasePath(stagingPath);
    // If "CONDA" is not the first in order of dynamic properties, the sdk_worker.sh script in chef needs to be updated.
    addDynamicProperty("CONDA", settings.getCurrentCondaEnvironment());
    addDynamicProperty(Settings.LOGSTASH_JOB_INFO, project.getName().toLowerCase() + "," + job.getName() + "," + job.getId() + "," + YarnRunner.APPID_PLACEHOLDER);
    StringBuilder dynamicPropertiesEncoded = new StringBuilder();
    /*
     * Split propertes with "@@"
     * https://github.com/apache/flink/blob/b410c393c960f55c09fadd4f22732d06f801b938/
     * flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java
     */
    if (!dynamicProperties.isEmpty()) {
        for (String s : dynamicProperties.keySet()) {
            dynamicPropertiesEncoded.append(s).append("=").append(dynamicProperties.get(s)).append("@@");
        }
    }
    for (String key : finalJobProps.keySet()) {
        dynamicPropertiesEncoded.append(key).append("=").append(finalJobProps.get(key)).append("@@");
    }
    if (dynamicPropertiesEncoded.length() > 0) {
        cluster.setDynamicPropertiesEncoded(dynamicPropertiesEncoded.substring(0, dynamicPropertiesEncoded.lastIndexOf("@@")));
    }
    builder.setJobType(JobType.FLINK);
    if (!Strings.isNullOrEmpty(flinkJobConfiguration.getAppName())) {
        flinkJobConfiguration.setAppName("Flink session with " + flinkJobConfiguration.getNumberOfTaskManagers() + " " + "TaskManagers");
    }
    cluster.setName(flinkJobConfiguration.getAppName());
    return builder.build(settings.getFlinkDir(), JobType.FLINK, services);
}

Example 2 with YarnRunner

use of io.hops.hopsworks.common.jobs.yarn.YarnRunner in project hopsworks by logicalclocks.

the class SparkYarnRunnerBuilder method getYarnRunner.

/**
 * Get a YarnRunner instance that will launch a Spark job.
 *
 * @param project name of the project
 * @param jobUser
 * @param services
 * @param dfsClient
 * @param yarnClient
 * @param settings
 * @return The YarnRunner instance to launch the Spark job on Yarn.
 * @throws IOException If creation failed.
 */
public YarnRunner getYarnRunner(Project project, String jobUser, Users hopsworksUser, AsynchronousJobExecutor services, final DistributedFileSystemOps dfsClient, final YarnClient yarnClient, Settings settings, String kafkaBrokersString, String hopsworksRestEndpoint, ServingConfig servingConfig, ServiceDiscoveryController serviceDiscoveryController) throws IOException, ServiceDiscoveryException, JobException, ApiKeyException {
    Map<String, ConfigProperty> jobHopsworksProps = new HashMap<>();
    JobType jobType = job.getJobConfig().getJobType();
    String appPath = ((SparkJobConfiguration) job.getJobConfig()).getAppPath();
    // Create a builder
    YarnRunner.Builder builder = new YarnRunner.Builder(Settings.SPARK_AM_MAIN);
    builder.setJobType(jobType);
    builder.setYarnClient(yarnClient);
    builder.setDfsClient(dfsClient);
    /**
     * * 1. Set stagingPath **
     */
    String stagingPath = "/Projects/" + project.getName() + "/" + Settings.PROJECT_STAGING_DIR + "/.sparkjobstaging-" + YarnRunner.APPID_PLACEHOLDER;
    builder.localResourcesBasePath(stagingPath);
    // //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    /**
     * * 2. Set job local resources, i.e. project certificates, job jar etc. **
     */
    // Add hdfs prefix so the monitor knows it should find it there
    builder.addFileToRemove("hdfs://" + stagingPath);
    // Add app file
    String appExecName = null;
    if (jobType == JobType.SPARK) {
        appExecName = Settings.SPARK_LOCRSC_APP_JAR;
    } else if (jobType == JobType.PYSPARK) {
        appExecName = appPath.substring(appPath.lastIndexOf(File.separator) + 1);
    }
    builder.addLocalResource(new LocalResourceDTO(appExecName, appPath, LocalResourceVisibility.APPLICATION.toString(), LocalResourceType.FILE.toString(), null), dfsClient);
    builder.addToAppMasterEnvironment(YarnRunner.KEY_CLASSPATH, Settings.SPARK_LOCRSC_APP_JAR);
    // Set executor extraJavaOptions to make parameters available to executors
    Map<String, String> extraJavaOptions = new HashMap<>();
    // These properties are set so that spark history server picks them up
    jobHopsworksProps.put(Settings.SPARK_DRIVER_STAGINGDIR_ENV, new ConfigProperty(Settings.SPARK_DRIVER_STAGINGDIR_ENV, HopsUtils.IGNORE, stagingPath));
    jobHopsworksProps.put(Settings.HOPSWORKS_APPID_PROPERTY, new ConfigProperty(Settings.HOPSWORKS_APPID_PROPERTY, HopsUtils.IGNORE, YarnRunner.APPID_PLACEHOLDER));
    extraJavaOptions.put(Settings.HOPSWORKS_APPID_PROPERTY, YarnRunner.APPID_PLACEHOLDER);
    extraJavaOptions.put(Settings.LOGSTASH_JOB_INFO, project.getName().toLowerCase() + "," + jobName + "," + job.getId() + "," + YarnRunner.APPID_PLACEHOLDER);
    // Set up command
    StringBuilder amargs = new StringBuilder("--class ");
    amargs.append(((SparkJobConfiguration) job.getJobConfig()).getMainClass());
    if (jobType == JobType.PYSPARK) {
        amargs.append(" --primary-py-file ").append(appExecName);
    }
    Map<String, String> finalJobProps = new HashMap<>();
    finalJobProps.putAll(sparkConfigurationUtil.setFrameworkProperties(project, job.getJobConfig(), settings, jobUser, hopsworksUser, extraJavaOptions, kafkaBrokersString, hopsworksRestEndpoint, servingConfig, serviceDiscoveryController));
    finalJobProps.put(Settings.SPARK_YARN_APPMASTER_SPARK_USER, jobUser);
    finalJobProps.put(Settings.SPARK_EXECUTOR_SPARK_USER, jobUser);
    finalJobProps.put(Settings.SPARK_YARN_APPMASTER_YARN_MODE, "true");
    finalJobProps.put(Settings.SPARK_YARN_APPMASTER_YARN_STAGING_DIR, stagingPath);
    // Parse properties from Spark config file
    Properties sparkProperties = new Properties();
    try (InputStream is = new FileInputStream(settings.getSparkDir() + "/" + Settings.SPARK_CONFIG_FILE)) {
        sparkProperties.load(is);
        // For every property that is in the spark configuration file but is not already set, create a system property.
        for (String property : sparkProperties.stringPropertyNames()) {
            if (!finalJobProps.containsKey(property)) {
                finalJobProps.put(property, sparkProperties.getProperty(property).trim());
            }
        }
    }
    for (String jvmOption : finalJobProps.get(Settings.SPARK_DRIVER_EXTRA_JAVA_OPTIONS).split(" +")) {
        builder.addJavaOption(jvmOption);
    }
    for (String key : finalJobProps.keySet()) {
        if (key.startsWith("spark.yarn.appMasterEnv.")) {
            builder.addToAppMasterEnvironment(key.replace("spark.yarn.appMasterEnv.", ""), finalJobProps.get(key));
        }
        addSystemProperty(key, finalJobProps.get(key));
    }
    builder.addToAppMasterEnvironment("CLASSPATH", finalJobProps.get(Settings.SPARK_DRIVER_EXTRACLASSPATH));
    for (String s : sysProps.keySet()) {
        String option = YarnRunner.escapeForShell("-D" + s + "=" + sysProps.get(s));
        builder.addJavaOption(option);
    }
    for (String s : jobArgs) {
        amargs.append(" --arg '").append(s).append("'");
    }
    amargs.append(" --dist-cache-conf 'distcache.conf'");
    builder.amArgs(amargs.toString());
    // Set up Yarn properties
    builder.amMemory(sparkJobConfiguration.getAmMemory());
    builder.amVCores(sparkJobConfiguration.getAmVCores());
    builder.amQueue(sparkJobConfiguration.getAmQueue());
    // pyfiles, jars and files are distributed as spark.yarn.dist.files
    String hopsFiles = finalJobProps.get("spark.yarn.dist.files");
    if (!Strings.isNullOrEmpty(hopsFiles)) {
        for (String filePath : hopsFiles.split(",")) {
            String fileName = filePath.substring(filePath.lastIndexOf("/") + 1);
            if (filePath.contains("#")) {
                fileName = filePath.split("#")[1];
                filePath = filePath.substring(0, filePath.indexOf("#"));
            }
            builder.addLocalResource(new LocalResourceDTO(fileName, filePath, LocalResourceVisibility.APPLICATION.toString(), LocalResourceType.FILE.toString(), null), dfsClient);
        }
    }
    String archives = finalJobProps.get("spark.yarn.dist.archives");
    if (!Strings.isNullOrEmpty(archives)) {
        for (String archivePath : archives.split(",")) {
            String fileName = archivePath.substring(archivePath.lastIndexOf("/") + 1);
            if (archivePath.contains("#")) {
                fileName = archivePath.split("#")[1];
                archivePath = archivePath.substring(0, archivePath.indexOf("#"));
            }
            builder.addLocalResource(new LocalResourceDTO(fileName, archivePath, LocalResourceVisibility.APPLICATION.toString(), LocalResourceType.ARCHIVE.toString(), null), dfsClient);
        }
    }
    // Set app name
    builder.appName(jobName);
    return builder.build(settings.getSparkDir(), JobType.SPARK, services);
}