Examples with LocalResourceDTO io.hops.hopsworks.persistence.entity.jobs.configuration.yarn.LocalResourceDTO used on opensource projects

Example 1 with LocalResourceDTO

use of io.hops.hopsworks.persistence.entity.jobs.configuration.yarn.LocalResourceDTO in project hopsworks by logicalclocks.

the class YarnRunner method addAllToLocalResources.

private Map<String, LocalResource> addAllToLocalResources(DistributedFileSystemOps dfs) throws IOException, URISyntaxException {
    Map<String, LocalResource> localResources = new HashMap<>();
    // Construct basepath
    FileSystem fs = dfsClient.getFilesystem();
    String hdfsPrefix = conf.get("fs.defaultFS");
    String basePath = hdfsPrefix + localResourcesBasePath;
    logger.log(Level.FINER, "Base path: {0}", basePath);
    // For all local resources with hdfs path: add local resource
    for (Entry<String, LocalResourceDTO> entry : amLocalResourcesOnHDFS.entrySet()) {
        logger.log(Level.FINE, "LocalResourceDTO to upload is :{0}", entry.toString());
        String key = entry.getKey();
        String pathToResource = entry.getValue().getPath();
        pathToResource = pathToResource.replaceFirst("hdfs:/*Projects", "hdfs:///Projects");
        pathToResource = pathToResource.replaceFirst("hdfs:/*user", "hdfs:///user");
        Path src = new Path(pathToResource);
        FileStatus scFileStat = fs.getFileStatus(src);
        LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(src), LocalResourceType.valueOf(entry.getValue().getType().toUpperCase()), LocalResourceVisibility.valueOf(entry.getValue().getVisibility().toUpperCase()), scFileStat.getLen(), scFileStat.getModificationTime(), entry.getValue().getPattern());
        localResources.put(key, scRsrc);
    }
    // as system properties (javaOptions)
    if (jobType == JobType.SPARK || jobType == JobType.PYSPARK) {
        StringBuilder uris = new StringBuilder();
        StringBuilder timestamps = new StringBuilder();
        StringBuilder sizes = new StringBuilder();
        StringBuilder visibilities = new StringBuilder();
        StringBuilder types = new StringBuilder();
        for (Entry<String, LocalResource> entry : localResources.entrySet()) {
            Path destPath = ConverterUtils.getPathFromYarnURL(entry.getValue().getResource());
            URI sparkUri = destPath.toUri();
            URI pathURI = new URI(sparkUri.getScheme(), sparkUri.getAuthority(), sparkUri.getPath(), null, entry.getKey());
            uris.append(pathURI.toString()).append(",");
            timestamps.append(entry.getValue().getTimestamp()).append(",");
            sizes.append(entry.getValue().getSize()).append(",");
            visibilities.append(entry.getValue().getVisibility()).append(",");
            types.append(entry.getValue().getType()).append(",");
        }
        StringBuilder distCacheConf = new StringBuilder();
        distCacheConf.append(Settings.SPARK_CACHE_FILENAMES + "\t" + uris.substring(0, uris.length() - 1) + "\n");
        distCacheConf.append(Settings.SPARK_CACHE_TIMESTAMPS + "\t" + timestamps.substring(0, timestamps.length() - 1) + "\n");
        distCacheConf.append(Settings.SPARK_CACHE_SIZES + "\t" + sizes.substring(0, sizes.length() - 1) + "\n");
        distCacheConf.append(Settings.SPARK_CACHE_VISIBILITIES + "\t" + visibilities.substring(0, visibilities.length() - 1) + "\n");
        distCacheConf.append(Settings.SPARK_CACHE_TYPES + "\t" + types.substring(0, types.length() - 1) + "\n");
        String distCacheConfPath = hdfsPrefix + localResourcesBasePath + "/distcache.conf";
        Path distCacheSrc = new Path(distCacheConfPath);
        dfs.create(new Path(distCacheConfPath), distCacheConf.toString());
        FileStatus distCacheFileStat = fs.getFileStatus(distCacheSrc);
        LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(distCacheSrc), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, distCacheFileStat.getLen(), distCacheFileStat.getModificationTime());
        localResources.put("distcache.conf", scRsrc);
    }
    return localResources;
}

Example 2 with LocalResourceDTO

use of io.hops.hopsworks.persistence.entity.jobs.configuration.yarn.LocalResourceDTO in project hopsworks by logicalclocks.

the class YarnRunner method copyUserCertificates.

/**
 * This method is only used by Spark family jobs. Flink jobs copy their
 * certificates in FlinkJob since it's a little bit problematic the way it
 * submits a job to Yarn
 *
 * @param project
 * @param jobType
 * @param dfso
 * @param username
 * @param applicationId
 */
private void copyUserCertificates(Project project, JobType jobType, DistributedFileSystemOps dfso, String username, String applicationId) {
    List<LocalResourceDTO> materialResources = new ArrayList<>(2);
    Map<String, String> systemProperties = new HashMap<>(2);
    HopsUtils.copyProjectUserCerts(project, username, services.getSettings().getHopsworksTmpCertDir(), services.getSettings().getHdfsTmpCertDir(), jobType, dfso, materialResources, applicationId, services.getCertificateMaterializer());
    for (LocalResourceDTO materialDTO : materialResources) {
        amLocalResourcesOnHDFS.put(materialDTO.getName(), materialDTO);
    }
    for (Map.Entry<String, String> sysProp : systemProperties.entrySet()) {
        String option = YarnRunner.escapeForShell("-D" + sysProp.getKey() + "=" + sysProp.getValue());
        javaOptions.add(option);
    }
}

Example 3 with LocalResourceDTO

use of io.hops.hopsworks.persistence.entity.jobs.configuration.yarn.LocalResourceDTO in project hopsworks by logicalclocks.

the class SparkYarnRunnerBuilder method getYarnRunner.

/**
 * Get a YarnRunner instance that will launch a Spark job.
 *
 * @param project name of the project
 * @param jobUser
 * @param services
 * @param dfsClient
 * @param yarnClient
 * @param settings
 * @return The YarnRunner instance to launch the Spark job on Yarn.
 * @throws IOException If creation failed.
 */
public YarnRunner getYarnRunner(Project project, String jobUser, Users hopsworksUser, AsynchronousJobExecutor services, final DistributedFileSystemOps dfsClient, final YarnClient yarnClient, Settings settings, String kafkaBrokersString, String hopsworksRestEndpoint, ServingConfig servingConfig, ServiceDiscoveryController serviceDiscoveryController) throws IOException, ServiceDiscoveryException, JobException, ApiKeyException {
    Map<String, ConfigProperty> jobHopsworksProps = new HashMap<>();
    JobType jobType = job.getJobConfig().getJobType();
    String appPath = ((SparkJobConfiguration) job.getJobConfig()).getAppPath();
    // Create a builder
    YarnRunner.Builder builder = new YarnRunner.Builder(Settings.SPARK_AM_MAIN);
    builder.setJobType(jobType);
    builder.setYarnClient(yarnClient);
    builder.setDfsClient(dfsClient);
    /**
     * * 1. Set stagingPath **
     */
    String stagingPath = "/Projects/" + project.getName() + "/" + Settings.PROJECT_STAGING_DIR + "/.sparkjobstaging-" + YarnRunner.APPID_PLACEHOLDER;
    builder.localResourcesBasePath(stagingPath);
    // //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    /**
     * * 2. Set job local resources, i.e. project certificates, job jar etc. **
     */
    // Add hdfs prefix so the monitor knows it should find it there
    builder.addFileToRemove("hdfs://" + stagingPath);
    // Add app file
    String appExecName = null;
    if (jobType == JobType.SPARK) {
        appExecName = Settings.SPARK_LOCRSC_APP_JAR;
    } else if (jobType == JobType.PYSPARK) {
        appExecName = appPath.substring(appPath.lastIndexOf(File.separator) + 1);
    }
    builder.addLocalResource(new LocalResourceDTO(appExecName, appPath, LocalResourceVisibility.APPLICATION.toString(), LocalResourceType.FILE.toString(), null), dfsClient);
    builder.addToAppMasterEnvironment(YarnRunner.KEY_CLASSPATH, Settings.SPARK_LOCRSC_APP_JAR);
    // Set executor extraJavaOptions to make parameters available to executors
    Map<String, String> extraJavaOptions = new HashMap<>();
    // These properties are set so that spark history server picks them up
    jobHopsworksProps.put(Settings.SPARK_DRIVER_STAGINGDIR_ENV, new ConfigProperty(Settings.SPARK_DRIVER_STAGINGDIR_ENV, HopsUtils.IGNORE, stagingPath));
    jobHopsworksProps.put(Settings.HOPSWORKS_APPID_PROPERTY, new ConfigProperty(Settings.HOPSWORKS_APPID_PROPERTY, HopsUtils.IGNORE, YarnRunner.APPID_PLACEHOLDER));
    extraJavaOptions.put(Settings.HOPSWORKS_APPID_PROPERTY, YarnRunner.APPID_PLACEHOLDER);
    extraJavaOptions.put(Settings.LOGSTASH_JOB_INFO, project.getName().toLowerCase() + "," + jobName + "," + job.getId() + "," + YarnRunner.APPID_PLACEHOLDER);
    // Set up command
    StringBuilder amargs = new StringBuilder("--class ");
    amargs.append(((SparkJobConfiguration) job.getJobConfig()).getMainClass());
    if (jobType == JobType.PYSPARK) {
        amargs.append(" --primary-py-file ").append(appExecName);
    }
    Map<String, String> finalJobProps = new HashMap<>();
    finalJobProps.putAll(sparkConfigurationUtil.setFrameworkProperties(project, job.getJobConfig(), settings, jobUser, hopsworksUser, extraJavaOptions, kafkaBrokersString, hopsworksRestEndpoint, servingConfig, serviceDiscoveryController));
    finalJobProps.put(Settings.SPARK_YARN_APPMASTER_SPARK_USER, jobUser);
    finalJobProps.put(Settings.SPARK_EXECUTOR_SPARK_USER, jobUser);
    finalJobProps.put(Settings.SPARK_YARN_APPMASTER_YARN_MODE, "true");
    finalJobProps.put(Settings.SPARK_YARN_APPMASTER_YARN_STAGING_DIR, stagingPath);
    // Parse properties from Spark config file
    Properties sparkProperties = new Properties();
    try (InputStream is = new FileInputStream(settings.getSparkDir() + "/" + Settings.SPARK_CONFIG_FILE)) {
        sparkProperties.load(is);
        // For every property that is in the spark configuration file but is not already set, create a system property.
        for (String property : sparkProperties.stringPropertyNames()) {
            if (!finalJobProps.containsKey(property)) {
                finalJobProps.put(property, sparkProperties.getProperty(property).trim());
            }
        }
    }
    for (String jvmOption : finalJobProps.get(Settings.SPARK_DRIVER_EXTRA_JAVA_OPTIONS).split(" +")) {
        builder.addJavaOption(jvmOption);
    }
    for (String key : finalJobProps.keySet()) {
        if (key.startsWith("spark.yarn.appMasterEnv.")) {
            builder.addToAppMasterEnvironment(key.replace("spark.yarn.appMasterEnv.", ""), finalJobProps.get(key));
        }
        addSystemProperty(key, finalJobProps.get(key));
    }
    builder.addToAppMasterEnvironment("CLASSPATH", finalJobProps.get(Settings.SPARK_DRIVER_EXTRACLASSPATH));
    for (String s : sysProps.keySet()) {
        String option = YarnRunner.escapeForShell("-D" + s + "=" + sysProps.get(s));
        builder.addJavaOption(option);
    }
    for (String s : jobArgs) {
        amargs.append(" --arg '").append(s).append("'");
    }
    amargs.append(" --dist-cache-conf 'distcache.conf'");
    builder.amArgs(amargs.toString());
    // Set up Yarn properties
    builder.amMemory(sparkJobConfiguration.getAmMemory());
    builder.amVCores(sparkJobConfiguration.getAmVCores());
    builder.amQueue(sparkJobConfiguration.getAmQueue());
    // pyfiles, jars and files are distributed as spark.yarn.dist.files
    String hopsFiles = finalJobProps.get("spark.yarn.dist.files");
    if (!Strings.isNullOrEmpty(hopsFiles)) {
        for (String filePath : hopsFiles.split(",")) {
            String fileName = filePath.substring(filePath.lastIndexOf("/") + 1);
            if (filePath.contains("#")) {
                fileName = filePath.split("#")[1];
                filePath = filePath.substring(0, filePath.indexOf("#"));
            }
            builder.addLocalResource(new LocalResourceDTO(fileName, filePath, LocalResourceVisibility.APPLICATION.toString(), LocalResourceType.FILE.toString(), null), dfsClient);
        }
    }
    String archives = finalJobProps.get("spark.yarn.dist.archives");
    if (!Strings.isNullOrEmpty(archives)) {
        for (String archivePath : archives.split(",")) {
            String fileName = archivePath.substring(archivePath.lastIndexOf("/") + 1);
            if (archivePath.contains("#")) {
                fileName = archivePath.split("#")[1];
                archivePath = archivePath.substring(0, archivePath.indexOf("#"));
            }
            builder.addLocalResource(new LocalResourceDTO(fileName, archivePath, LocalResourceVisibility.APPLICATION.toString(), LocalResourceType.ARCHIVE.toString(), null), dfsClient);
        }
    }
    // Set app name
    builder.appName(jobName);
    return builder.build(settings.getSparkDir(), JobType.SPARK, services);
}

Example 4 with LocalResourceDTO

use of io.hops.hopsworks.persistence.entity.jobs.configuration.yarn.LocalResourceDTO in project hopsworks by logicalclocks.

the class HopsUtils method copyProjectUserCerts.

/**
 * Utility method that copies project user certificates from the Database, to
 * either hdfs to be passed as LocalResources to the YarnJob or to used
 * by another method.
 *
 * @param project
 * @param username
 * @param localTmpDir
 * @param remoteTmpDir
 * @param jobType
 * @param dfso
 * @param projectLocalResources
 * @param applicationId
 */
public static void copyProjectUserCerts(Project project, String username, String localTmpDir, String remoteTmpDir, JobType jobType, DistributedFileSystemOps dfso, List<LocalResourceDTO> projectLocalResources, String applicationId, CertificateMaterializer certMat) {
    // Let the Certificate Materializer handle the certificates
    UserCerts userCert = new UserCerts(project.getName(), username);
    try {
        certMat.materializeCertificatesLocal(username, project.getName());
        CertificateMaterializer.CryptoMaterial material = certMat.getUserMaterial(username, project.getName());
        userCert.setUserKey(material.getKeyStore().array());
        userCert.setUserCert(material.getTrustStore().array());
        userCert.setUserKeyPwd(new String(material.getPassword()));
    } catch (IOException | CryptoPasswordNotFoundException ex) {
        throw new RuntimeException("Could not materialize user certificates", ex);
    }
    // Check if the user certificate was actually retrieved
    if (userCert.getUserCert() != null && userCert.getUserCert().length > 0 && userCert.getUserKey() != null && userCert.getUserKey().length > 0) {
        Map<String, byte[]> certFiles = new HashMap<>();
        certFiles.put(Settings.T_CERTIFICATE, userCert.getUserCert());
        certFiles.put(Settings.K_CERTIFICATE, userCert.getUserKey());
        try {
            String kCertName = HopsUtils.getProjectKeystoreName(project.getName(), username);
            String tCertName = HopsUtils.getProjectTruststoreName(project.getName(), username);
            String passName = getProjectMaterialPasswordName(project.getName(), username);
            try {
                if (jobType != null) {
                    switch(jobType) {
                        case PYSPARK:
                        case SPARK:
                            Map<String, File> certs = new HashMap<>();
                            certs.put(Settings.K_CERTIFICATE, new File(localTmpDir + File.separator + kCertName));
                            certs.put(Settings.T_CERTIFICATE, new File(localTmpDir + File.separator + tCertName));
                            certs.put(Settings.CRYPTO_MATERIAL_PASSWORD, new File(localTmpDir + File.separator + passName));
                            for (Map.Entry<String, File> entry : certs.entrySet()) {
                                // by the YarnJob
                                if (!dfso.exists(remoteTmpDir)) {
                                    Path remoteTmpDirPath = new Path(remoteTmpDir);
                                    dfso.mkdir(remoteTmpDirPath, FsPermission.getDirDefault());
                                    dfso.setPermission(remoteTmpDirPath, FsPermissions.rwxrwxrwx);
                                }
                                // Put project certificates in its own dir
                                String certUser = project.getName() + "__" + username;
                                String remoteTmpProjDir = remoteTmpDir + File.separator + certUser;
                                if (!dfso.exists(remoteTmpProjDir)) {
                                    Path remoteTmpProjDirPath = new Path(remoteTmpProjDir);
                                    dfso.mkdir(remoteTmpProjDirPath, FsPermission.getDirDefault());
                                    dfso.setPermission(remoteTmpProjDirPath, FsPermissions.rwxrwx___);
                                    dfso.setOwner(remoteTmpProjDirPath, certUser, certUser);
                                }
                                String remoteProjAppDir = remoteTmpProjDir + File.separator + applicationId;
                                Path remoteProjAppPath = new Path(remoteProjAppDir);
                                if (!dfso.exists(remoteProjAppDir)) {
                                    dfso.mkdir(remoteProjAppPath, FsPermission.getDirDefault());
                                    dfso.setPermission(remoteProjAppPath, FsPermissions.rwxrwx___);
                                    dfso.setOwner(remoteProjAppPath, certUser, certUser);
                                }
                                dfso.copyToHDFSFromLocal(false, entry.getValue().getAbsolutePath(), remoteProjAppDir + File.separator + entry.getValue().getName());
                                dfso.setPermission(new Path(remoteProjAppDir + File.separator + entry.getValue().getName()), FsPermissions.rwx______);
                                dfso.setOwner(new Path(remoteProjAppDir + File.separator + entry.getValue().getName()), certUser, certUser);
                                projectLocalResources.add(new LocalResourceDTO(entry.getKey(), "hdfs://" + remoteProjAppDir + File.separator + entry.getValue().getName(), LocalResourceVisibility.APPLICATION.toString(), LocalResourceType.FILE.toString(), null));
                            }
                            break;
                        default:
                            break;
                    }
                }
            } catch (IOException ex) {
                LOG.log(Level.SEVERE, "Error writing project user certificates to local fs", ex);
            }
        } finally {
            if (jobType != null) {
                certMat.removeCertificatesLocal(username, project.getName());
            }
        }
    }
}