Examples with SparkConfigurationUtil io.hops.hopsworks.common.util.SparkConfigurationUtil used on opensource projects

Example 1 with SparkConfigurationUtil

use of io.hops.hopsworks.common.util.SparkConfigurationUtil in project hopsworks by logicalclocks.

the class JobController method putJob.

public Jobs putJob(Users user, Project project, Jobs job, JobConfiguration config) throws JobException {
    try {
        if (config.getJobType() == JobType.SPARK || config.getJobType() == JobType.PYSPARK) {
            SparkConfigurationUtil sparkConfigurationUtil = new SparkConfigurationUtil();
            SparkJobConfiguration sparkJobConfiguration = (SparkJobConfiguration) config;
            sparkConfigurationUtil.validateExecutorMemory(sparkJobConfiguration.getExecutorMemory(), settings);
        }
        job = jobFacade.put(user, project, config, job);
    } catch (IllegalStateException ise) {
        if (ise.getCause() instanceof JAXBException) {
            throw new JobException(RESTCodes.JobErrorCode.JOB_CONFIGURATION_CONVERT_TO_JSON_ERROR, Level.FINE, "Unable to create json from JobConfiguration", ise.getMessage(), ise);
        } else {
            throw ise;
        }
    }
    if (config.getSchedule() != null) {
        scheduler.scheduleJobPeriodic(job);
    }
    activityFacade.persistActivity(ActivityFacade.CREATED_JOB + getJobNameForActivity(job.getName()), project, user, ActivityFlag.JOB);
    return job;
}

Example 2 with SparkConfigurationUtil

use of io.hops.hopsworks.common.util.SparkConfigurationUtil in project hopsworks by logicalclocks.

the class JupyterConfigFilesGenerator method createSparkMagicConfig.

public void createSparkMagicConfig(Writer out, Project project, JupyterSettings js, String hdfsUser, Users hopsworksUser, String confDirPath) throws IOException, ServiceDiscoveryException, JobException, ApiKeyException {
    SparkJobConfiguration sparkJobConfiguration = (SparkJobConfiguration) js.getJobConfig();
    // If user selected Python we should use the default spark configuration for Spark/PySpark kernels
    if (js.isPythonKernel()) {
        sparkJobConfiguration = (SparkJobConfiguration) jobController.getConfiguration(project, JobType.SPARK, true);
    }
    SparkConfigurationUtil sparkConfigurationUtil = new SparkConfigurationUtil();
    Map<String, String> extraJavaOptions = new HashMap<>();
    extraJavaOptions.put(Settings.LOGSTASH_JOB_INFO, project.getName().toLowerCase() + ",jupyter,notebook,?");
    HashMap<String, String> finalSparkConfiguration = new HashMap<>();
    finalSparkConfiguration.put(Settings.SPARK_DRIVER_STAGINGDIR_ENV, "hdfs:///Projects/" + project.getName() + "/Resources/.sparkStaging");
    // Set Hopsworks consul service domain, don't use the address, use the name
    String hopsworksRestEndpoint = "https://" + serviceDiscoveryController.constructServiceFQDNWithPort(ServiceDiscoveryController.HopsworksService.HOPSWORKS_APP);
    finalSparkConfiguration.putAll(sparkConfigurationUtil.setFrameworkProperties(project, sparkJobConfiguration, settings, hdfsUser, hopsworksUser, extraJavaOptions, kafkaBrokers.getKafkaBrokersString(), hopsworksRestEndpoint, servingConfig, serviceDiscoveryController));
    StringBuilder sparkConfBuilder = new StringBuilder();
    ArrayList<String> keys = new ArrayList<>(finalSparkConfiguration.keySet());
    Collections.sort(keys);
    for (String configKey : keys) {
        sparkConfBuilder.append("\t\"" + configKey + "\":\"" + finalSparkConfiguration.get(configKey) + "\"," + "\n");
    }
    sparkConfBuilder.deleteCharAt(sparkConfBuilder.lastIndexOf(","));
    try {
        Service livyService = serviceDiscoveryController.getAnyAddressOfServiceWithDNS(ServiceDiscoveryController.HopsworksService.LIVY);
        SparkMagicConfigTemplateBuilder templateBuilder = SparkMagicConfigTemplateBuilder.newBuilder().setLivyIp(livyService.getAddress()).setJupyterHome(confDirPath).setDriverCores(Integer.parseInt(finalSparkConfiguration.get(Settings.SPARK_DRIVER_CORES_ENV))).setDriverMemory(finalSparkConfiguration.get(Settings.SPARK_DRIVER_MEMORY_ENV)).setLivyStartupTimeout(settings.getLivyStartupTimeout());
        if (sparkJobConfiguration.isDynamicAllocationEnabled() || sparkJobConfiguration.getExperimentType() != null) {
            templateBuilder.setNumExecutors(1);
        } else {
            templateBuilder.setNumExecutors(Integer.parseInt(finalSparkConfiguration.get(Settings.SPARK_NUMBER_EXECUTORS_ENV)));
        }
        templateBuilder.setExecutorCores(Integer.parseInt(finalSparkConfiguration.get(Settings.SPARK_EXECUTOR_CORES_ENV))).setExecutorMemory(finalSparkConfiguration.get(Settings.SPARK_EXECUTOR_MEMORY_ENV)).setHdfsUser(hdfsUser).setYarnQueue(sparkJobConfiguration.getAmQueue()).setHadoopHome(settings.getHadoopSymbolicLinkDir()).setHadoopVersion(settings.getHadoopVersion()).setSparkConfiguration(sparkConfBuilder.toString());
        Map<String, Object> dataModel = new HashMap<>(1);
        dataModel.put("conf", templateBuilder.build());
        templateEngine.template(SparkMagicConfigTemplate.TEMPLATE_NAME, dataModel, out);
    } catch (TemplateException | ServiceDiscoveryException ex) {
        throw new IOException(ex);
    }
}