Examples with SparkJobConfiguration io.hops.hopsworks.persistence.entity.jobs.configuration.spark.SparkJobConfiguration used on opensource projects

Example 1 with SparkJobConfiguration

use of io.hops.hopsworks.persistence.entity.jobs.configuration.spark.SparkJobConfiguration in project hopsworks by logicalclocks.

the class JupyterSettingsFacade method findByProjectUser.

public JupyterSettings findByProjectUser(Project project, String email) {
    JupyterSettingsPK pk = new JupyterSettingsPK(project.getId(), email);
    JupyterSettings js;
    js = em.find(JupyterSettings.class, pk);
    if (js == null) {
        String secret = DigestUtils.sha256Hex(Integer.toString(ThreadLocalRandom.current().nextInt()));
        js = new JupyterSettings(pk);
        js.setSecret(secret);
        js.setJobConfig(new SparkJobConfiguration(ExperimentType.EXPERIMENT));
        js.setBaseDir(Utils.getProjectPath(project.getName()) + Settings.ServiceDataset.JUPYTER.getName());
        persist(js);
    }
    if (js.getJobConfig() == null) {
        js.setJobConfig(new SparkJobConfiguration(ExperimentType.EXPERIMENT));
    }
    return js;
}

Example 2 with SparkJobConfiguration

use of io.hops.hopsworks.persistence.entity.jobs.configuration.spark.SparkJobConfiguration in project hopsworks by logicalclocks.

the class SparkJob method setupJob.

@Override
protected boolean setupJob() {
    SparkJobConfiguration jobconfig = (SparkJobConfiguration) jobs.getJobConfig();
    // Then: actually get to running.
    if (jobconfig.getAppName() == null || jobconfig.getAppName().isEmpty()) {
        jobconfig.setAppName("Untitled Spark Job");
    }
    // If runnerbuilder is not null, it has been instantiated by child class,
    if (runnerbuilder == null) {
        runnerbuilder = new SparkYarnRunnerBuilder(jobs);
        runnerbuilder.setJobName(jobconfig.getAppName());
    }
    if (jobconfig.getLocalResources() != null) {
        runnerbuilder.addExtraFiles(Arrays.asList(jobconfig.getLocalResources()));
    }
    // Set project specific resources, i.e. Kafka certificates
    runnerbuilder.addExtraFiles(projectLocalResources);
    if (jobSystemProperties != null && !jobSystemProperties.isEmpty()) {
        for (Entry<String, String> jobSystemProperty : jobSystemProperties.entrySet()) {
            runnerbuilder.addSystemProperty(jobSystemProperty.getKey(), jobSystemProperty.getValue());
        }
    }
    String stdFinalDestination = Utils.getProjectPath(jobs.getProject().getName()) + Settings.BaseDataset.LOGS.getName() + "/" + JobType.SPARK.getName();
    setStdOutFinalDestination(stdFinalDestination);
    setStdErrFinalDestination(stdFinalDestination);
    try {
        runner = runnerbuilder.getYarnRunner(jobs.getProject(), jobUser, user, services, services.getFileOperations(hdfsUser.getUserName()), yarnClient, settings, kafkaBrokersString, hopsworksRestEndpoint, servingConfig, serviceDiscoveryController);
    } catch (Exception e) {
        LOG.log(Level.WARNING, "Failed to create YarnRunner.", e);
        try {
            writeToLogs(e.getLocalizedMessage());
        } catch (IOException ex) {
            LOG.log(Level.SEVERE, "Failed to write logs for failed application.", ex);
        }
        return false;
    }
    return true;
}

Example 3 with SparkJobConfiguration

use of io.hops.hopsworks.persistence.entity.jobs.configuration.spark.SparkJobConfiguration in project hopsworks by logicalclocks.

the class SparkController method sanityCheck.

private void sanityCheck(Jobs job, Users user) throws GenericException, ProjectException {
    if (job == null) {
        throw new IllegalArgumentException("Trying to start job but job is not provided");
    } else if (user == null) {
        throw new IllegalArgumentException("Trying to start job but user is not provided");
    } else if (job.getJobType() != JobType.SPARK && job.getJobType() != JobType.PYSPARK) {
        throw new IllegalArgumentException("Job configuration is not a Spark job configuration. Type: " + job.getJobType());
    }
    SparkJobConfiguration jobConf = (SparkJobConfiguration) job.getJobConfig();
    if (jobConf == null) {
        throw new IllegalArgumentException("Trying to start job but JobConfiguration is null");
    }
    String path = jobConf.getAppPath();
    if (Strings.isNullOrEmpty(path) || !(path.endsWith(".jar") || path.endsWith(".py") || path.endsWith(".ipynb"))) {
        throw new IllegalArgumentException("Path does not point to a .jar, .py or .ipynb file.");
    }
    inspectDependencies(job.getProject(), user, (SparkJobConfiguration) job.getJobConfig());
}

Example 4 with SparkJobConfiguration

use of io.hops.hopsworks.persistence.entity.jobs.configuration.spark.SparkJobConfiguration in project hopsworks by logicalclocks.

the class SparkController method inspectProgram.

public SparkJobConfiguration inspectProgram(SparkJobConfiguration existingConfig, String path, DistributedFileSystemOps udfso) throws JobException {
    SparkJobConfiguration sparkConfig = null;
    if (existingConfig == null) {
        sparkConfig = new SparkJobConfiguration();
    } else {
        sparkConfig = existingConfig;
    }
    // If the main program is in a jar, try to set main class from it
    if (path.endsWith(".jar")) {
        try (JarInputStream jis = new JarInputStream(udfso.open(path))) {
            Manifest mf = jis.getManifest();
            if (mf != null) {
                Attributes atts = mf.getMainAttributes();
                if (atts.containsKey(Attributes.Name.MAIN_CLASS)) {
                    sparkConfig.setMainClass(atts.getValue(Attributes.Name.MAIN_CLASS));
                } else {
                    sparkConfig.setMainClass(null);
                }
            }
        } catch (IOException ex) {
            throw new JobException(RESTCodes.JobErrorCode.JAR_INSPECTION_ERROR, Level.SEVERE, "Failed to inspect jar at:" + path, ex.getMessage(), ex);
        }
    } else {
        // In that case we should not override it and set the experimentType, only set it if no default exists
        if (existingConfig == null) {
            sparkConfig.setExperimentType(ExperimentType.EXPERIMENT);
        }
        sparkConfig.setMainClass(Settings.SPARK_PY_MAINCLASS);
    }
    sparkConfig.setAppPath(path);
    return sparkConfig;
}

Example 5 with SparkJobConfiguration

use of io.hops.hopsworks.persistence.entity.jobs.configuration.spark.SparkJobConfiguration in project hopsworks by logicalclocks.

the class AbstractExecutionController method start.

@Override
@TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED)
public Execution start(Jobs job, String args, Users user) throws JobException, GenericException, ServiceException, ProjectException {
    // If the limit for the number of executions for this job has been reached, return an error
    checkExecutionLimit(job);
    // A user should not be able to start a job if the project is prepaid and it doesn't have quota.
    if (job.getProject().getPaymentType().equals(PaymentType.PREPAID)) {
        YarnProjectsQuota projectQuota = yarnProjectsQuotaFacade.findByProjectName(job.getProject().getName());
        if (projectQuota == null || projectQuota.getQuotaRemaining() <= 0) {
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_QUOTA_ERROR, Level.FINE);
        }
    }
    // If enabled and nodemanagers are all offline throw an JobException exception
    if (settings.isCheckingForNodemanagerStatusEnabled() && job.getJobType() != JobType.PYTHON) {
        hostServicesFacade.findServices("nodemanager").stream().filter(s -> s.getStatus() == ServiceStatus.Started).findFirst().orElseThrow(() -> new JobException(RESTCodes.JobErrorCode.NODEMANAGERS_OFFLINE, Level.SEVERE));
    }
    Execution exec;
    switch(job.getJobType()) {
        case FLINK:
            // Materialize certs
            return flinkController.startJob(job, user);
        case SPARK:
            exec = sparkController.startJob(job, args, user);
            if (exec == null) {
                throw new IllegalArgumentException("Problem getting execution object for: " + job.getJobType());
            }
            SparkJobConfiguration config = (SparkJobConfiguration) job.getJobConfig();
            String path = config.getAppPath();
            String pathOfInode;
            try {
                pathOfInode = Utils.prepPath(path);
            } catch (UnsupportedEncodingException ex) {
                throw new JobException(RESTCodes.JobErrorCode.JOB_START_FAILED, Level.FINE, "Job name: " + job.getName(), ex.getMessage(), ex);
            }
            Inode inode = inodeController.getInodeAtPath(pathOfInode);
            String inodeName = inode.getInodePK().getName();
            activityFacade.persistActivity(ActivityFacade.EXECUTED_JOB + inodeName, job.getProject(), user, ActivityFlag.JOB);
            break;
        case PYSPARK:
            if (job.getProject().getPythonEnvironment() == null) {
                throw new ProjectException(RESTCodes.ProjectErrorCode.ANACONDA_NOT_ENABLED, Level.FINEST);
            }
            exec = sparkController.startJob(job, args, user);
            if (exec == null) {
                throw new IllegalArgumentException("Error while getting execution object for: " + job.getJobType());
            }
            break;
        default:
            throw new GenericException(RESTCodes.GenericErrorCode.UNKNOWN_ACTION, Level.FINE, "Unsupported job type: " + job.getJobType());
    }
    return exec;
}