Examples with Jobs io.hops.hopsworks.persistence.entity.jobs.description.Jobs used on opensource projects

Example 16 with Jobs

use of io.hops.hopsworks.persistence.entity.jobs.description.Jobs in project hopsworks by logicalclocks.

the class JobsResource method put.

@ApiOperation(value = "Create or Update a Job.", response = JobDTO.class)
@PUT
@Path("{name}")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response put(@ApiParam(value = "Job configuration parameters", required = true) JobConfiguration config, @ApiParam(value = "name", required = true) @PathParam("name") String name, @Context SecurityContext sc, @Context UriInfo uriInfo) throws JobException {
    if (config == null) {
        throw new IllegalArgumentException("Job configuration was not provided.");
    }
    Users user = jWTHelper.getUserPrincipal(sc);
    if (Strings.isNullOrEmpty(config.getAppName())) {
        config.setAppName(name);
    }
    if (!HopsUtils.jobNameValidator(config.getAppName(), Settings.FILENAME_DISALLOWED_CHARS)) {
        throw new JobException(RESTCodes.JobErrorCode.JOB_NAME_INVALID, Level.FINE, "job name: " + config.getAppName());
    }
    // Check if job with same name exists so we update it instead of creating it
    Response.Status status = Response.Status.CREATED;
    Jobs job = jobFacade.findByProjectAndName(project, config.getAppName());
    if (job != null) {
        status = Response.Status.OK;
    }
    job = jobController.putJob(user, project, job, config);
    JobDTO dto = jobsBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.JOBS), job);
    UriBuilder builder = uriInfo.getAbsolutePathBuilder().path(Integer.toString(dto.getId()));
    if (status == Response.Status.CREATED) {
        return Response.created(builder.build()).entity(dto).build();
    } else {
        return Response.ok(builder.build()).entity(dto).build();
    }
}

Example 17 with Jobs

use of io.hops.hopsworks.persistence.entity.jobs.description.Jobs in project hopsworks by logicalclocks.

the class ProjectController method removeMemberFromTeam.

public void removeMemberFromTeam(Project project, Users userToBeRemoved) throws ProjectException, ServiceException, IOException, GenericException, JobException, HopsSecurityException, TensorBoardException, FeaturestoreException {
    ProjectTeam projectTeam = projectTeamFacade.findProjectTeam(project, userToBeRemoved);
    if (projectTeam == null) {
        throw new ProjectException(RESTCodes.ProjectErrorCode.TEAM_MEMBER_NOT_FOUND, Level.FINE, "project: " + project + ", user: " + userToBeRemoved.getEmail());
    }
    // Not able to remove project owner regardless of who is trying to remove the member
    if (project.getOwner().equals(userToBeRemoved)) {
        throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_OWNER_NOT_ALLOWED, Level.FINE);
    }
    projectTeamFacade.removeProjectTeam(project, userToBeRemoved);
    String hdfsUser = hdfsUsersController.getHdfsUserName(project, userToBeRemoved);
    YarnClientWrapper yarnClientWrapper = ycs.getYarnClientSuper(settings.getConfiguration());
    YarnClient client = yarnClientWrapper.getYarnClient();
    try {
        Set<String> hdfsUsers = new HashSet<>();
        hdfsUsers.add(hdfsUser);
        List<ApplicationReport> projectsApps = client.getApplications(null, hdfsUsers, null, EnumSet.of(YarnApplicationState.ACCEPTED, YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING, YarnApplicationState.RUNNING, YarnApplicationState.SUBMITTED));
        // kill jupyter for this user
        JupyterProject jupyterProject = jupyterFacade.findByUser(hdfsUser);
        if (jupyterProject != null) {
            jupyterController.shutdown(project, hdfsUser, userToBeRemoved, jupyterProject.getSecret(), jupyterProject.getCid(), jupyterProject.getPort());
        }
        // kill running TB if any
        tensorBoardController.cleanup(project, userToBeRemoved);
        // kill all jobs run by this user.
        // kill jobs
        List<Jobs> running = jobFacade.getRunningJobs(project, hdfsUser);
        if (running != null && !running.isEmpty()) {
            for (Jobs job : running) {
                executionController.stop(job);
            }
        }
        // wait that log aggregation for the jobs finish
        for (ApplicationReport appReport : projectsApps) {
            FinalApplicationStatus finalState = appReport.getFinalApplicationStatus();
            while (finalState.equals(FinalApplicationStatus.UNDEFINED)) {
                client.killApplication(appReport.getApplicationId());
                appReport = client.getApplicationReport(appReport.getApplicationId());
                finalState = appReport.getFinalApplicationStatus();
            }
            YarnLogUtil.waitForLogAggregation(client, appReport.getApplicationId());
        }
    } catch (YarnException | IOException | InterruptedException e) {
        throw new ProjectException(RESTCodes.ProjectErrorCode.KILL_MEMBER_JOBS, Level.SEVERE, "project: " + project + ", user: " + userToBeRemoved, e.getMessage(), e);
    } finally {
        ycs.closeYarnClient(yarnClientWrapper);
    }
    // trigger project team role remove handlers
    ProjectTeamRoleHandler.runProjectTeamRoleRemoveMembersHandlers(projectTeamRoleHandlers, project, Collections.singletonList(userToBeRemoved));
    // Revoke privileges for online feature store
    if (projectServiceFacade.isServiceEnabledForProject(project, ProjectServiceEnum.FEATURESTORE)) {
        Featurestore featurestore = featurestoreController.getProjectFeaturestore(project);
        onlineFeaturestoreController.removeOnlineFeaturestoreUser(featurestore, userToBeRemoved);
    }
    kafkaController.removeProjectMemberFromTopics(project, userToBeRemoved);
    certificateMaterializer.forceRemoveLocalMaterial(userToBeRemoved.getUsername(), project.getName(), null, false);
    try {
        certificatesController.revokeUserSpecificCertificates(project, userToBeRemoved);
    } catch (HopsSecurityException ex) {
        if (ex.getErrorCode() != RESTCodes.SecurityErrorCode.CERTIFICATE_NOT_FOUND) {
            LOGGER.log(Level.SEVERE, "Could not delete certificates when removing member " + userToBeRemoved.getUsername() + " from project " + project.getName() + ". Manual cleanup is needed!!!", ex);
            throw ex;
        }
    } catch (IOException | GenericException ex) {
        LOGGER.log(Level.SEVERE, "Could not delete certificates when removing member " + userToBeRemoved.getUsername() + " from project " + project.getName() + ". Manual cleanup is needed!!!", ex);
        throw ex;
    }
    // TODO: projectTeam might be null?
    hdfsUsersController.removeMember(projectTeam);
}

Example 18 with Jobs

use of io.hops.hopsworks.persistence.entity.jobs.description.Jobs in project hopsworks by logicalclocks.

the class ProjectController method addTourFilesToProject.

public String addTourFilesToProject(String username, Project project, DistributedFileSystemOps dfso, DistributedFileSystemOps udfso, TourProjectType projectType, ProvTypeDTO projectProvCore) throws DatasetException, HopsSecurityException, ProjectException, JobException, GenericException, ServiceException {
    String tourFilesDataset = Settings.HOPS_TOUR_DATASET;
    Users user = userFacade.findByEmail(username);
    if (null != projectType) {
        String projectPath = Utils.getProjectPath(project.getName());
        switch(projectType) {
            case SPARK:
                datasetController.createDataset(user, project, tourFilesDataset, "files for guide projects", Provenance.getDatasetProvCore(projectProvCore, Provenance.MLType.DATASET), false, DatasetAccessPermission.EDITABLE, dfso);
                String exampleDir = settings.getSparkDir() + Settings.SPARK_EXAMPLES_DIR + "/";
                try {
                    File dir = new File(exampleDir);
                    File[] file = dir.listFiles((File dir1, String name) -> name.matches("spark-examples(.*).jar"));
                    if (file.length == 0) {
                        throw new IllegalStateException("No spark-examples*.jar was found in " + dir.getAbsolutePath());
                    }
                    if (file.length > 1) {
                        LOGGER.log(Level.WARNING, "More than one spark-examples*.jar found in {0}.", dir.getAbsolutePath());
                    }
                    String hdfsJarPath = projectPath + tourFilesDataset + "/spark-examples.jar";
                    udfso.copyToHDFSFromLocal(false, file[0].getAbsolutePath(), hdfsJarPath);
                    String datasetGroup = hdfsUsersController.getHdfsGroupName(project, tourFilesDataset);
                    String userHdfsName = hdfsUsersController.getHdfsUserName(project, user);
                    udfso.setPermission(new Path(hdfsJarPath), udfso.getParentPermission(new Path(hdfsJarPath)));
                    udfso.setOwner(new Path(projectPath + tourFilesDataset + "/spark-examples.jar"), userHdfsName, datasetGroup);
                } catch (IOException ex) {
                    throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_TOUR_FILES_ERROR, Level.SEVERE, "project: " + project.getName(), ex.getMessage(), ex);
                }
                break;
            case KAFKA:
                datasetController.createDataset(user, project, tourFilesDataset, "files for guide projects", Provenance.getDatasetProvCore(projectProvCore, Provenance.MLType.DATASET), false, DatasetAccessPermission.EDITABLE, dfso);
                // Get the JAR from /user/<super user>
                String kafkaExampleSrc = "/user/" + settings.getSparkUser() + "/" + settings.getHopsExamplesSparkFilename();
                String kafkaExampleDst = projectPath + tourFilesDataset + "/" + settings.getHopsExamplesSparkFilename();
                try {
                    udfso.copyInHdfs(new Path(kafkaExampleSrc), new Path(kafkaExampleDst));
                    String datasetGroup = hdfsUsersController.getHdfsGroupName(project, tourFilesDataset);
                    String userHdfsName = hdfsUsersController.getHdfsUserName(project, user);
                    udfso.setPermission(new Path(kafkaExampleDst), udfso.getParentPermission(new Path(kafkaExampleDst)));
                    udfso.setOwner(new Path(kafkaExampleDst), userHdfsName, datasetGroup);
                } catch (IOException ex) {
                    throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_TOUR_FILES_ERROR, Level.SEVERE, "project: " + project.getName(), ex.getMessage(), ex);
                }
                break;
            case ML:
                tourFilesDataset = Settings.HOPS_DL_TOUR_DATASET;
                datasetController.createDataset(user, project, tourFilesDataset, "sample training data for notebooks", Provenance.getDatasetProvCore(projectProvCore, Provenance.MLType.DATASET), false, DatasetAccessPermission.EDITABLE, dfso);
                String DLDataSrc = "/user/" + settings.getHdfsSuperUser() + "/" + Settings.HOPS_DEEP_LEARNING_TOUR_DATA + "/*";
                String DLDataDst = projectPath + Settings.HOPS_DL_TOUR_DATASET;
                String DLNotebooksSrc = "/user/" + settings.getHdfsSuperUser() + "/" + Settings.HOPS_DEEP_LEARNING_TOUR_NOTEBOOKS;
                String DLNotebooksDst = projectPath + Settings.HOPS_TOUR_DATASET_JUPYTER;
                try {
                    udfso.copyInHdfs(new Path(DLDataSrc), new Path(DLDataDst));
                    String datasetGroup = hdfsUsersController.getHdfsGroupName(project, Settings.HOPS_DL_TOUR_DATASET);
                    String userHdfsName = hdfsUsersController.getHdfsUserName(project, user);
                    Inode tourDs = inodeController.getInodeAtPath(DLDataDst);
                    datasetController.recChangeOwnershipAndPermission(new Path(DLDataDst), FsPermission.createImmutable(tourDs.getPermission()), userHdfsName, datasetGroup, dfso, udfso);
                    udfso.copyInHdfs(new Path(DLNotebooksSrc + "/*"), new Path(DLNotebooksDst));
                    datasetGroup = hdfsUsersController.getHdfsGroupName(project, Settings.HOPS_TOUR_DATASET_JUPYTER);
                    Inode jupyterDS = inodeController.getInodeAtPath(DLNotebooksDst);
                    datasetController.recChangeOwnershipAndPermission(new Path(DLNotebooksDst), FsPermission.createImmutable(jupyterDS.getPermission()), userHdfsName, datasetGroup, dfso, udfso);
                } catch (IOException ex) {
                    throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_TOUR_FILES_ERROR, Level.SEVERE, "project: " + project.getName(), ex.getMessage(), ex);
                }
                break;
            case FS:
                datasetController.createDataset(user, project, tourFilesDataset, "files for guide projects", Provenance.getDatasetProvCore(projectProvCore, Provenance.MLType.DATASET), false, DatasetAccessPermission.EDITABLE, dfso);
                // Get the JAR from /user/<super user>
                String featurestoreExampleJarSrc = "/user/" + settings.getSparkUser() + "/" + settings.getHopsExamplesFeaturestoreTourFilename();
                String featurestoreExampleJarDst = projectPath + tourFilesDataset + "/" + settings.getHopsExamplesFeaturestoreTourFilename();
                // Get the sample data and notebooks from /user/<super user>/featurestore_demo/
                String featurestoreExampleDataSrc = "/user/" + settings.getHdfsSuperUser() + "/" + Settings.HOPS_FEATURESTORE_TOUR_DATA + "/data";
                String featurestoreExampleDataDst = projectPath + tourFilesDataset;
                try {
                    // Move example .jar file to HDFS
                    udfso.copyInHdfs(new Path(featurestoreExampleJarSrc), new Path(featurestoreExampleJarDst));
                    String datasetGroup = hdfsUsersController.getHdfsGroupName(project, tourFilesDataset);
                    String userHdfsName = hdfsUsersController.getHdfsUserName(project, user);
                    udfso.setPermission(new Path(featurestoreExampleJarDst), udfso.getParentPermission(new Path(featurestoreExampleJarDst)));
                    udfso.setOwner(new Path(featurestoreExampleJarDst), userHdfsName, datasetGroup);
                    // Move example data and notebooks to HDFS
                    udfso.copyInHdfs(new Path(featurestoreExampleDataSrc), new Path(featurestoreExampleDataDst));
                    datasetGroup = hdfsUsersController.getHdfsGroupName(project, tourFilesDataset);
                    userHdfsName = hdfsUsersController.getHdfsUserName(project, user);
                    Inode featurestoreDataDst = inodeController.getInodeAtPath(featurestoreExampleDataDst);
                    datasetController.recChangeOwnershipAndPermission(new Path(featurestoreExampleDataDst), FsPermission.createImmutable(featurestoreDataDst.getPermission()), userHdfsName, datasetGroup, dfso, udfso);
                    // Move example notebooks to Jupyter dataset
                    String featurestoreExampleNotebooksSrc = "/user/" + settings.getHdfsSuperUser() + "/" + Settings.HOPS_FEATURESTORE_TOUR_DATA + "/notebooks";
                    String featurestoreExampleNotebooksDst = projectPath + Settings.HOPS_TOUR_DATASET_JUPYTER;
                    udfso.copyInHdfs(new Path(featurestoreExampleNotebooksSrc + "/*"), new Path(featurestoreExampleNotebooksDst));
                    datasetGroup = hdfsUsersController.getHdfsGroupName(project, Settings.HOPS_TOUR_DATASET_JUPYTER);
                    Inode featurestoreNotebooksDst = inodeController.getInodeAtPath(featurestoreExampleNotebooksDst);
                    datasetController.recChangeOwnershipAndPermission(new Path(featurestoreExampleNotebooksDst), FsPermission.createImmutable(featurestoreNotebooksDst.getPermission()), userHdfsName, datasetGroup, dfso, udfso);
                } catch (IOException ex) {
                    throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_TOUR_FILES_ERROR, Level.SEVERE, "project: " + project.getName(), ex.getMessage(), ex);
                }
                SparkJobConfiguration sparkJobConfiguration = new SparkJobConfiguration();
                sparkJobConfiguration.setAmQueue("default");
                sparkJobConfiguration.setAmMemory(1024);
                sparkJobConfiguration.setAmVCores(1);
                sparkJobConfiguration.setAppPath("hdfs://" + featurestoreExampleJarDst);
                sparkJobConfiguration.setMainClass(Settings.HOPS_FEATURESTORE_TOUR_JOB_CLASS);
                sparkJobConfiguration.setDefaultArgs("--input TestJob/data");
                sparkJobConfiguration.setExecutorInstances(1);
                sparkJobConfiguration.setExecutorCores(1);
                sparkJobConfiguration.setExecutorMemory(2024);
                sparkJobConfiguration.setExecutorGpus(0);
                sparkJobConfiguration.setDynamicAllocationEnabled(true);
                sparkJobConfiguration.setDynamicAllocationMinExecutors(1);
                sparkJobConfiguration.setDynamicAllocationMaxExecutors(3);
                sparkJobConfiguration.setDynamicAllocationInitialExecutors(1);
                sparkJobConfiguration.setAppName(Settings.HOPS_FEATURESTORE_TOUR_JOB_NAME);
                sparkJobConfiguration.setLocalResources(new LocalResourceDTO[0]);
                Jobs job = jobController.putJob(user, project, null, sparkJobConfiguration);
                activityFacade.persistActivity(ActivityFacade.CREATED_JOB + job.getName(), project, user, ActivityFlag.SERVICE);
                executionController.start(job, Settings.HOPS_FEATURESTORE_TOUR_JOB_INPUT_PARAM + tourFilesDataset + "/data", user);
                activityFacade.persistActivity(ActivityFacade.RAN_JOB + job.getName(), project, user, ActivityFlag.SERVICE);
                break;
            default:
                break;
        }
    }
    return tourFilesDataset;
}

Example 19 with Jobs

use of io.hops.hopsworks.persistence.entity.jobs.description.Jobs in project hopsworks by logicalclocks.

the class TestAlertManagerConfigTimer method createAlertReceiver.

private AlertReceiver createAlertReceiver(Integer id, Receiver receiver, AlertType alertType) throws JsonProcessingException {
    AlertReceiver alertReceiver = new AlertReceiver();
    alertReceiver.setId(id);
    alertReceiver.setName(receiver.getName());
    alertReceiver.setConfig(toJson(receiver));
    Project project = new Project("project1");
    Jobs job = new Jobs();
    job.setProject(project);
    job.setName("job1");
    Featurestore featurestore = new Featurestore();
    featurestore.setProject(project);
    Featuregroup featuregroup = new Featuregroup();
    featuregroup.setFeaturestore(featurestore);
    featuregroup.setName("fg1");
    List<JobAlert> jobAlerts = new ArrayList<>();
    jobAlerts.add(new JobAlert(random.nextInt(1000), JobAlertStatus.FAILED, alertType, AlertSeverity.CRITICAL, new Date(), job, alertReceiver));
    jobAlerts.add(new JobAlert(random.nextInt(1000), JobAlertStatus.KILLED, alertType, AlertSeverity.WARNING, new Date(), job, alertReceiver));
    alertReceiver.setJobAlertCollection(jobAlerts);
    List<FeatureGroupAlert> featureGroupAlerts = new ArrayList<>();
    featureGroupAlerts.add(new FeatureGroupAlert(random.nextInt(1000), ValidationRuleAlertStatus.FAILURE, alertType, AlertSeverity.CRITICAL, new Date(), featuregroup, alertReceiver));
    featureGroupAlerts.add(new FeatureGroupAlert(random.nextInt(1000), ValidationRuleAlertStatus.WARNING, alertType, AlertSeverity.WARNING, new Date(), featuregroup, alertReceiver));
    alertReceiver.setFeatureGroupAlertCollection(featureGroupAlerts);
    List<ProjectServiceAlert> projectServiceAlerts = new ArrayList<>();
    projectServiceAlerts.add(new ProjectServiceAlert(random.nextInt(1000), ProjectServiceEnum.FEATURESTORE, ProjectServiceAlertStatus.VALIDATION_FAILURE, alertType, AlertSeverity.WARNING, new Date(), project, alertReceiver));
    projectServiceAlerts.add(new ProjectServiceAlert(random.nextInt(1000), ProjectServiceEnum.JOBS, ProjectServiceAlertStatus.JOB_FAILED, alertType, AlertSeverity.WARNING, new Date(), project, alertReceiver));
    alertReceiver.setProjectServiceAlertCollection(projectServiceAlerts);
    return alertReceiver;
}

Example 20 with Jobs

use of io.hops.hopsworks.persistence.entity.jobs.description.Jobs in project hopsworks by logicalclocks.

the class ExperimentsController method versionProgram.

public String versionProgram(Project project, Users user, String jobName, String kernelId, String mlId) throws JobException, ServiceException {
    if (!Strings.isNullOrEmpty(jobName)) {
        // experiment in job
        Jobs experimentJob = jobController.getJob(project, jobName);
        SparkJobConfiguration sparkJobConf = (SparkJobConfiguration) experimentJob.getJobConfig();
        String suffix = sparkJobConf.getAppPath().substring(sparkJobConf.getAppPath().lastIndexOf("."));
        String relativePath = Settings.HOPS_EXPERIMENTS_DATASET + "/" + mlId + "/program" + suffix;
        jobController.versionProgram(sparkJobConf, project, user, new Path(Utils.getProjectPath(project.getName()) + relativePath));
        return relativePath;
    } else if (!Strings.isNullOrEmpty(kernelId)) {
        // experiment in jupyter
        String relativePath = Settings.HOPS_EXPERIMENTS_DATASET + "/" + mlId + "/program.ipynb";
        Path path = new Path(Utils.getProjectPath(project.getName()) + "/" + relativePath);
        jupyterController.versionProgram(project, user, kernelId, path);
        return relativePath;
    }
    return null;
}