Examples with DefaultJobConfiguration io.hops.hopsworks.persistence.entity.project.jobs.DefaultJobConfiguration used on opensource projects

Example 1 with DefaultJobConfiguration

use of io.hops.hopsworks.persistence.entity.project.jobs.DefaultJobConfiguration in project hopsworks by logicalclocks.

the class DefaultJobConfigurationResource method getByType.

@ApiOperation(value = "Get the default job configuration by JobType", response = DefaultJobConfigurationDTO.class)
@GET
@Path("{type}")
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response getByType(@PathParam("type") JobType jobType, @Context UriInfo uriInfo, @Context SecurityContext sc) throws ProjectException {
    ResourceRequest resourceRequest = new ResourceRequest(ResourceRequest.Name.JOBCONFIG);
    DefaultJobConfiguration defaultJobConfiguration = projectController.getProjectDefaultJobConfiguration(project, jobType);
    if (defaultJobConfiguration == null) {
        throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_DEFAULT_JOB_CONFIG_NOT_FOUND, Level.FINEST);
    } else {
        DefaultJobConfigurationDTO defaultJobConfigurationDTO = this.defaultJobConfigurationBuilder.build(uriInfo, resourceRequest, defaultJobConfiguration, jobType);
        return Response.ok(defaultJobConfigurationDTO).build();
    }
}

Example 2 with DefaultJobConfiguration

use of io.hops.hopsworks.persistence.entity.project.jobs.DefaultJobConfiguration in project hopsworks by logicalclocks.

the class DefaultJobConfigurationFacade method findByProject.

public CollectionInfo findByProject(Integer offset, Integer limit, Set<? extends AbstractFacade.FilterBy> filter, Set<? extends AbstractFacade.SortBy> sort, Project project) {
    String queryStr = buildQuery("SELECT c FROM DefaultJobConfiguration c ", filter, sort, "c.project = :project");
    String queryCountStr = buildQuery("SELECT COUNT(DISTINCT c.defaultJobConfigurationPK.type) FROM DefaultJobConfiguration c ", filter, sort, "c.project = :project ");
    Query query = em.createQuery(queryStr, DefaultJobConfiguration.class).setParameter("project", project);
    Query queryCount = em.createQuery(queryCountStr, DefaultJobConfiguration.class).setParameter("project", project);
    return findAll(offset, limit, filter, query, queryCount);
}

Example 3 with DefaultJobConfiguration

use of io.hops.hopsworks.persistence.entity.project.jobs.DefaultJobConfiguration in project hopsworks by logicalclocks.

the class JobController method getConfiguration.

@TransactionAttribute(TransactionAttributeType.NEVER)
public JobConfiguration getConfiguration(Project project, JobType jobType, boolean useDefaultConfig) {
    Optional<DefaultJobConfiguration> defaultConfig;
    if (jobType.equals(JobType.SPARK) || jobType.equals(JobType.PYSPARK)) {
        /**
         * The Spark and PySpark configuration is stored in the same configuration entry in the
         * database for DefaultJobConfiguration. Namely in a PySpark configuration. We infer the JobType based on if
         * you set a .jar or .py. However when creating the DefaultJobConfiguration, as part of the PK the JobType
         * needs to be set. So for now PySpark/Spark shares the same configuration.
         */
        defaultConfig = project.getDefaultJobConfigurationCollection().stream().filter(conf -> conf.getDefaultJobConfigurationPK().getType().equals(JobType.PYSPARK)).findFirst();
        defaultConfig.ifPresent(defaultJobConfiguration -> ((SparkJobConfiguration) defaultJobConfiguration.getJobConfig()).setMainClass(null));
    } else {
        defaultConfig = project.getDefaultJobConfigurationCollection().stream().filter(conf -> conf.getDefaultJobConfigurationPK().getType().equals(jobType)).findFirst();
    }
    if (defaultConfig.isPresent()) {
        return defaultConfig.get().getJobConfig();
    } else if (useDefaultConfig) {
        switch(jobType) {
            case SPARK:
            case PYSPARK:
                return new SparkJobConfiguration();
            case FLINK:
                return new FlinkJobConfiguration();
            default:
                throw new IllegalArgumentException("Job type not supported: " + jobType);
        }
    } else {
        return null;
    }
}

Example 4 with DefaultJobConfiguration

use of io.hops.hopsworks.persistence.entity.project.jobs.DefaultJobConfiguration in project hopsworks by logicalclocks.

the class DefaultJobConfigurationFacade method createOrUpdate.

public DefaultJobConfiguration createOrUpdate(Project project, JobConfiguration jobConfiguration, JobType jobType, DefaultJobConfiguration currentConfig) throws ProjectException {
    if (jobConfiguration instanceof SparkJobConfiguration) {
        ((SparkJobConfiguration) jobConfiguration).setMainClass(Settings.SPARK_PY_MAINCLASS);
        jobType = JobType.PYSPARK;
    }
    // create
    if (currentConfig == null) {
        currentConfig = new DefaultJobConfiguration();
        DefaultJobConfigurationPK pk = new DefaultJobConfigurationPK();
        pk.setProjectId(project.getId());
        pk.setType(jobType);
        currentConfig.setDefaultJobConfigurationPK(pk);
        currentConfig.setJobConfig(jobConfiguration);
        project.getDefaultJobConfigurationCollection().add(currentConfig);
        em.merge(project);
        return currentConfig;
    // update
    } else {
        for (DefaultJobConfiguration dc : project.getDefaultJobConfigurationCollection()) {
            if (dc.getDefaultJobConfigurationPK().getType().equals(jobType)) {
                dc.setJobConfig(jobConfiguration);
                em.merge(project);
                return dc;
            }
        }
        throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_DEFAULT_JOB_CONFIG_NOT_FOUND, Level.FINEST);
    }
}

Example 5 with DefaultJobConfiguration

use of io.hops.hopsworks.persistence.entity.project.jobs.DefaultJobConfiguration in project hopsworks by logicalclocks.

the class DefaultJobConfigurationResource method put.

@ApiOperation(value = "Create or update the default job configuration", response = DefaultJobConfigurationDTO.class)
@PUT
@Path("{type}")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API, Audience.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.JOB }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response put(@ApiParam(value = "Job configuration", required = true) JobConfiguration config, @PathParam("type") JobType type, @Context UriInfo uriInfo, @Context SecurityContext sc) throws ProjectException {
    Response.Status status = Response.Status.CREATED;
    HopsUtils.validateJobConfigurationType(config, type);
    DefaultJobConfiguration currentConfig = projectController.getProjectDefaultJobConfiguration(project, type);
    if (currentConfig != null) {
        status = Response.Status.OK;
    }
    DefaultJobConfiguration defaultConfig = projectController.createOrUpdateDefaultJobConfig(this.project, config, type, currentConfig);
    DefaultJobConfigurationDTO defaultJobConfigurationDTO = this.defaultJobConfigurationBuilder.build(uriInfo, new ResourceRequest(ResourceRequest.Name.JOBCONFIG), defaultConfig, type);
    UriBuilder builder = uriInfo.getAbsolutePathBuilder();
    if (status == Response.Status.CREATED) {
        return Response.created(builder.build()).entity(defaultJobConfigurationDTO).build();
    } else {
        return Response.ok(builder.build()).entity(defaultJobConfigurationDTO).build();
    }
}