Examples with ProjectServiceEnum io.hops.hopsworks.persistence.entity.project.service.ProjectServiceEnum used on opensource projects

Example 1 with ProjectServiceEnum

use of io.hops.hopsworks.persistence.entity.project.service.ProjectServiceEnum in project hopsworks by logicalclocks.

the class ProjectController method createProject.

/**
 * Creates a new project(project), the related DIR, the different services in
 * the project, and the master of the
 * project.
 * <p>
 * This needs to be an atomic operation (all or nothing) REQUIRES_NEW will
 * make sure a new transaction is created even
 * if this method is called from within a transaction.
 *
 * @param projectDTO
 * @param owner
 * @param sessionId
 * @return
 */
public Project createProject(ProjectDTO projectDTO, Users owner, String sessionId) throws DatasetException, GenericException, KafkaException, ProjectException, UserException, HopsSecurityException, ServiceException, FeaturestoreException, ElasticException, SchemaException, IOException {
    Long startTime = System.currentTimeMillis();
    // check that the project name is ok
    String projectName = projectDTO.getProjectName();
    FolderNameValidator.isValidProjectName(projectUtils, projectName);
    List<ProjectServiceEnum> projectServices = new ArrayList<>();
    if (projectDTO.getServices() != null) {
        for (String s : projectDTO.getServices()) {
            ProjectServiceEnum se = ProjectServiceEnum.valueOf(s.toUpperCase());
            projectServices.add(se);
        }
    }
    LOGGER.log(Level.FINE, () -> "PROJECT CREATION TIME. Step 1: " + (System.currentTimeMillis() - startTime));
    DistributedFileSystemOps dfso = null;
    Project project = null;
    try {
        dfso = dfs.getDfsOps();
        /*
       * create a project in the database
       * if the creation go through it means that there is no other project with
       * the same name.
       * this project creation act like a lock, no other project can be created
       * with the same name
       * until this project is removed from the database
       */
        try {
            project = createProject(projectName, owner, projectDTO.getDescription(), dfso);
        } catch (EJBException ex) {
            LOGGER.log(Level.WARNING, null, ex);
            Path dummy = new Path("/tmp/" + projectName);
            try {
                dfso.rm(dummy, true);
            } catch (IOException e) {
                LOGGER.log(Level.SEVERE, null, e);
            }
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_EXISTS, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 2 (hdfs): {0}", System.currentTimeMillis() - startTime);
        verifyProject(project, dfso, sessionId);
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 3 (verify): {0}", System.currentTimeMillis() - startTime);
        // Run the handlers.
        try {
            ProjectHandler.runProjectPreCreateHandlers(projectHandlers, project);
        } catch (ProjectException ex) {
            cleanup(project, sessionId, null, true, owner);
            throw ex;
        }
        List<Future<?>> projectCreationFutures = new ArrayList<>();
        // This is an async call
        try {
            projectCreationFutures.add(certificatesController.generateCertificates(project, owner));
        } catch (Exception ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new HopsSecurityException(RESTCodes.SecurityErrorCode.CERT_CREATION_ERROR, Level.SEVERE, "project: " + project.getName() + "owner: " + owner.getUsername(), ex.getMessage(), ex);
        }
        String username = hdfsUsersController.getHdfsUserName(project, owner);
        if (username == null || username.isEmpty()) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new UserException(RESTCodes.UserErrorCode.USER_WAS_NOT_FOUND, Level.SEVERE, "project: " + project.getName() + "owner: " + owner.getUsername());
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 4 (certs): {0}", System.currentTimeMillis() - startTime);
        // all the verifications have passed, we can now create the project
        // create the project folder
        ProvTypeDTO provType = settings.getProvType().dto;
        try {
            mkProjectDIR(projectName, dfso);
            fsProvController.updateProjectProvType(project, provType, dfso);
        } catch (IOException | EJBException | ProvenanceException ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_FOLDER_NOT_CREATED, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 5 (folders): {0}", System.currentTimeMillis() - startTime);
        // update the project with the project folder inode
        try {
            setProjectInode(project, dfso);
        } catch (IOException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_INODE_CREATION_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 6 (inodes): {0}", System.currentTimeMillis() - startTime);
        // set payment and quotas
        try {
            setProjectOwnerAndQuotas(project, dfso, owner);
        } catch (IOException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new ProjectException(RESTCodes.ProjectErrorCode.QUOTA_ERROR, Level.SEVERE, "project: " + project.getName(), ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 7 (quotas): {0}", System.currentTimeMillis() - startTime);
        try {
            hdfsUsersController.addProjectFolderOwner(project, dfso);
            createProjectLogResources(owner, project, dfso);
        } catch (IOException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_SET_PERMISSIONS_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 8 (logs): {0}", System.currentTimeMillis() - startTime);
        // inconsistencies
        try {
            elasticController.deleteProjectIndices(project);
            elasticController.deleteProjectSavedObjects(projectName);
            LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 9 (elastic cleanup): {0}", System.currentTimeMillis() - startTime);
        } catch (ElasticException ex) {
            LOGGER.log(Level.FINE, "Error while cleaning old project indices", ex);
        }
        logProject(project, OperationType.Add);
        // enable services
        for (ProjectServiceEnum service : projectServices) {
            try {
                projectCreationFutures.addAll(addService(project, service, owner, dfso, provType));
            } catch (RESTException | IOException ex) {
                cleanup(project, sessionId, projectCreationFutures);
                throw ex;
            }
        }
        try {
            for (Future f : projectCreationFutures) {
                if (f != null) {
                    f.get();
                }
            }
        } catch (InterruptedException | ExecutionException ex) {
            LOGGER.log(Level.SEVERE, "Error while waiting for the certificate generation thread to finish. Will try to " + "cleanup...", ex);
            cleanup(project, sessionId, projectCreationFutures);
            throw new HopsSecurityException(RESTCodes.SecurityErrorCode.CERT_CREATION_ERROR, Level.SEVERE);
        }
        // Run the handlers.
        try {
            ProjectHandler.runProjectPostCreateHandlers(projectHandlers, project);
        } catch (ProjectException ex) {
            cleanup(project, sessionId, projectCreationFutures);
            throw ex;
        }
        try {
            project = environmentController.createEnv(project, owner);
        } catch (PythonException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_ANACONDA_ENABLE_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 10 (env): {0}", System.currentTimeMillis() - startTime);
        return project;
    } finally {
        if (dfso != null) {
            dfso.close();
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 11 (close): {0}", System.currentTimeMillis() - startTime);
    }
}

Example 2 with ProjectServiceEnum

use of io.hops.hopsworks.persistence.entity.project.service.ProjectServiceEnum in project hopsworks by logicalclocks.

the class ProjectController method getProjectByID.

/**
 * Project info as data transfer object that can be sent to the user.
 *
 * @param projectID of the project
 * @return project DTO that contains team members and services
 */
public ProjectDTO getProjectByID(Integer projectID) throws ProjectException {
    Project project = projectFacade.find(projectID);
    if (project == null) {
        throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + projectID);
    }
    // find the project as an inode from hops database
    Inode inode = inodeController.getInodeAtPath(Utils.getProjectPath(project.getName()));
    List<ProjectTeam> projectTeam = projectTeamFacade.findMembersByProject(project);
    List<ProjectServiceEnum> projectServices = projectServicesFacade.findEnabledServicesForProject(project);
    List<String> services = new ArrayList<>();
    for (ProjectServiceEnum s : projectServices) {
        services.add(s.toString());
    }
    QuotasDTO quotas = getQuotasInternal(project);
    return new ProjectDTO(project, inode.getId(), services, projectTeam, quotas, settings.getHopsExamplesSparkFilename(), projectUtils.dockerImageIsPreinstalled(project.getDockerImage()), projectUtils.isOldDockerImage(project.getDockerImage()));
}

Example 3 with ProjectServiceEnum

use of io.hops.hopsworks.persistence.entity.project.service.ProjectServiceEnum in project hopsworks by logicalclocks.

the class ProjectServiceFacade method persistServicesForProject.

// TODO: write tests for this
public void persistServicesForProject(Project project, ProjectServiceEnum[] services) {
    // TODO: use copy instead
    List<ProjectServices> newSrvs = new ArrayList<>(services.length);
    List<ProjectServices> toPersist = new ArrayList<>(services.length);
    for (ProjectServiceEnum sse : services) {
        ProjectServices c = new ProjectServices(project, sse);
        newSrvs.add(c);
        toPersist.add(c);
    }
    List<ProjectServices> current = getAllProjectServicesForProject(project);
    toPersist.removeAll(current);
    current.removeAll(newSrvs);
    for (ProjectServices se : toPersist) {
        em.persist(se);
    }
    for (ProjectServices se : current) {
        em.remove(se);
    }
}

Example 4 with ProjectServiceEnum

use of io.hops.hopsworks.persistence.entity.project.service.ProjectServiceEnum in project hopsworks by logicalclocks.

the class ProjectService method updateProject.

@PUT
@Path("{projectId}")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER })
public Response updateProject(ProjectDTO projectDTO, @PathParam("projectId") Integer id, @Context SecurityContext sc) throws ProjectException, DatasetException, HopsSecurityException, ServiceException, FeaturestoreException, ElasticException, SchemaException, KafkaException, ProvenanceException, IOException, UserException {
    RESTApiJsonResponse json = new RESTApiJsonResponse();
    Users user = jWTHelper.getUserPrincipal(sc);
    Project project = projectController.findProjectById(id);
    boolean updated = false;
    if (projectController.updateProjectDescription(project, projectDTO.getDescription(), user)) {
        json.setSuccessMessage(ResponseMessages.PROJECT_DESCRIPTION_CHANGED);
        updated = true;
    }
    if (projectController.updateProjectRetention(project, projectDTO.getRetentionPeriod(), user)) {
        json.setSuccessMessage(json.getSuccessMessage() + "\n" + ResponseMessages.PROJECT_RETENTON_CHANGED);
        updated = true;
    }
    if (!projectDTO.getServices().isEmpty()) {
        // Create dfso here and pass them to the different controllers
        DistributedFileSystemOps dfso = dfs.getDfsOps();
        DistributedFileSystemOps udfso = dfs.getDfsOps(hdfsUsersBean.getHdfsUserName(project, user));
        for (String s : projectDTO.getServices()) {
            ProjectServiceEnum se = null;
            se = ProjectServiceEnum.valueOf(s.toUpperCase());
            ProvTypeDTO projectMetaStatus = fsProvenanceController.getProjectProvType(user, project);
            List<Future<?>> serviceFutureList = projectController.addService(project, se, user, dfso, udfso, projectMetaStatus);
            if (serviceFutureList != null) {
                // Wait for the futures
                for (Future f : serviceFutureList) {
                    try {
                        f.get();
                    } catch (InterruptedException | ExecutionException e) {
                        throw new ServiceException(RESTCodes.ServiceErrorCode.SERVICE_GENERIC_ERROR, Level.SEVERE, "service: " + s, e.getMessage(), e);
                    }
                }
                // Service successfully enabled
                json.setSuccessMessage(json.getSuccessMessage() + "\n" + ResponseMessages.PROJECT_SERVICE_ADDED + s);
                updated = true;
            }
        }
        // close dfsos
        if (dfso != null) {
            dfso.close();
        }
        if (udfso != null) {
            dfs.closeDfsClient(udfso);
        }
    }
    if (!updated) {
        json.setSuccessMessage(ResponseMessages.NOTHING_TO_UPDATE);
    }
    return noCacheResponse.getNoCacheResponseBuilder(Response.Status.CREATED).entity(json).build();
}

Example 5 with ProjectServiceEnum

use of io.hops.hopsworks.persistence.entity.project.service.ProjectServiceEnum in project hopsworks by logicalclocks.

the class ProjectController method getProjectByName.

/**
 * Project info as data transfer object that can be sent to the user.
 *
 * @param name
 * @return project DTO that contains team members and services
 */
public ProjectDTO getProjectByName(String name) throws ProjectException {
    // find the project entity from hopsworks database
    Project project = projectFacade.findByName(name);
    if (project == null) {
        throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "project: " + name);
    }
    // find the project as an inode from hops database
    Inode inode = inodeController.getInodeAtPath(Utils.getProjectPath(name));
    List<ProjectTeam> projectTeam = projectTeamFacade.findMembersByProject(project);
    List<ProjectServiceEnum> projectServices = projectServicesFacade.findEnabledServicesForProject(project);
    List<String> services = new ArrayList<>();
    for (ProjectServiceEnum s : projectServices) {
        services.add(s.toString());
    }
    Inode parent;
    List<InodeView> kids = new ArrayList<>();
    Collection<Dataset> dsInProject = project.getDatasetCollection();
    Collection<DatasetSharedWith> dsSharedWithProject = project.getDatasetSharedWithCollection();
    for (Dataset ds : dsInProject) {
        parent = inodes.findParent(ds.getInode());
        kids.add(new InodeView(parent, ds, inodeController.getPath(ds.getInode())));
    }
    for (DatasetSharedWith ds : dsSharedWithProject) {
        parent = inodes.findParent(ds.getDataset().getInode());
        kids.add(new InodeView(parent, ds, inodeController.getPath(ds.getDataset().getInode())));
    }
    // send the project back to client
    return new ProjectDTO(project, inode.getId(), services, projectTeam, kids, projectUtils.dockerImageIsPreinstalled(project.getDockerImage()), projectUtils.isOldDockerImage(project.getDockerImage()));
}