Examples with ProjectException io.hops.hopsworks.exceptions.ProjectException used on opensource projects

Example 26 with ProjectException

use of io.hops.hopsworks.exceptions.ProjectException in project hopsworks by logicalclocks.

the class SecretsController method getShared.

/**
 * Gets a decrypted shared secret depending on its Visibility. It will throw an exception
 * if the Visibility was set to PRIVATE or the caller is not member of the Project
 * the Secret is shared with.
 *
 * @param caller The user who requested the Secret
 * @param ownerUser the user owner of the secret
 * @param secretName Identifier of the Secret
 * @return The decrypted Secret
 * @throws UserException
 * @throws ServiceException
 * @throws ProjectException
 */
public SecretPlaintext getShared(Users caller, Users ownerUser, String secretName) throws UserException, ServiceException, ProjectException {
    checkIfUserIsNull(caller);
    checkIfNameIsNullOrEmpty(secretName);
    checkIfUserIsNull(ownerUser);
    Secret storedSecret = secretsFacade.findById(new SecretId(ownerUser.getUid(), secretName));
    checkIfSecretIsNull(storedSecret, secretName, ownerUser);
    if (storedSecret.getVisibilityType() == null || storedSecret.getVisibilityType().equals(VisibilityType.PRIVATE)) {
        throw new UserException(RESTCodes.UserErrorCode.ACCESS_CONTROL, Level.FINE, "Secret is Private", "User " + caller.getUsername() + " requested PRIVATE secret <" + ownerUser.getUid() + ", " + secretName + ">");
    }
    Integer projectId = storedSecret.getProjectIdScope();
    if (projectId == null) {
        throw new ServiceException(RESTCodes.ServiceErrorCode.SERVICE_GENERIC_ERROR, Level.WARNING, "Visibility's Project ID is empty", "Secret " + secretName + " visibility is PROJECT but Project ID is null");
    }
    Project project = projectFacade.find(projectId);
    if (project == null) {
        throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "Project with ID " + projectId + " does not exist!", "User " + caller.getUsername() + " requested shared Secret " + secretName + " but Project with ID " + projectId + "does not exist");
    }
    // Check if caller is member of the Project
    for (ProjectTeam projectTeam : project.getProjectTeamCollection()) {
        if (caller.getUid().equals(projectTeam.getUser().getUid())) {
            try {
                return decrypt(ownerUser, storedSecret);
            } catch (IOException | GeneralSecurityException ex) {
                throw new UserException(RESTCodes.UserErrorCode.SECRET_ENCRYPTION_ERROR, Level.SEVERE, "Error decrypting Secret", "Could not decrypt Secret " + secretName, ex);
            }
        }
    }
    // Check if caller is a member of some shared project
    throw new UserException(RESTCodes.UserErrorCode.ACCESS_CONTROL, Level.FINE, "Not authorized to access Secret " + secretName, "User " + caller.getUsername() + " tried to access shared Secret " + secretName + " but they are not member of Project " + project.getName());
}

Example 27 with ProjectException

use of io.hops.hopsworks.exceptions.ProjectException in project hopsworks by logicalclocks.

the class DockerRegistryMngrImpl method gc.

@Override
public void gc() throws IOException, ServiceException, ProjectException {
    // 1. Get all conda commands of type REMOVE. Should be only 1 REMOVE per project
    final List<CondaCommands> condaCommandsRemove = condaCommandFacade.findByStatusAndCondaOp(CondaStatus.NEW, CondaOp.REMOVE);
    LOG.log(Level.FINE, "condaCommandsRemove: " + condaCommandsRemove);
    try {
        for (CondaCommands cc : condaCommandsRemove) {
            // We do not want to remove the base image! Get arguments from command as project may have already been deleted.
            String projectDockerImage = cc.getArg();
            String projectDockerRepoName = projectUtils.getProjectDockerRepoName(projectDockerImage);
            if (!projectUtils.dockerImageIsPreinstalled(projectDockerImage)) {
                try {
                    // 1. Get and delete all the tags for each repository(project)
                    List<String> projectTags = deleteProjectImagesOnRegistry(projectDockerImage);
                    for (String tag : projectTags) {
                        // Issue system command (kagent) to remove docker image from each host's docker daemon
                        dockerImagesGC(projectUtils.getRegistryURL() + "/" + projectDockerRepoName + ":" + tag);
                    }
                } catch (Exception ex) {
                    LOG.log(Level.WARNING, "Could not complete docker registry cleanup for: " + cc, ex);
                    try {
                        commandsController.updateCondaCommandStatus(cc.getId(), CondaStatus.FAILED, cc.getArg(), cc.getOp(), "Could not complete docker registry cleanup: " + ex.getMessage());
                    } catch (ServiceException | ProjectException e) {
                        LOG.log(Level.WARNING, "Could not change conda command status to NEW.", e);
                    }
                }
            }
            commandsController.updateCondaCommandStatus(cc.getId(), CondaStatus.SUCCESS, cc.getArg(), cc.getOp());
        }
    } finally {
        // Run docker gc in cli
        if (!condaCommandsRemove.isEmpty()) {
            runRegistryGC();
        }
    }
}

Example 28 with ProjectException

use of io.hops.hopsworks.exceptions.ProjectException in project hopsworks by logicalclocks.

the class LibraryInstaller method exportEnvironment.

public void exportEnvironment(CondaCommands cc) throws IOException, ServiceException, ServiceDiscoveryException, ProjectException {
    Project project = projectFacade.findById(cc.getProjectId().getId()).orElseThrow(() -> new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + cc.getProjectId().getId()));
    exportEnvironment(project, cc.getUserId(), cc.getArg());
}

Example 29 with ProjectException

use of io.hops.hopsworks.exceptions.ProjectException in project hopsworks by logicalclocks.

the class LibraryInstaller method createNewImage.

private void createNewImage(CondaCommands cc) throws IOException, ServiceDiscoveryException {
    File baseDir = new File("/tmp/docker/" + cc.getProjectId().getName());
    baseDir.mkdirs();
    try {
        File home = new File(System.getProperty("user.home"));
        File condarc = new File(home, ".condarc");
        File pip = new File(home, ".pip");
        FileUtils.copyFileToDirectory(condarc, baseDir);
        FileUtils.copyDirectoryToDirectory(pip, baseDir);
        File dockerFile = new File(baseDir, "dockerFile_" + cc.getProjectId().getName());
        try (BufferedWriter writer = new BufferedWriter(new FileWriter(dockerFile))) {
            String baseImage = projectUtils.getFullBaseImageName();
            // image that does not contain a python environment
            if (!Strings.isNullOrEmpty(cc.getEnvironmentFile()) && !cc.getInstallJupyter()) {
                baseImage = baseImage.replace(settings.getBaseDockerImagePythonName(), settings.getBaseNonPythonDockerImage());
            }
            writer.write("# syntax=docker/dockerfile:experimental");
            writer.newLine();
            writer.write("FROM " + baseImage);
            writer.newLine();
            // copy it in the image, create the env and delete it
            if (!Strings.isNullOrEmpty(cc.getEnvironmentFile())) {
                writer.write("RUN rm -f /root/.condarc");
                writer.newLine();
                // Materialize IMPORT FILE
                String environmentFilePath = cc.getEnvironmentFile();
                String environmentFile = FilenameUtils.getName(environmentFilePath);
                writer.write("COPY .condarc .pip " + environmentFile + " /root/");
                writer.newLine();
                copyCondaArtifactToLocal(environmentFilePath, baseDir + File.separator + environmentFile);
                if (environmentFilePath.endsWith(".yml")) {
                    if (cc.getInstallJupyter()) {
                        writer.write("RUN conda env update -f /root/" + environmentFile + " -n " + settings.getCurrentCondaEnvironment());
                    } else {
                        writer.write("RUN conda env create -f /root/" + environmentFile + " -p " + anaconda_project_dir);
                    }
                } else if (environmentFilePath.endsWith("/requirements.txt")) {
                    if (cc.getInstallJupyter()) {
                        writer.write("RUN pip install -r /root/" + environmentFile);
                    } else {
                        writer.write("RUN conda create -y -p " + anaconda_project_dir + " python=" + settings.getDockerBaseImagePythonVersion() + " && pip install -r /root/" + environmentFile);
                    }
                }
                writer.write(" && " + getCleanupCommand() + " && " + anaconda_dir + "/bin/conda list -n " + settings.getCurrentCondaEnvironment());
            }
        }
        Project project = projectFacade.findById(cc.getProjectId().getId()).orElseThrow(() -> new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + cc.getProjectId().getId()));
        String initialDockerImage = projectUtils.getInitialDockerImageName(project);
        LOG.log(Level.FINEST, "project-initialDockerImage:" + initialDockerImage);
        ProcessDescriptor processDescriptor = new ProcessDescriptor.Builder().addCommand("/usr/bin/sudo").addCommand(prog).addCommand("create").addCommand(dockerFile.getAbsolutePath()).addCommand(projectUtils.getRegistryURL() + "/" + initialDockerImage).redirectErrorStream(true).setCurrentWorkingDirectory(baseDir).setWaitTimeout(1, TimeUnit.HOURS).build();
        ProcessResult processResult = osProcessExecutor.execute(processDescriptor);
        if (processResult.getExitCode() != 0) {
            String errorMsg = "Could not create the docker image. Exit code: " + processResult.getExitCode() + " out: " + processResult.getStdout() + "\n err: " + processResult.getStderr() + "||\n";
            throw new IOException(errorMsg);
        } else {
            project = projectFacade.findById(cc.getProjectId().getId()).orElseThrow(() -> new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + cc.getProjectId().getId()));
            project.setDockerImage(initialDockerImage);
            project = projectFacade.update(project);
            projectFacade.flushEm();
            setPipConflicts(project);
            environmentController.updateInstalledDependencies(project);
            exportEnvironment(project, cc.getUserId(), Utils.getProjectPath(project.getName()) + Settings.PROJECT_PYTHON_ENVIRONMENT_FILE);
        }
    } catch (ServiceException | ProjectException | PythonException e) {
        LOG.log(Level.SEVERE, "Failed to persist python deps", e);
    } finally {
        FileUtils.deleteDirectory(baseDir);
    }
}

Example 30 with ProjectException

use of io.hops.hopsworks.exceptions.ProjectException in project hopsworks by logicalclocks.

the class ProjectController method addMember.

public boolean addMember(ProjectTeam projectTeam, Project project, Users newMember, Users owner, DistributedFileSystemOps dfso) throws UserException, KafkaException, ProjectException, FeaturestoreException, IOException {
    if (projectTeam.getTeamRole() == null || (!projectTeam.getTeamRole().equals(ProjectRoleTypes.DATA_SCIENTIST.getRole()) && !projectTeam.getTeamRole().equals(ProjectRoleTypes.DATA_OWNER.getRole()))) {
        projectTeam.setTeamRole(ProjectRoleTypes.DATA_SCIENTIST.getRole());
    }
    projectTeam.setTimestamp(new Date());
    if (newMember != null && !projectTeamFacade.isUserMemberOfProject(project, newMember)) {
        // this makes sure that the member is added to the project sent as the
        // first param b/c the security check was made on the parameter sent as path.
        projectTeam.getProjectTeamPK().setProjectId(project.getId());
        projectTeam.setProject(project);
        projectTeam.setUser(newMember);
        project.getProjectTeamCollection().add(projectTeam);
        projectFacade.update(project);
        hdfsUsersController.addNewProjectMember(projectTeam, dfso);
        // Add user to kafka topics ACLs by default
        if (projectServicesFacade.isServiceEnabledForProject(project, ProjectServiceEnum.KAFKA)) {
            kafkaController.addProjectMemberToTopics(project, newMember.getEmail());
        }
        // if online-featurestore service is enabled in the project, give new member access to it
        if (projectServiceFacade.isServiceEnabledForProject(project, ProjectServiceEnum.FEATURESTORE) && settings.isOnlineFeaturestore()) {
            Featurestore featurestore = featurestoreController.getProjectFeaturestore(project);
            onlineFeaturestoreController.createDatabaseUser(projectTeam.getUser(), featurestore, projectTeam.getTeamRole());
        }
        // TODO: This should now be a REST call
        Future<CertificatesController.CertsResult> certsResultFuture = null;
        try {
            certsResultFuture = certificatesController.generateCertificates(project, newMember);
            certsResultFuture.get();
        } catch (Exception ex) {
            try {
                if (certsResultFuture != null) {
                    certsResultFuture.get();
                }
                certificatesController.revokeUserSpecificCertificates(project, newMember);
            } catch (IOException | InterruptedException | ExecutionException | HopsSecurityException | GenericException e) {
                String failedUser = project.getName() + HdfsUsersController.USER_NAME_DELIMITER + newMember.getUsername();
                LOGGER.log(Level.SEVERE, "Could not delete user certificates for user " + failedUser + ". Manual cleanup is needed!!! ", e);
            }
            LOGGER.log(Level.SEVERE, "error while creating certificates, jupyter kernel: " + ex.getMessage(), ex);
            hdfsUsersController.removeMember(projectTeam);
            projectTeamFacade.removeProjectTeam(project, newMember);
            throw new EJBException("Could not create certificates for user");
        }
        // trigger project team role update handlers
        ProjectTeamRoleHandler.runProjectTeamRoleAddMembersHandlers(projectTeamRoleHandlers, project, Collections.singletonList(newMember), ProjectRoleTypes.fromString(projectTeam.getTeamRole()), false);
        String message = "You have been added to project " + project.getName() + " with a role " + projectTeam.getTeamRole() + ".";
        messageController.send(newMember, owner, "You have been added to a project.", message, message, "");
        LOGGER.log(Level.FINE, "{0} - member added to project : {1}.", new Object[] { newMember.getEmail(), project.getName() });
        logActivity(ActivityFacade.NEW_MEMBER + projectTeam.getProjectTeamPK().getTeamMember(), owner, project, ActivityFlag.MEMBER);
        return true;
    } else {
        return false;
    }
}