Examples with KafkaException io.hops.hopsworks.exceptions.KafkaException used on opensource projects

Example 6 with KafkaException

use of io.hops.hopsworks.exceptions.KafkaException in project hopsworks by logicalclocks.

the class ProjectController method createProject.

/**
 * Creates a new project(project), the related DIR, the different services in
 * the project, and the master of the
 * project.
 * <p>
 * This needs to be an atomic operation (all or nothing) REQUIRES_NEW will
 * make sure a new transaction is created even
 * if this method is called from within a transaction.
 *
 * @param projectDTO
 * @param owner
 * @param sessionId
 * @return
 */
public Project createProject(ProjectDTO projectDTO, Users owner, String sessionId) throws DatasetException, GenericException, KafkaException, ProjectException, UserException, HopsSecurityException, ServiceException, FeaturestoreException, ElasticException, SchemaException, IOException {
    Long startTime = System.currentTimeMillis();
    // check that the project name is ok
    String projectName = projectDTO.getProjectName();
    FolderNameValidator.isValidProjectName(projectUtils, projectName);
    List<ProjectServiceEnum> projectServices = new ArrayList<>();
    if (projectDTO.getServices() != null) {
        for (String s : projectDTO.getServices()) {
            ProjectServiceEnum se = ProjectServiceEnum.valueOf(s.toUpperCase());
            projectServices.add(se);
        }
    }
    LOGGER.log(Level.FINE, () -> "PROJECT CREATION TIME. Step 1: " + (System.currentTimeMillis() - startTime));
    DistributedFileSystemOps dfso = null;
    Project project = null;
    try {
        dfso = dfs.getDfsOps();
        /*
       * create a project in the database
       * if the creation go through it means that there is no other project with
       * the same name.
       * this project creation act like a lock, no other project can be created
       * with the same name
       * until this project is removed from the database
       */
        try {
            project = createProject(projectName, owner, projectDTO.getDescription(), dfso);
        } catch (EJBException ex) {
            LOGGER.log(Level.WARNING, null, ex);
            Path dummy = new Path("/tmp/" + projectName);
            try {
                dfso.rm(dummy, true);
            } catch (IOException e) {
                LOGGER.log(Level.SEVERE, null, e);
            }
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_EXISTS, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 2 (hdfs): {0}", System.currentTimeMillis() - startTime);
        verifyProject(project, dfso, sessionId);
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 3 (verify): {0}", System.currentTimeMillis() - startTime);
        // Run the handlers.
        try {
            ProjectHandler.runProjectPreCreateHandlers(projectHandlers, project);
        } catch (ProjectException ex) {
            cleanup(project, sessionId, null, true, owner);
            throw ex;
        }
        List<Future<?>> projectCreationFutures = new ArrayList<>();
        // This is an async call
        try {
            projectCreationFutures.add(certificatesController.generateCertificates(project, owner));
        } catch (Exception ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new HopsSecurityException(RESTCodes.SecurityErrorCode.CERT_CREATION_ERROR, Level.SEVERE, "project: " + project.getName() + "owner: " + owner.getUsername(), ex.getMessage(), ex);
        }
        String username = hdfsUsersController.getHdfsUserName(project, owner);
        if (username == null || username.isEmpty()) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new UserException(RESTCodes.UserErrorCode.USER_WAS_NOT_FOUND, Level.SEVERE, "project: " + project.getName() + "owner: " + owner.getUsername());
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 4 (certs): {0}", System.currentTimeMillis() - startTime);
        // all the verifications have passed, we can now create the project
        // create the project folder
        ProvTypeDTO provType = settings.getProvType().dto;
        try {
            mkProjectDIR(projectName, dfso);
            fsProvController.updateProjectProvType(project, provType, dfso);
        } catch (IOException | EJBException | ProvenanceException ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_FOLDER_NOT_CREATED, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 5 (folders): {0}", System.currentTimeMillis() - startTime);
        // update the project with the project folder inode
        try {
            setProjectInode(project, dfso);
        } catch (IOException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_INODE_CREATION_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 6 (inodes): {0}", System.currentTimeMillis() - startTime);
        // set payment and quotas
        try {
            setProjectOwnerAndQuotas(project, dfso, owner);
        } catch (IOException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures, true, owner);
            throw new ProjectException(RESTCodes.ProjectErrorCode.QUOTA_ERROR, Level.SEVERE, "project: " + project.getName(), ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 7 (quotas): {0}", System.currentTimeMillis() - startTime);
        try {
            hdfsUsersController.addProjectFolderOwner(project, dfso);
            createProjectLogResources(owner, project, dfso);
        } catch (IOException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_SET_PERMISSIONS_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 8 (logs): {0}", System.currentTimeMillis() - startTime);
        // inconsistencies
        try {
            elasticController.deleteProjectIndices(project);
            elasticController.deleteProjectSavedObjects(projectName);
            LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 9 (elastic cleanup): {0}", System.currentTimeMillis() - startTime);
        } catch (ElasticException ex) {
            LOGGER.log(Level.FINE, "Error while cleaning old project indices", ex);
        }
        logProject(project, OperationType.Add);
        // enable services
        for (ProjectServiceEnum service : projectServices) {
            try {
                projectCreationFutures.addAll(addService(project, service, owner, dfso, provType));
            } catch (RESTException | IOException ex) {
                cleanup(project, sessionId, projectCreationFutures);
                throw ex;
            }
        }
        try {
            for (Future f : projectCreationFutures) {
                if (f != null) {
                    f.get();
                }
            }
        } catch (InterruptedException | ExecutionException ex) {
            LOGGER.log(Level.SEVERE, "Error while waiting for the certificate generation thread to finish. Will try to " + "cleanup...", ex);
            cleanup(project, sessionId, projectCreationFutures);
            throw new HopsSecurityException(RESTCodes.SecurityErrorCode.CERT_CREATION_ERROR, Level.SEVERE);
        }
        // Run the handlers.
        try {
            ProjectHandler.runProjectPostCreateHandlers(projectHandlers, project);
        } catch (ProjectException ex) {
            cleanup(project, sessionId, projectCreationFutures);
            throw ex;
        }
        try {
            project = environmentController.createEnv(project, owner);
        } catch (PythonException | EJBException ex) {
            cleanup(project, sessionId, projectCreationFutures);
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_ANACONDA_ENABLE_ERROR, Level.SEVERE, "project: " + projectName, ex.getMessage(), ex);
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 10 (env): {0}", System.currentTimeMillis() - startTime);
        return project;
    } finally {
        if (dfso != null) {
            dfso.close();
        }
        LOGGER.log(Level.FINE, "PROJECT CREATION TIME. Step 11 (close): {0}", System.currentTimeMillis() - startTime);
    }
}

Example 7 with KafkaException

use of io.hops.hopsworks.exceptions.KafkaException in project hopsworks by logicalclocks.

the class SubjectsController method deleteSubjectsVersion.

public Integer deleteSubjectsVersion(Project project, String subject, String version) throws SchemaException, KafkaException {
    validateSubject(subject, false);
    validateVersion(version);
    if (!subjectsFacade.getListOfSubjects(project).contains(subject)) {
        throw new SchemaException(RESTCodes.SchemaRegistryErrorCode.SUBJECT_NOT_FOUND, Level.FINE, "subject=" + subject);
    }
    Optional<Subjects> optional;
    if (version.equals("latest")) {
        optional = subjectsFacade.findSubjectLatestVersion(project, subject);
    } else {
        optional = subjectsFacade.findSubjectByNameAndVersion(project, subject, Integer.valueOf(version));
    }
    if (!optional.isPresent()) {
        throw new SchemaException(RESTCodes.SchemaRegistryErrorCode.VERSION_NOT_FOUND, Level.FINE, "project=" + project.getName() + ", subject=" + subject + ", version=" + version);
    }
    Integer versionToDelete = optional.get().getVersion();
    if (!projectTopicsFacade.findTopiscBySubjectAndVersion(project, subject, versionToDelete).isEmpty()) {
        throw new KafkaException(RESTCodes.KafkaErrorCode.SCHEMA_IN_USE, Level.FINE, "project=" + project.getName() + ", subject=" + subject + ", version=" + versionToDelete);
    }
    subjectsFacade.remove(optional.get());
    subjectsCompatibilityFacade.getSubjectCompatibility(project, subject).ifPresent(sc -> subjectsCompatibilityFacade.remove(sc));
    return versionToDelete;
}

Example 8 with KafkaException

use of io.hops.hopsworks.exceptions.KafkaException in project hopsworks by logicalclocks.

the class SubjectsController method deleteSubject.

public List<Integer> deleteSubject(Project project, String subject) throws SchemaException, KafkaException {
    validateSubject(subject, false);
    if (!projectTopicsFacade.findTopicsBySubject(project, subject).isEmpty()) {
        throw new KafkaException(RESTCodes.KafkaErrorCode.SCHEMA_IN_USE, Level.FINE, "project=" + project.getName() + ", subject=" + subject);
    }
    List<Integer> versions = getSubjectVersions(project, subject);
    Integer deleted = subjectsFacade.deleteSubject(project, subject);
    if (versions.size() != deleted) {
        throw new SchemaException(RESTCodes.SchemaRegistryErrorCode.INTERNAL_SERVER_ERROR, Level.FINE, "error deleting " + "subject. versions=" + Arrays.toString(versions.toArray()) + ", but deleted " + deleted + " items.");
    }
    subjectsCompatibilityFacade.getSubjectCompatibility(project, subject).ifPresent(sc -> subjectsCompatibilityFacade.remove(sc));
    return versions;
}

Example 9 with KafkaException

use of io.hops.hopsworks.exceptions.KafkaException in project hopsworks by logicalclocks.

the class ProjectController method addMember.

public boolean addMember(ProjectTeam projectTeam, Project project, Users newMember, Users owner, DistributedFileSystemOps dfso) throws UserException, KafkaException, ProjectException, FeaturestoreException, IOException {
    if (projectTeam.getTeamRole() == null || (!projectTeam.getTeamRole().equals(ProjectRoleTypes.DATA_SCIENTIST.getRole()) && !projectTeam.getTeamRole().equals(ProjectRoleTypes.DATA_OWNER.getRole()))) {
        projectTeam.setTeamRole(ProjectRoleTypes.DATA_SCIENTIST.getRole());
    }
    projectTeam.setTimestamp(new Date());
    if (newMember != null && !projectTeamFacade.isUserMemberOfProject(project, newMember)) {
        // this makes sure that the member is added to the project sent as the
        // first param b/c the security check was made on the parameter sent as path.
        projectTeam.getProjectTeamPK().setProjectId(project.getId());
        projectTeam.setProject(project);
        projectTeam.setUser(newMember);
        project.getProjectTeamCollection().add(projectTeam);
        projectFacade.update(project);
        hdfsUsersController.addNewProjectMember(projectTeam, dfso);
        // Add user to kafka topics ACLs by default
        if (projectServicesFacade.isServiceEnabledForProject(project, ProjectServiceEnum.KAFKA)) {
            kafkaController.addProjectMemberToTopics(project, newMember.getEmail());
        }
        // if online-featurestore service is enabled in the project, give new member access to it
        if (projectServiceFacade.isServiceEnabledForProject(project, ProjectServiceEnum.FEATURESTORE) && settings.isOnlineFeaturestore()) {
            Featurestore featurestore = featurestoreController.getProjectFeaturestore(project);
            onlineFeaturestoreController.createDatabaseUser(projectTeam.getUser(), featurestore, projectTeam.getTeamRole());
        }
        // TODO: This should now be a REST call
        Future<CertificatesController.CertsResult> certsResultFuture = null;
        try {
            certsResultFuture = certificatesController.generateCertificates(project, newMember);
            certsResultFuture.get();
        } catch (Exception ex) {
            try {
                if (certsResultFuture != null) {
                    certsResultFuture.get();
                }
                certificatesController.revokeUserSpecificCertificates(project, newMember);
            } catch (IOException | InterruptedException | ExecutionException | HopsSecurityException | GenericException e) {
                String failedUser = project.getName() + HdfsUsersController.USER_NAME_DELIMITER + newMember.getUsername();
                LOGGER.log(Level.SEVERE, "Could not delete user certificates for user " + failedUser + ". Manual cleanup is needed!!! ", e);
            }
            LOGGER.log(Level.SEVERE, "error while creating certificates, jupyter kernel: " + ex.getMessage(), ex);
            hdfsUsersController.removeMember(projectTeam);
            projectTeamFacade.removeProjectTeam(project, newMember);
            throw new EJBException("Could not create certificates for user");
        }
        // trigger project team role update handlers
        ProjectTeamRoleHandler.runProjectTeamRoleAddMembersHandlers(projectTeamRoleHandlers, project, Collections.singletonList(newMember), ProjectRoleTypes.fromString(projectTeam.getTeamRole()), false);
        String message = "You have been added to project " + project.getName() + " with a role " + projectTeam.getTeamRole() + ".";
        messageController.send(newMember, owner, "You have been added to a project.", message, message, "");
        LOGGER.log(Level.FINE, "{0} - member added to project : {1}.", new Object[] { newMember.getEmail(), project.getName() });
        logActivity(ActivityFacade.NEW_MEMBER + projectTeam.getProjectTeamPK().getTeamMember(), owner, project, ActivityFlag.MEMBER);
        return true;
    } else {
        return false;
    }
}

Example 10 with KafkaException

use of io.hops.hopsworks.exceptions.KafkaException in project hopsworks by logicalclocks.

the class TopicsBuilder method buildTopicDetails.

public PartitionDetailsDTO buildTopicDetails(UriInfo uriInfo, Project project, String topicName) throws KafkaException {
    PartitionDetailsDTO dto = new PartitionDetailsDTO();
    dto.setHref(topicUri(uriInfo, project, topicName).build());
    try {
        List<PartitionDetailsDTO> list = kafkaController.getTopicDetails(project, topicName).get(3000, TimeUnit.MILLISECONDS);
        dto.setCount(Integer.toUnsignedLong(list.size()));
        list.forEach(dto::addItem);
        return dto;
    } catch (InterruptedException | ExecutionException | TimeoutException e) {
        throw new KafkaException(RESTCodes.KafkaErrorCode.TOPIC_FETCH_FAILED, Level.WARNING, "Topic name: " + topicName, e.getMessage(), e);
    }
}