Examples with ProjectTopics io.hops.hopsworks.persistence.entity.kafka.ProjectTopics used on opensource projects

Example 1 with ProjectTopics

use of io.hops.hopsworks.persistence.entity.kafka.ProjectTopics in project hopsworks by logicalclocks.

the class ServingUtil method validateKafkaTopicSchema.

private void validateKafkaTopicSchema(Project project, Serving serving, TopicDTO topic) throws ServingException {
    // if an existing topic, check schema
    if (topic != null && !topic.getName().equals("NONE") && !topic.getName().equals("CREATE")) {
        ProjectTopics projectTopic = projectTopicsFacade.findTopicByNameAndProject(project, topic.getName()).orElseThrow(() -> new ServingException(RESTCodes.ServingErrorCode.KAFKA_TOPIC_NOT_FOUND, Level.SEVERE, null));
        Subjects subjects = projectTopic.getSubjects();
        if (!subjects.getSubject().equalsIgnoreCase(Settings.INFERENCE_SCHEMANAME)) {
            throw new ServingException(RESTCodes.ServingErrorCode.KAFKA_TOPIC_NOT_VALID, Level.FINE, "Inference logging" + " requires a Kafka topic with schema '" + Settings.INFERENCE_SCHEMANAME + "'");
        }
    }
}

Example 2 with ProjectTopics

use of io.hops.hopsworks.persistence.entity.kafka.ProjectTopics in project hopsworks by logicalclocks.

the class KafkaController method findTopicsByProject.

public List<TopicDTO> findTopicsByProject(Project project) {
    List<ProjectTopics> ptList = projectTopicsFacade.findTopicsByProject(project);
    List<TopicDTO> topics = new ArrayList<>();
    for (ProjectTopics pt : ptList) {
        topics.add(new TopicDTO(pt.getTopicName(), pt.getSubjects().getSubject(), pt.getSubjects().getVersion(), false));
    }
    return topics;
}

Example 3 with ProjectTopics

use of io.hops.hopsworks.persistence.entity.kafka.ProjectTopics in project hopsworks by logicalclocks.

the class KafkaController method getSubjectForTopic.

public SubjectDTO getSubjectForTopic(Project project, String topic) throws KafkaException, ProjectException {
    Optional<ProjectTopics> pt = projectTopicsFacade.findTopicByNameAndProject(project, topic);
    if (!pt.isPresent()) {
        SharedTopics sharedTopic = sharedTopicsFacade.findSharedTopicByProjectAndTopic(project.getId(), topic).orElseThrow(() -> new KafkaException(RESTCodes.KafkaErrorCode.TOPIC_NOT_SHARED, Level.FINE, "topic: " + topic + ", project: " + project.getName()));
        Project sharedWithProject = projectFacade.findById(sharedTopic.getProjectId()).orElseThrow(() -> new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + sharedTopic.getSharedTopicsPK().getProjectId()));
        pt = projectTopicsFacade.findTopicByNameAndProject(sharedWithProject, topic);
    }
    if (!pt.isPresent()) {
        throw new KafkaException(RESTCodes.KafkaErrorCode.TOPIC_NOT_FOUND, Level.FINE, "project=" + project.getName() + ", topic=" + topic);
    }
    return new SubjectDTO(pt.get().getSubjects());
}

Example 4 with ProjectTopics

use of io.hops.hopsworks.persistence.entity.kafka.ProjectTopics in project hopsworks by logicalclocks.

the class KafkaController method addAclsToTopic.

private Pair<TopicAcls, Response.Status> addAclsToTopic(String topicName, Integer projectId, String selectedProjectName, String userEmail, String permissionType, String operationType, String host, String role) throws ProjectException, KafkaException, UserException {
    if (Strings.isNullOrEmpty(topicName) || userEmail == null) {
        throw new IllegalArgumentException("Topic and userEmail must be provided.");
    }
    // get the project id
    Project topicOwnerProject = Optional.ofNullable(projectFacade.find(projectId)).orElseThrow(() -> new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + projectId));
    if (!topicOwnerProject.getName().equals(selectedProjectName)) {
        if (projectFacade.findByName(selectedProjectName) == null) {
            throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "The specified project " + "for the topic" + topicName + " was not found");
        }
    }
    ProjectTopics pt = projectTopicsFacade.findTopicByNameAndProject(topicOwnerProject, topicName).orElseThrow(() -> new KafkaException(RESTCodes.KafkaErrorCode.TOPIC_NOT_FOUND, Level.FINE, "Topic: " + topicName));
    // should not be able to create multiple ACLs at the same time
    if (userEmail.equals("*")) {
        throw new KafkaException(RESTCodes.KafkaErrorCode.ACL_FOR_ANY_USER, Level.FINE, "topic: " + topicName);
    }
    // fetch the user name from database
    Users user = Optional.ofNullable(userFacade.findByEmail(userEmail)).orElseThrow(() -> new UserException(RESTCodes.UserErrorCode.USER_WAS_NOT_FOUND, Level.FINE, "user: " + userEmail));
    String principalName = KafkaConst.buildPrincipalName(selectedProjectName, user.getUsername());
    Optional<TopicAcls> optionalAcl = topicAclsFacade.getTopicAcls(topicName, principalName, permissionType, operationType, host, role);
    if (optionalAcl.isPresent()) {
        return Pair.of(optionalAcl.get(), Response.Status.OK);
    }
    TopicAcls acl = topicAclsFacade.addAclsToTopic(pt, user, permissionType, operationType, host, role, principalName);
    return Pair.of(acl, Response.Status.CREATED);
}

Example 5 with ProjectTopics

use of io.hops.hopsworks.persistence.entity.kafka.ProjectTopics in project hopsworks by logicalclocks.

the class ZookeeperTopicCleanerTimer method execute.

// Run once per hour
@Schedule(persistent = false, minute = "0", hour = "*")
public void execute(Timer timer) {
    LOGGER.log(Level.INFO, "Running ZookeeperTopicCleanerTimer.");
    try {
        String zkConnectionString = kafkaBrokers.getZookeeperConnectionString();
        Set<String> zkTopics = new HashSet<>();
        try {
            zk = new ZooKeeper(zkConnectionString, Settings.ZOOKEEPER_SESSION_TIMEOUT_MS, new ZookeeperWatcher());
            List<String> topics = zk.getChildren("/brokers/topics", false);
            zkTopics.addAll(topics);
        } catch (IOException ex) {
            LOGGER.log(Level.SEVERE, "Unable to find the zookeeper server: ", ex.toString());
        } catch (KeeperException | InterruptedException ex) {
            LOGGER.log(Level.SEVERE, "Cannot retrieve topic list from Zookeeper", ex);
        } finally {
            if (zk != null) {
                try {
                    zk.close();
                } catch (InterruptedException ex) {
                    LOGGER.log(Level.SEVERE, "Unable to close zookeeper connection", ex);
                }
                zk = null;
            }
        }
        List<ProjectTopics> dbProjectTopics = em.createNamedQuery("ProjectTopics.findAll").getResultList();
        Set<String> dbTopics = new HashSet<>();
        for (ProjectTopics pt : dbProjectTopics) {
            dbTopics.add(pt.getTopicName());
        }
        /*
       * To remove topics from zookeeper which do not exist in database. This
       * situation
       * happens when a hopsworks project is deleted, because all the topics in
       * the project
       * will be deleted (cascade delete) without deleting them from the Kafka
       * cluster.
       * 1. get all topics from zookeeper
       * 2. get the topics which exist in zookeeper, but not in database
       * zkTopics.removeAll(dbTopics);
       * 3. remove those topics
       */
        zkTopics.removeAll(dbTopics);
        // DON'T remove offset topic
        zkTopics.remove(offsetTopic);
        if (!zkTopics.isEmpty()) {
            // blocks until all are deleted
            try {
                hopsKafkaAdminClient.deleteTopics(zkTopics).all().get();
                LOGGER.log(Level.INFO, "Removed topics {0} from Kafka", new Object[] { zkTopics });
            } catch (ExecutionException | InterruptedException ex) {
                LOGGER.log(Level.SEVERE, "Error dropping topics from Kafka", ex);
            }
        }
    } catch (ServiceDiscoveryException ex) {
        LOGGER.log(Level.SEVERE, "Could not discover Zookeeper server addresses", ex);
    } catch (Exception ex) {
        LOGGER.log(Level.SEVERE, "Got an exception while cleaning up kafka topics", ex);
    }
}