Examples with ProjectTopics io.hops.hopsworks.persistence.entity.kafka.ProjectTopics used on opensource projects

Example 6 with ProjectTopics

use of io.hops.hopsworks.persistence.entity.kafka.ProjectTopics in project hopsworks by logicalclocks.

the class KafkaServingHelper method setupKafkaServingTopic.

/**
 * Sets up the kafka topic for logging inference requests for models being served on Hopsworks. This kafka topic
 * for logging is setup automatically when updating or creating a model serving instance if the user have specified
 * that option in the configuration of the serving.
 *
 * @param project the project where the serving resides and where the kafka topic is to be created
 * @param servingWrapper the internal representation of the serving instance
 * @param newDbServing the new serving to save in the database (either a new insertion or an update)
 * @param oldDbServing the old serving in the database (in case of an update)
 * @throws KafkaException
 * @throws ProjectException
 * @throws UserException
 * @throws ServiceException
 * @throws ServingException
 */
public void setupKafkaServingTopic(Project project, ServingWrapper servingWrapper, Serving newDbServing, Serving oldDbServing) throws KafkaException, ProjectException, UserException, ServingException, InterruptedException, ExecutionException {
    if (servingWrapper.getKafkaTopicDTO() != null && servingWrapper.getKafkaTopicDTO().getName() != null && servingWrapper.getKafkaTopicDTO().getName().equalsIgnoreCase("NONE")) {
        // The User has decided to not log the serving requests
        newDbServing.setKafkaTopic(null);
    } else if (servingWrapper.getKafkaTopicDTO() != null && servingWrapper.getKafkaTopicDTO().getName() != null && servingWrapper.getKafkaTopicDTO().getName().equalsIgnoreCase("CREATE")) {
        // The user is creating a new Kafka Topic
        ProjectTopics topic = setupKafkaTopic(project, servingWrapper);
        newDbServing.setKafkaTopic(topic);
    } else if (servingWrapper.getKafkaTopicDTO() != null && servingWrapper.getKafkaTopicDTO().getName() != null && !(servingWrapper.getKafkaTopicDTO().getName().equalsIgnoreCase("CREATE") || servingWrapper.getKafkaTopicDTO().getName().equalsIgnoreCase("NONE"))) {
        if (oldDbServing != null && oldDbServing.getKafkaTopic() != null && oldDbServing.getKafkaTopic().getTopicName().equals(servingWrapper.getKafkaTopicDTO().getName())) {
            // This is an update and the topic name hasn't changed.
            newDbServing.setKafkaTopic(oldDbServing.getKafkaTopic());
        } else {
            // The user has selected a an already existing Kafka topic. Check that it matches the schema requirements
            ProjectTopics topic = checkSchemaRequirements(project, servingWrapper);
            newDbServing.setKafkaTopic(topic);
        }
    }
}

Example 7 with ProjectTopics

use of io.hops.hopsworks.persistence.entity.kafka.ProjectTopics in project hopsworks by logicalclocks.

the class KafkaServingHelper method setupKafkaTopic.

private ProjectTopics setupKafkaTopic(Project project, ServingWrapper servingWrapper) throws KafkaException, UserException, ProjectException, InterruptedException, ExecutionException {
    try {
        // Check that the user is not trying to create a topic with  more replicas than brokers.
        if (servingWrapper.getKafkaTopicDTO().getNumOfReplicas() != null && (servingWrapper.getKafkaTopicDTO().getNumOfReplicas() <= 0 || servingWrapper.getKafkaTopicDTO().getNumOfReplicas() > kafkaBrokers.getBrokerEndpoints().size())) {
            throw new KafkaException(RESTCodes.KafkaErrorCode.TOPIC_REPLICATION_ERROR, Level.FINE);
        } else if (servingWrapper.getKafkaTopicDTO().getNumOfReplicas() == null) {
            // set default value
            servingWrapper.getKafkaTopicDTO().setNumOfReplicas(settings.getKafkaDefaultNumReplicas());
        }
    } catch (IOException | KeeperException | InterruptedException e) {
        throw new KafkaException(RESTCodes.KafkaErrorCode.BROKER_METADATA_ERROR, Level.SEVERE, "", e.getMessage(), e);
    }
    // Check that the user is not trying to create a topic with negative partitions
    if (servingWrapper.getKafkaTopicDTO().getNumOfPartitions() != null && servingWrapper.getKafkaTopicDTO().getNumOfPartitions() <= 0) {
        throw new KafkaException(RESTCodes.KafkaErrorCode.BAD_NUM_PARTITION, Level.FINE, "less than 0");
    } else if (servingWrapper.getKafkaTopicDTO().getNumOfPartitions() == null) {
        // set default value
        servingWrapper.getKafkaTopicDTO().setNumOfPartitions(settings.getKafkaDefaultNumPartitions());
    }
    String servingTopicName = getServingTopicName(servingWrapper);
    TopicDTO topicDTO = new TopicDTO(servingTopicName, servingWrapper.getKafkaTopicDTO().getNumOfReplicas(), servingWrapper.getKafkaTopicDTO().getNumOfPartitions(), Settings.INFERENCE_SCHEMANAME, Settings.INFERENCE_SCHEMAVERSION);
    ProjectTopics pt = kafkaController.createTopicInProject(project, topicDTO);
    // Add the ACLs for this topic. By default all users should be able to do everything
    for (ProjectTeam projectTeam : project.getProjectTeamCollection()) {
        AclDTO aclDto = new AclDTO(project.getName(), projectTeam.getUser().getEmail(), "allow", Settings.KAFKA_ACL_WILDCARD, Settings.KAFKA_ACL_WILDCARD, Settings.KAFKA_ACL_WILDCARD);
        kafkaController.addAclsToTopic(topicDTO.getName(), project.getId(), aclDto);
    }
    return pt;
}

Example 8 with ProjectTopics

use of io.hops.hopsworks.persistence.entity.kafka.ProjectTopics in project hopsworks by logicalclocks.

the class KafkaController method acceptSharedTopic.

public void acceptSharedTopic(Project project, String topicName) throws KafkaException, ProjectException, UserException {
    ProjectTopics pt = projectTopicsFacade.findTopicByName(topicName).orElseThrow(() -> new KafkaException(RESTCodes.KafkaErrorCode.TOPIC_NOT_FOUND, Level.FINE, "topicName: " + topicName));
    if (!sharedTopicsFacade.findSharedTopicByProjectAndTopic(project.getId(), topicName).isPresent()) {
        throw new KafkaException(RESTCodes.KafkaErrorCode.TOPIC_NOT_SHARED, Level.FINE, "topic: " + topicName + ", project: " + project.getName());
    }
    addFullPermissionAclsToTopic(project.getName(), topicName, pt.getProject().getId());
    sharedTopicsFacade.acceptSharedTopic(pt.getProject().getId(), topicName, project.getId());
}

Example 9 with ProjectTopics

use of io.hops.hopsworks.persistence.entity.kafka.ProjectTopics in project hopsworks by logicalclocks.

the class KafkaController method updateTopicSubjectVersion.

public void updateTopicSubjectVersion(Project project, String topic, String subject, Integer version) throws KafkaException, SchemaException {
    ProjectTopics pt = projectTopicsFacade.findTopicByNameAndProject(project, topic).orElseThrow(() -> new KafkaException(RESTCodes.KafkaErrorCode.TOPIC_NOT_FOUND, Level.FINE, "topic: " + topic));
    String topicSubject = pt.getSubjects().getSubject();
    Subjects st = subjectsFacade.findSubjectByNameAndVersion(project, subject, version).orElseThrow(() -> new SchemaException(RESTCodes.SchemaRegistryErrorCode.VERSION_NOT_FOUND, Level.FINE, "schema: " + topicSubject + ", version: " + version));
    projectTopicsFacade.updateTopicSchemaVersion(pt, st);
}

Example 10 with ProjectTopics

use of io.hops.hopsworks.persistence.entity.kafka.ProjectTopics in project hopsworks by logicalclocks.

the class KafkaController method removeTopicFromProject.

public void removeTopicFromProject(Project project, String topicName) throws KafkaException {
    ProjectTopics pt = projectTopicsFacade.findTopicByNameAndProject(project, topicName).orElseThrow(() -> new KafkaException(RESTCodes.KafkaErrorCode.TOPIC_NOT_FOUND, Level.FINE, "topic: " + topicName));
    // remove from database
    projectTopicsFacade.remove(pt);
    /*
     * What is the possibility of the program failing below? The topic is
     * removed from
     * db, but not yet from zk. *
     * Possibilities:
     * 1. ZkClient is unable to establish a connection, maybe due to timeouts.
     * 2. In case delete.topic.enable is not set to true in the Kafka server
     * configuration, delete topic marks a topic for deletion. Subsequent
     * topic (with the same name) create operation fails.
     */
    // remove from zookeeper
    hopsKafkaAdminClient.deleteTopics(Collections.singleton(pt.getTopicName()));
}