Examples with SubjectDTO io.hops.hopsworks.common.dao.kafka.schemas.SubjectDTO used on opensource projects

Example 1 with SubjectDTO

use of io.hops.hopsworks.common.dao.kafka.schemas.SubjectDTO in project hopsworks by logicalclocks.

the class KafkaController method getSubjectForTopic.

public SubjectDTO getSubjectForTopic(Project project, String topic) throws KafkaException, ProjectException {
    Optional<ProjectTopics> pt = projectTopicsFacade.findTopicByNameAndProject(project, topic);
    if (!pt.isPresent()) {
        SharedTopics sharedTopic = sharedTopicsFacade.findSharedTopicByProjectAndTopic(project.getId(), topic).orElseThrow(() -> new KafkaException(RESTCodes.KafkaErrorCode.TOPIC_NOT_SHARED, Level.FINE, "topic: " + topic + ", project: " + project.getName()));
        Project sharedWithProject = projectFacade.findById(sharedTopic.getProjectId()).orElseThrow(() -> new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "projectId: " + sharedTopic.getSharedTopicsPK().getProjectId()));
        pt = projectTopicsFacade.findTopicByNameAndProject(sharedWithProject, topic);
    }
    if (!pt.isPresent()) {
        throw new KafkaException(RESTCodes.KafkaErrorCode.TOPIC_NOT_FOUND, Level.FINE, "project=" + project.getName() + ", topic=" + topic);
    }
    return new SubjectDTO(pt.get().getSubjects());
}

Example 2 with SubjectDTO

use of io.hops.hopsworks.common.dao.kafka.schemas.SubjectDTO in project hopsworks by logicalclocks.

the class KafkaResource method getSchema.

@ApiOperation(value = "Get avro schema of a subject and version.")
@GET
@Path("/subjects/{subject}/versions/{version}/schema")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@AllowedProjectRoles({ AllowedProjectRoles.DATA_OWNER, AllowedProjectRoles.DATA_SCIENTIST })
@JWTRequired(acceptedTokens = { Audience.API }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
@ApiKeyRequired(acceptedScopes = { ApiScope.KAFKA }, allowedUserRoles = { "HOPS_ADMIN", "HOPS_USER" })
public Response getSchema(@PathParam("subject") String subject, @PathParam("version") String version, @Context SecurityContext sc) {
    try {
        SubjectDTO dto = subjectsController.getSubjectDetails(project, subject, version);
        GenericEntity<String> entity = new GenericEntity<String>(dto.getSchema()) {
        };
        return Response.ok().entity(entity).build();
    } catch (SchemaException e) {
        SchemaRegistryError error = new SchemaRegistryError(e.getErrorCode().getCode(), e.getErrorCode().getMessage());
        return Response.status(e.getErrorCode().getRespStatus()).entity(error).build();
    }
}

Example 3 with SubjectDTO

use of io.hops.hopsworks.common.dao.kafka.schemas.SubjectDTO in project hopsworks by logicalclocks.

the class OnlineFeaturegroupController method createOnlineKafkaTopic.

// For ingesting data in the online feature store, we setup a new topic for each feature group
// The topic schema is also registered so it's available both for the hsfs library and for the collector
private void createOnlineKafkaTopic(Project project, Integer featureGroupId, String featureGroupEntityName, String avroSchema) throws KafkaException, SchemaException, ProjectException, UserException {
    String topicName = onlineFeatureGroupTopicName(project.getId(), featureGroupId, featureGroupEntityName);
    SubjectDTO topicSubject = subjectsController.registerNewSubject(project, topicName, avroSchema, false);
    subjectsCompatibilityController.setSubjectCompatibility(project, topicName, SchemaCompatibility.NONE);
    // TODO(Fabio): Make Kafka topics configurable
    TopicDTO topicDTO = new TopicDTO(topicName, 1, settings.getOnlineFsThreadNumber(), topicSubject.getSubject(), topicSubject.getVersion());
    kafkaController.createTopic(project, topicDTO);
}

Example 4 with SubjectDTO

use of io.hops.hopsworks.common.dao.kafka.schemas.SubjectDTO in project hopsworks by logicalclocks.

the class OnlineFeaturegroupController method alterOnlineFeatureGroupSchema.

public void alterOnlineFeatureGroupSchema(Featuregroup featureGroup, List<FeatureGroupFeatureDTO> newFeatures, List<FeatureGroupFeatureDTO> fullNewSchema, Project project, Users user) throws FeaturestoreException, SchemaException, SQLException, KafkaException {
    String tableName = Utils.getFeatureStoreEntityName(featureGroup.getName(), featureGroup.getVersion());
    String topicName = onlineFeatureGroupTopicName(project.getId(), featureGroup.getId(), tableName);
    alterMySQLTableColumns(featureGroup.getFeaturestore(), tableName, newFeatures, project, user);
    // publish new version of avro schema
    String avroSchema = avroSchemaConstructorController.constructSchema(featureGroup.getName(), Utils.getFeaturestoreName(project), fullNewSchema);
    schemasController.validateSchema(project, avroSchema);
    SubjectDTO topicSubject = subjectsController.registerNewSubject(project, topicName, avroSchema, false);
    kafkaController.updateTopicSchemaVersion(project, topicName, topicSubject.getVersion());
}

Example 5 with SubjectDTO

use of io.hops.hopsworks.common.dao.kafka.schemas.SubjectDTO in project hopsworks by logicalclocks.

the class SubjectsController method registerNewSubject.

public SubjectDTO registerNewSubject(Project project, String subject, String schemaContent, boolean isEnablingKafkaService) throws KafkaException, SchemaException {
    validateSubject(subject, isEnablingKafkaService);
    Schema schema;
    try {
        schema = new Schema.Parser().parse(schemaContent);
    } catch (SchemaParseException e) {
        throw new SchemaException(RESTCodes.SchemaRegistryErrorCode.INVALID_AVRO_SCHEMA, Level.FINE, "schema=" + schemaContent);
    }
    // check if schema exists - return current id
    Optional<Subjects> optionalSubject = subjectsFacade.findSubjectByNameAndSchema(project, subject, schema.toString());
    if (optionalSubject.isPresent()) {
        Subjects subjects = optionalSubject.get();
        return new SubjectDTO(subjects.getSchema().getId(), subjects.getSubject(), subjects.getVersion());
    }
    // check if schema compatible - return 409 of not
    if (!isCompatible(project, subject, schema)) {
        throw new SchemaException(RESTCodes.SchemaRegistryErrorCode.INCOMPATIBLE_AVRO_SCHEMA, Level.FINE, "Subject=" + subject + ", project=" + project.getName());
    }
    Integer latestVersion = subjectsFacade.findSubjectByName(project, subject).stream().map(Subjects::getVersion).max(Integer::compareTo).orElse(0);
    Schemas schemas = schemasController.addNewSchema(project, schema.toString());
    Integer id = subjectsFacade.insertNewSubject(project, subject, schemas, latestVersion + 1);
    return new SubjectDTO(id, subject, latestVersion + 1);
}