Search in sources :

Example 1 with QuickTopicType

use of com.bakdata.quick.common.type.QuickTopicType in project quick by bakdata.

the class InferTopicTypeService method inferSerDes.

private Optional<TopicSchemaTypes> inferSerDes(final String topic) {
    final InferConsumer inferConsumer = new InferConsumer(this.bootstrapServer, topic, this.appId);
    final ConsumerRecords<byte[], byte[]> records = inferConsumer.fetchRecords();
    if (records.isEmpty()) {
        return Optional.empty();
    }
    final List<byte[]> keys = new ArrayList<>();
    final List<byte[]> values = new ArrayList<>();
    for (final ConsumerRecord<byte[], byte[]> record : records) {
        keys.add(record.key());
        values.add(record.value());
    }
    final QuickTopicType keyType = inferSerDe(keys);
    final QuickTopicType valueType = inferSerDe(values);
    return Optional.of(new TopicSchemaTypes(keyType, valueType));
}
Also used : TopicSchemaTypes(com.bakdata.quick.common.api.model.TopicSchemaTypes) ArrayList(java.util.ArrayList) QuickTopicType(com.bakdata.quick.common.type.QuickTopicType)

Example 2 with QuickTopicType

use of com.bakdata.quick.common.type.QuickTopicType in project quick by bakdata.

the class GraphQLUtils method getRootType.

/**
 * Returns the name of the type in a topic backed by this registry.
 *
 * @param registry registry backing a topic
 * @return name of the type
 */
public static String getRootType(final QuickTopicType type, final TypeDefinitionRegistry registry) {
    if (type != QuickTopicType.SCHEMA) {
        return Objects.requireNonNull(TYPE_TO_GQL_SCALAR_NAME_MAP.get(type));
    }
    final Map<String, Integer> objectFiledCount = new HashMap<>();
    final List<String> skippableTypes = new ArrayList<>();
    for (final Entry<String, TypeDefinition> next : registry.types().entrySet()) {
        final String typeName = next.getKey();
        final TypeDefinition typeDefinition = next.getValue();
        if (typeName.equals(QUERY_TYPE) || skippableTypes.contains(typeName)) {
            // we can skip the query type and type we've already seen
            continue;
        }
        if (typeDefinition instanceof ObjectTypeDefinition) {
            final ObjectTypeDefinition objectTypeDefinition = (ObjectTypeDefinition) typeDefinition;
            final List<FieldDefinition> fieldDefinitions = objectTypeDefinition.getFieldDefinitions();
            final int objectTypesAsFieldsCount = (int) fieldDefinitions.stream().map(FieldDefinition::getType).map(GraphQLUtils::getNameOfType).filter(name -> registry.types().containsKey(name)).peek(// seen as field => can not be root object
            skippableTypes::add).count();
            objectFiledCount.put(typeName, objectTypesAsFieldsCount);
        } else {
            objectFiledCount.put(typeName, 0);
        }
    }
    return Collections.max(objectFiledCount.entrySet(), Entry.comparingByValue()).getKey();
}
Also used : ObjectTypeDefinition(graphql.language.ObjectTypeDefinition) FieldDefinition(graphql.language.FieldDefinition) TypeDefinitionRegistry(graphql.schema.idl.TypeDefinitionRegistry) HashMap(java.util.HashMap) BadArgumentException(com.bakdata.quick.common.exception.BadArgumentException) TypeDefinition(graphql.language.TypeDefinition) ArrayList(java.util.ArrayList) EnumTypeDefinition(graphql.language.EnumTypeDefinition) Objects(java.util.Objects) Type(graphql.language.Type) Scalars(graphql.Scalars) List(java.util.List) EnumValueDefinition(graphql.language.EnumValueDefinition) Map(java.util.Map) Entry(java.util.Map.Entry) NonNullType(graphql.language.NonNullType) TypeName(graphql.language.TypeName) ListType(graphql.language.ListType) Collections(java.util.Collections) QuickTopicType(com.bakdata.quick.common.type.QuickTopicType) ObjectTypeDefinition(graphql.language.ObjectTypeDefinition) HashMap(java.util.HashMap) FieldDefinition(graphql.language.FieldDefinition) ArrayList(java.util.ArrayList) ObjectTypeDefinition(graphql.language.ObjectTypeDefinition) TypeDefinition(graphql.language.TypeDefinition) EnumTypeDefinition(graphql.language.EnumTypeDefinition)

Example 3 with QuickTopicType

use of com.bakdata.quick.common.type.QuickTopicType in project quick by bakdata.

the class QuickTopicTypeService method fromTopicData.

private <K, V> Single<QuickTopicData<K, V>> fromTopicData(final TopicData topicData) {
    final QuickTopicType keyType = topicData.getKeyType();
    final QuickTopicType valueType = topicData.getValueType();
    final Serde<K> keySerde = keyType.getSerde();
    final Serde<V> valueSerde = valueType.getSerde();
    final TypeResolver<K> keyResolver = keyType.getTypeResolver();
    final TypeResolver<V> valueResolver = valueType.getTypeResolver();
    final Map<String, String> configs = Map.of("schema.registry.url", this.schemaRegistryUrl);
    keySerde.configure(configs, true);
    valueSerde.configure(configs, false);
    final String topic = topicData.getName();
    // configure key and value resolver - only required if we handle avro
    final Completable configureResolver = Completable.mergeArray(this.configureResolver(keyType, KEY.asSubject(topic), keyResolver), this.configureResolver(valueType, VALUE.asSubject(topic), valueResolver));
    final QuickData<K> keyInfo = new QuickData<>(keyType, keySerde, keyResolver);
    final QuickData<V> valueInfo = new QuickData<>(valueType, valueSerde, valueResolver);
    final QuickTopicData<K, V> info = new QuickTopicData<>(topic, topicData.getWriteType(), keyInfo, valueInfo);
    // first we need to configure the resolver, then we can publish the info
    return configureResolver.andThen(Single.just(info));
}
Also used : Completable(io.reactivex.Completable) QuickData(com.bakdata.quick.common.type.QuickTopicData.QuickData) QuickTopicData(com.bakdata.quick.common.type.QuickTopicData) QuickTopicType(com.bakdata.quick.common.type.QuickTopicType)

Example 4 with QuickTopicType

use of com.bakdata.quick.common.type.QuickTopicType in project quick by bakdata.

the class KafkaTopicService method createTopic.

@SuppressWarnings("RxReturnValueIgnored")
@Override
public Completable createTopic(final String name, final QuickTopicType keyType, final QuickTopicType valueType, final TopicCreationData requestData) {
    log.info("Create new topic {} with data {}", name, requestData);
    // we don't need the cache, so make sure we get the current information
    this.schemaRegistryClient.reset();
    // first, check if topic might already exist in topic registry or kafka itself and then make also sure there
    final Completable kafkaStateCheck = Completable.mergeArray(this.checkKafka(name), this.checkTopicRegistry(name));
    final Single<Optional<QuickSchemas>> keySchema = this.getQuickSchemas(requestData.getKeySchema()).cache();
    final Single<Optional<QuickSchemas>> valueSchema = this.getQuickSchemas(requestData.getValueSchema()).cache();
    final Completable schemaRegistryCheck = Completable.defer(() -> Completable.mergeArray(keySchema.flatMapCompletable(schema -> this.checkSchemaRegistry(name + "-key", schema)), valueSchema.flatMapCompletable(schema -> this.checkSchemaRegistry(name + "-value", schema))));
    final Completable stateCheck = kafkaStateCheck.andThen(schemaRegistryCheck);
    // create topic in kafka and deploy a mirror application
    final Completable kafkaTopicCreation = this.createKafkaTopic(name);
    final Completable mirrorCreation = this.createMirror(name, requestData.getRetentionTime());
    // default to mutable topic write type
    final TopicWriteType writeType = Objects.requireNonNullElse(requestData.getWriteType(), TopicWriteType.MUTABLE);
    // register at topic registry (value schema can be nullable)
    // todo evaluate whether the schema should be part of the topic registry
    final Completable topicRegister = Completable.defer(() -> {
        log.debug("Register subject '{}' with topic registry", name);
        return valueSchema.flatMapCompletable(schema -> {
            final String graphQLSchema = schema.map(QuickSchemas::getGraphQLSchema).orElse(null);
            final TopicData topicData = new TopicData(name, writeType, keyType, valueType, graphQLSchema);
            return this.topicRegistryClient.register(name, topicData);
        });
    });
    // register potential avro schema with the schema registry
    final Completable keyRegister = keySchema.flatMapCompletable(schemas -> this.registerSchema(name, schemas, KEY));
    final Completable valueRegister = valueSchema.flatMapCompletable(schemas -> this.registerSchema(name, schemas, VALUE));
    final Completable creationOperations = Completable.mergeArray(kafkaTopicCreation, mirrorCreation, topicRegister, keyRegister, valueRegister);
    return stateCheck.andThen(creationOperations.doOnError(ignored -> this.deleteTopic(name)));
}
Also used : MirrorCreationData(com.bakdata.quick.common.api.model.manager.creation.MirrorCreationData) TopicRegistryClient(com.bakdata.quick.common.api.client.TopicRegistryClient) SchemaRegistryClient(io.confluent.kafka.schemaregistry.client.SchemaRegistryClient) KEY(com.bakdata.quick.common.api.model.KeyValueEnum.KEY) TopicWriteType(com.bakdata.quick.common.api.model.TopicWriteType) Completable(io.reactivex.Completable) GatewayClient(com.bakdata.quick.common.api.client.GatewayClient) Singleton(javax.inject.Singleton) Single(io.reactivex.Single) Supplier(java.util.function.Supplier) GraphQLToAvroConverter(com.bakdata.quick.manager.graphql.GraphQLToAvroConverter) Value(lombok.Value) AdminClient(org.apache.kafka.clients.admin.AdminClient) KafkaConfig(com.bakdata.quick.common.config.KafkaConfig) MirrorService(com.bakdata.quick.manager.mirror.MirrorService) Duration(java.time.Duration) KeyValueEnum(com.bakdata.quick.common.api.model.KeyValueEnum) QuickTopicConfig(com.bakdata.quick.common.config.QuickTopicConfig) QuickTopicType(com.bakdata.quick.common.type.QuickTopicType) Schema(org.apache.avro.Schema) ResourcePrefix(com.bakdata.quick.manager.k8s.resource.QuickResources.ResourcePrefix) NewTopic(org.apache.kafka.clients.admin.NewTopic) GatewaySchema(com.bakdata.quick.common.api.model.manager.GatewaySchema) IOException(java.io.IOException) VALUE(com.bakdata.quick.common.api.model.KeyValueEnum.VALUE) BadArgumentException(com.bakdata.quick.common.exception.BadArgumentException) TopicCreationData(com.bakdata.quick.common.api.model.manager.creation.TopicCreationData) Objects(java.util.Objects) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) TopicData(com.bakdata.quick.common.api.model.TopicData) GatewayService(com.bakdata.quick.manager.gateway.GatewayService) Optional(java.util.Optional) CachedSchemaRegistryClient(io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient) Completable(io.reactivex.Completable) Optional(java.util.Optional) TopicData(com.bakdata.quick.common.api.model.TopicData) TopicWriteType(com.bakdata.quick.common.api.model.TopicWriteType)

Aggregations

QuickTopicType (com.bakdata.quick.common.type.QuickTopicType)4 BadArgumentException (com.bakdata.quick.common.exception.BadArgumentException)2 Completable (io.reactivex.Completable)2 ArrayList (java.util.ArrayList)2 List (java.util.List)2 Objects (java.util.Objects)2 GatewayClient (com.bakdata.quick.common.api.client.GatewayClient)1 TopicRegistryClient (com.bakdata.quick.common.api.client.TopicRegistryClient)1 KeyValueEnum (com.bakdata.quick.common.api.model.KeyValueEnum)1 KEY (com.bakdata.quick.common.api.model.KeyValueEnum.KEY)1 VALUE (com.bakdata.quick.common.api.model.KeyValueEnum.VALUE)1 TopicData (com.bakdata.quick.common.api.model.TopicData)1 TopicSchemaTypes (com.bakdata.quick.common.api.model.TopicSchemaTypes)1 TopicWriteType (com.bakdata.quick.common.api.model.TopicWriteType)1 GatewaySchema (com.bakdata.quick.common.api.model.manager.GatewaySchema)1 MirrorCreationData (com.bakdata.quick.common.api.model.manager.creation.MirrorCreationData)1 TopicCreationData (com.bakdata.quick.common.api.model.manager.creation.TopicCreationData)1 KafkaConfig (com.bakdata.quick.common.config.KafkaConfig)1 QuickTopicConfig (com.bakdata.quick.common.config.QuickTopicConfig)1 QuickTopicData (com.bakdata.quick.common.type.QuickTopicData)1