use of com.bakdata.quick.common.type.QuickTopicType in project quick by bakdata.
the class InferTopicTypeService method inferSerDes.
private Optional<TopicSchemaTypes> inferSerDes(final String topic) {
final InferConsumer inferConsumer = new InferConsumer(this.bootstrapServer, topic, this.appId);
final ConsumerRecords<byte[], byte[]> records = inferConsumer.fetchRecords();
if (records.isEmpty()) {
return Optional.empty();
}
final List<byte[]> keys = new ArrayList<>();
final List<byte[]> values = new ArrayList<>();
for (final ConsumerRecord<byte[], byte[]> record : records) {
keys.add(record.key());
values.add(record.value());
}
final QuickTopicType keyType = inferSerDe(keys);
final QuickTopicType valueType = inferSerDe(values);
return Optional.of(new TopicSchemaTypes(keyType, valueType));
}
use of com.bakdata.quick.common.type.QuickTopicType in project quick by bakdata.
the class GraphQLUtils method getRootType.
/**
* Returns the name of the type in a topic backed by this registry.
*
* @param registry registry backing a topic
* @return name of the type
*/
public static String getRootType(final QuickTopicType type, final TypeDefinitionRegistry registry) {
if (type != QuickTopicType.SCHEMA) {
return Objects.requireNonNull(TYPE_TO_GQL_SCALAR_NAME_MAP.get(type));
}
final Map<String, Integer> objectFiledCount = new HashMap<>();
final List<String> skippableTypes = new ArrayList<>();
for (final Entry<String, TypeDefinition> next : registry.types().entrySet()) {
final String typeName = next.getKey();
final TypeDefinition typeDefinition = next.getValue();
if (typeName.equals(QUERY_TYPE) || skippableTypes.contains(typeName)) {
// we can skip the query type and type we've already seen
continue;
}
if (typeDefinition instanceof ObjectTypeDefinition) {
final ObjectTypeDefinition objectTypeDefinition = (ObjectTypeDefinition) typeDefinition;
final List<FieldDefinition> fieldDefinitions = objectTypeDefinition.getFieldDefinitions();
final int objectTypesAsFieldsCount = (int) fieldDefinitions.stream().map(FieldDefinition::getType).map(GraphQLUtils::getNameOfType).filter(name -> registry.types().containsKey(name)).peek(// seen as field => can not be root object
skippableTypes::add).count();
objectFiledCount.put(typeName, objectTypesAsFieldsCount);
} else {
objectFiledCount.put(typeName, 0);
}
}
return Collections.max(objectFiledCount.entrySet(), Entry.comparingByValue()).getKey();
}
use of com.bakdata.quick.common.type.QuickTopicType in project quick by bakdata.
the class QuickTopicTypeService method fromTopicData.
private <K, V> Single<QuickTopicData<K, V>> fromTopicData(final TopicData topicData) {
final QuickTopicType keyType = topicData.getKeyType();
final QuickTopicType valueType = topicData.getValueType();
final Serde<K> keySerde = keyType.getSerde();
final Serde<V> valueSerde = valueType.getSerde();
final TypeResolver<K> keyResolver = keyType.getTypeResolver();
final TypeResolver<V> valueResolver = valueType.getTypeResolver();
final Map<String, String> configs = Map.of("schema.registry.url", this.schemaRegistryUrl);
keySerde.configure(configs, true);
valueSerde.configure(configs, false);
final String topic = topicData.getName();
// configure key and value resolver - only required if we handle avro
final Completable configureResolver = Completable.mergeArray(this.configureResolver(keyType, KEY.asSubject(topic), keyResolver), this.configureResolver(valueType, VALUE.asSubject(topic), valueResolver));
final QuickData<K> keyInfo = new QuickData<>(keyType, keySerde, keyResolver);
final QuickData<V> valueInfo = new QuickData<>(valueType, valueSerde, valueResolver);
final QuickTopicData<K, V> info = new QuickTopicData<>(topic, topicData.getWriteType(), keyInfo, valueInfo);
// first we need to configure the resolver, then we can publish the info
return configureResolver.andThen(Single.just(info));
}
use of com.bakdata.quick.common.type.QuickTopicType in project quick by bakdata.
the class KafkaTopicService method createTopic.
@SuppressWarnings("RxReturnValueIgnored")
@Override
public Completable createTopic(final String name, final QuickTopicType keyType, final QuickTopicType valueType, final TopicCreationData requestData) {
log.info("Create new topic {} with data {}", name, requestData);
// we don't need the cache, so make sure we get the current information
this.schemaRegistryClient.reset();
// first, check if topic might already exist in topic registry or kafka itself and then make also sure there
final Completable kafkaStateCheck = Completable.mergeArray(this.checkKafka(name), this.checkTopicRegistry(name));
final Single<Optional<QuickSchemas>> keySchema = this.getQuickSchemas(requestData.getKeySchema()).cache();
final Single<Optional<QuickSchemas>> valueSchema = this.getQuickSchemas(requestData.getValueSchema()).cache();
final Completable schemaRegistryCheck = Completable.defer(() -> Completable.mergeArray(keySchema.flatMapCompletable(schema -> this.checkSchemaRegistry(name + "-key", schema)), valueSchema.flatMapCompletable(schema -> this.checkSchemaRegistry(name + "-value", schema))));
final Completable stateCheck = kafkaStateCheck.andThen(schemaRegistryCheck);
// create topic in kafka and deploy a mirror application
final Completable kafkaTopicCreation = this.createKafkaTopic(name);
final Completable mirrorCreation = this.createMirror(name, requestData.getRetentionTime());
// default to mutable topic write type
final TopicWriteType writeType = Objects.requireNonNullElse(requestData.getWriteType(), TopicWriteType.MUTABLE);
// register at topic registry (value schema can be nullable)
// todo evaluate whether the schema should be part of the topic registry
final Completable topicRegister = Completable.defer(() -> {
log.debug("Register subject '{}' with topic registry", name);
return valueSchema.flatMapCompletable(schema -> {
final String graphQLSchema = schema.map(QuickSchemas::getGraphQLSchema).orElse(null);
final TopicData topicData = new TopicData(name, writeType, keyType, valueType, graphQLSchema);
return this.topicRegistryClient.register(name, topicData);
});
});
// register potential avro schema with the schema registry
final Completable keyRegister = keySchema.flatMapCompletable(schemas -> this.registerSchema(name, schemas, KEY));
final Completable valueRegister = valueSchema.flatMapCompletable(schemas -> this.registerSchema(name, schemas, VALUE));
final Completable creationOperations = Completable.mergeArray(kafkaTopicCreation, mirrorCreation, topicRegister, keyRegister, valueRegister);
return stateCheck.andThen(creationOperations.doOnError(ignored -> this.deleteTopic(name)));
}
Aggregations