Examples with TopicCreationData com.bakdata.quick.common.api.model.manager.creation.TopicCreationData used on opensource projects

Example 6 with TopicCreationData

use of com.bakdata.quick.common.api.model.manager.creation.TopicCreationData in project quick by bakdata.

the class KafkaTopicServiceTest method shouldNotCreateTopicIfSubjectExists.

@Test
@Disabled("Compatibility test not supported in SR mock yet")
void shouldNotCreateTopicIfSubjectExists() throws RestClientException, IOException {
    final String topicName = UUID.randomUUID().toString();
    this.successfulMock();
    this.schemaRegistry.registerValueSchema(topicName, this.graphQLToAvroConverter.convertToSchema(SCHEMA));
    when(this.gatewayService.getGateway(GATEWAY_SCHEMA.getGateway())).thenReturn(Single.just(new GatewayDescription("test", 1, "latest")));
    when(this.gatewayClient.getWriteSchema(anyString(), anyString())).thenReturn(Single.just(new SchemaData(SCHEMA)));
    final TopicCreationData requestData = new TopicCreationData(TopicWriteType.MUTABLE, GATEWAY_SCHEMA, null, null);
    final Throwable throwable = this.topicService.createTopic(topicName, QuickTopicType.DOUBLE, QuickTopicType.SCHEMA, requestData).blockingGet();
    assertThat(throwable).isNotNull().isExactlyInstanceOf(BadArgumentException.class).extracting(Throwable::getMessage).asString().endsWith("already exists");
    assertThat(kafkaCluster.exists(topicName)).isFalse();
    assertThat(this.topicRegistryClient.topicDataExists(topicName).blockingGet()).isFalse();
    assertThat(this.schemaRegistry.getSchemaRegistryClient().getAllSubjects()).isEmpty();
}

Example 7 with TopicCreationData

use of com.bakdata.quick.common.api.model.manager.creation.TopicCreationData in project quick by bakdata.

the class KafkaTopicServiceTest method shouldNotCreateTopicThatAlreadyExistsInRegistry.

@Test
void shouldNotCreateTopicThatAlreadyExistsInRegistry() throws RestClientException, IOException {
    final String topicName = UUID.randomUUID().toString();
    this.successfulMock();
    final TopicCreationData requestData = new TopicCreationData(TopicWriteType.MUTABLE, null, null, null);
    this.topicRegistryClient.register(topicName, new TopicData(topicName, TopicWriteType.MUTABLE, null, null, null)).blockingAwait();
    assertThat(this.topicRegistryClient.topicDataExists(topicName).blockingGet()).isTrue();
    final Throwable exception = this.topicService.createTopic(topicName, QuickTopicType.DOUBLE, QuickTopicType.DOUBLE, requestData).blockingGet();
    final String expectedErrorMsg = String.format("Topic \"%s\" already exists", topicName);
    assertThat(exception).isExactlyInstanceOf(BadArgumentException.class).extracting(Throwable::getMessage).asString().startsWith(expectedErrorMsg);
    assertThat(kafkaCluster.exists(topicName)).isFalse();
    assertThat(this.schemaRegistry.getSchemaRegistryClient().getAllSubjects()).isEmpty();
}

Example 8 with TopicCreationData

use of com.bakdata.quick.common.api.model.manager.creation.TopicCreationData in project quick by bakdata.

the class KafkaTopicService method createTopic.

@SuppressWarnings("RxReturnValueIgnored")
@Override
public Completable createTopic(final String name, final QuickTopicType keyType, final QuickTopicType valueType, final TopicCreationData requestData) {
    log.info("Create new topic {} with data {}", name, requestData);
    // we don't need the cache, so make sure we get the current information
    this.schemaRegistryClient.reset();
    // first, check if topic might already exist in topic registry or kafka itself and then make also sure there
    final Completable kafkaStateCheck = Completable.mergeArray(this.checkKafka(name), this.checkTopicRegistry(name));
    final Single<Optional<QuickSchemas>> keySchema = this.getQuickSchemas(requestData.getKeySchema()).cache();
    final Single<Optional<QuickSchemas>> valueSchema = this.getQuickSchemas(requestData.getValueSchema()).cache();
    final Completable schemaRegistryCheck = Completable.defer(() -> Completable.mergeArray(keySchema.flatMapCompletable(schema -> this.checkSchemaRegistry(name + "-key", schema)), valueSchema.flatMapCompletable(schema -> this.checkSchemaRegistry(name + "-value", schema))));
    final Completable stateCheck = kafkaStateCheck.andThen(schemaRegistryCheck);
    // create topic in kafka and deploy a mirror application
    final Completable kafkaTopicCreation = this.createKafkaTopic(name);
    final Completable mirrorCreation = this.createMirror(name, requestData.getRetentionTime());
    // default to mutable topic write type
    final TopicWriteType writeType = Objects.requireNonNullElse(requestData.getWriteType(), TopicWriteType.MUTABLE);
    // register at topic registry (value schema can be nullable)
    // todo evaluate whether the schema should be part of the topic registry
    final Completable topicRegister = Completable.defer(() -> {
        log.debug("Register subject '{}' with topic registry", name);
        return valueSchema.flatMapCompletable(schema -> {
            final String graphQLSchema = schema.map(QuickSchemas::getGraphQLSchema).orElse(null);
            final TopicData topicData = new TopicData(name, writeType, keyType, valueType, graphQLSchema);
            return this.topicRegistryClient.register(name, topicData);
        });
    });
    // register potential avro schema with the schema registry
    final Completable keyRegister = keySchema.flatMapCompletable(schemas -> this.registerSchema(name, schemas, KEY));
    final Completable valueRegister = valueSchema.flatMapCompletable(schemas -> this.registerSchema(name, schemas, VALUE));
    final Completable creationOperations = Completable.mergeArray(kafkaTopicCreation, mirrorCreation, topicRegister, keyRegister, valueRegister);
    return stateCheck.andThen(creationOperations.doOnError(ignored -> this.deleteTopic(name)));
}

Example 9 with TopicCreationData

use of com.bakdata.quick.common.api.model.manager.creation.TopicCreationData in project quick by bakdata.

the class KafkaTopicServiceTest method shouldNotCreateTopicThatAlreadyExists.

@Test
void shouldNotCreateTopicThatAlreadyExists() {
    final String topicName = UUID.randomUUID().toString();
    this.successfulMock();
    final TopicCreationData requestData = new TopicCreationData(TopicWriteType.MUTABLE, null, null, null);
    this.topicService.createTopic(topicName, QuickTopicType.DOUBLE, QuickTopicType.DOUBLE, requestData).blockingAwait();
    assertThat(kafkaCluster.exists(topicName)).isTrue();
    final Throwable exception = this.topicService.createTopic(topicName, QuickTopicType.DOUBLE, QuickTopicType.DOUBLE, requestData).blockingGet();
    final String expectedErrorMsg = String.format("Topic \"%s\" already exists", topicName);
    assertThat(exception).isExactlyInstanceOf(BadArgumentException.class).extracting(Throwable::getMessage).asString().startsWith(expectedErrorMsg);
}

Example 10 with TopicCreationData

use of com.bakdata.quick.common.api.model.manager.creation.TopicCreationData in project quick by bakdata.

the class KafkaTopicServiceTest method shouldThrowExceptionForNonExistingGateway.

@Test
void shouldThrowExceptionForNonExistingGateway() {
    final String topicName = UUID.randomUUID().toString();
    this.successfulMock();
    // error thrown when checking if gateway exists
    final Throwable error = new NotFoundException(String.format("Could not find resource %s", GATEWAY_SCHEMA.getGateway()));
    when(this.gatewayService.getGateway(GATEWAY_SCHEMA.getGateway())).thenReturn(Single.error(error));
    // resource doesn't exist, traefik cannot route it; This should not be called since we check existence before
    final HttpException clientException = new HttpClientResponseException("Not found", HttpResponse.notFound());
    when(this.gatewayClient.getWriteSchema(anyString(), anyString())).thenReturn(Single.error(clientException));
    final TopicCreationData requestData = new TopicCreationData(TopicWriteType.MUTABLE, GATEWAY_SCHEMA, null, null);
    final Completable completable = this.topicService.createTopic(topicName, QuickTopicType.DOUBLE, QuickTopicType.SCHEMA, requestData);
    final Throwable actual = completable.blockingGet();
    assertThat(actual).isNotNull().isInstanceOf(QuickException.class).hasMessageStartingWith("Could not find resource test");
    verify(this.gatewayClient, never()).getWriteSchema(anyString(), anyString());
}