Search in sources :

Example 41 with KafkaTopic

use of io.strimzi.api.kafka.model.KafkaTopic in project strimzi by strimzi.

the class MirrorMaker2IsolatedST method testMirrorMaker2TlsAndTlsClientAuth.

/**
 * Test mirroring messages by MirrorMaker 2.0 over tls transport using mutual tls auth
 */
@SuppressWarnings({ "checkstyle:MethodLength" })
@ParallelNamespaceTest
@Tag(ACCEPTANCE)
@KRaftNotSupported("UserOperator is not supported by KRaft mode and is used in this test case")
void testMirrorMaker2TlsAndTlsClientAuth(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext, clusterOperator.getDeploymentNamespace());
    String kafkaClusterSourceName = testStorage.getClusterName() + "-source";
    String kafkaClusterTargetName = testStorage.getClusterName() + "-target";
    String sourceMirroredTopicName = kafkaClusterSourceName + "." + testStorage.getTopicName();
    String kafkaUserSourceName = testStorage.getUserName() + "-source";
    String kafkaUserTargetName = testStorage.getUserName() + "-target";
    // Deploy source kafka with tls listener and mutual tls auth
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterSourceName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationTls()).build()).endKafka().endSpec().build());
    // Deploy target kafka with tls listener and mutual tls auth
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterTargetName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationTls()).build()).endKafka().endSpec().build());
    // Deploy topic
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, testStorage.getTopicName(), 3).build(), KafkaUserTemplates.tlsUser(kafkaClusterSourceName, kafkaUserSourceName).build(), KafkaUserTemplates.tlsUser(kafkaClusterTargetName, kafkaUserTargetName).build());
    // Initialize CertSecretSource with certificate and secret names for source
    CertSecretSource certSecretSource = new CertSecretSource();
    certSecretSource.setCertificate("ca.crt");
    certSecretSource.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterSourceName));
    // Initialize CertSecretSource with certificate and secret names for target
    CertSecretSource certSecretTarget = new CertSecretSource();
    certSecretTarget.setCertificate("ca.crt");
    certSecretTarget.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterTargetName));
    KafkaClients clients = new KafkaClientsBuilder().withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterSourceName)).withUserName(kafkaUserSourceName).withNamespaceName(testStorage.getNamespaceName()).withTopicName(testStorage.getTopicName()).withMessageCount(MESSAGE_COUNT).build();
    // Check brokers availability
    LOGGER.info("Messages exchange - topic {}, cluster {} and message count of {}", testStorage.getTopicName(), kafkaClusterSourceName, MESSAGE_COUNT);
    resourceManager.createResource(extensionContext, clients.producerTlsStrimzi(kafkaClusterSourceName), clients.consumerTlsStrimzi(kafkaClusterSourceName));
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    resourceManager.createResource(extensionContext, KafkaMirrorMaker2Templates.kafkaMirrorMaker2(testStorage.getClusterName(), kafkaClusterTargetName, kafkaClusterSourceName, 1, true).editSpec().editMatchingCluster(spec -> spec.getAlias().equals(kafkaClusterSourceName)).withNewKafkaClientAuthenticationTls().withNewCertificateAndKey().withSecretName(kafkaUserSourceName).withCertificate("user.crt").withKey("user.key").endCertificateAndKey().endKafkaClientAuthenticationTls().withNewTls().withTrustedCertificates(certSecretSource).endTls().endCluster().editMatchingCluster(spec -> spec.getAlias().equals(kafkaClusterTargetName)).withNewKafkaClientAuthenticationTls().withNewCertificateAndKey().withSecretName(kafkaUserTargetName).withCertificate("user.crt").withKey("user.key").endCertificateAndKey().endKafkaClientAuthenticationTls().withNewTls().withTrustedCertificates(certSecretTarget).endTls().endCluster().endSpec().build());
    LOGGER.info("Changing to mirrored topic - topic {}, cluster {}, user {}", sourceMirroredTopicName, kafkaClusterTargetName, kafkaClusterTargetName);
    clients = new KafkaClientsBuilder(clients).withTopicName(sourceMirroredTopicName).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterTargetName)).withUserName(kafkaUserTargetName).build();
    LOGGER.info("Now messages should be mirrored to target topic and cluster");
    resourceManager.createResource(extensionContext, clients.consumerTlsStrimzi(kafkaClusterTargetName));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Messages successfully mirrored");
    KafkaTopicUtils.waitForKafkaTopicCreation(testStorage.getNamespaceName(), sourceMirroredTopicName);
    KafkaTopic mirroredTopic = KafkaTopicResource.kafkaTopicClient().inNamespace(testStorage.getNamespaceName()).withName(sourceMirroredTopicName).get();
    assertThat(mirroredTopic.getSpec().getPartitions(), is(3));
    assertThat(mirroredTopic.getMetadata().getLabels().get(Labels.STRIMZI_CLUSTER_LABEL), is(kafkaClusterTargetName));
}
Also used : KafkaClientsBuilder(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder) KafkaListenerAuthenticationTls(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationTls) KafkaClients(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) KafkaTopic(io.strimzi.api.kafka.model.KafkaTopic) TestStorage(io.strimzi.systemtest.storage.TestStorage) Matchers.containsString(org.hamcrest.Matchers.containsString) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest) Tag(org.junit.jupiter.api.Tag)

Example 42 with KafkaTopic

use of io.strimzi.api.kafka.model.KafkaTopic in project strimzi by strimzi.

the class MirrorMaker2IsolatedST method testMirrorMaker2TlsAndScramSha512Auth.

/**
 * Test mirroring messages by MirrorMaker 2.0 over tls transport using scram-sha-512 auth
 */
@ParallelNamespaceTest
@KRaftNotSupported("UserOperator is not supported by KRaft mode and is used in this test case")
void testMirrorMaker2TlsAndScramSha512Auth(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext, clusterOperator.getDeploymentNamespace());
    String kafkaClusterSourceName = testStorage.getClusterName() + "-source";
    String kafkaClusterTargetName = testStorage.getClusterName() + "-target";
    String sourceMirroredTopicName = kafkaClusterSourceName + "." + testStorage.getTopicName();
    String kafkaUserSourceName = testStorage.getUserName() + "-source";
    String kafkaUserTargetName = testStorage.getUserName() + "-target";
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterSourceName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationScramSha512()).build()).endKafka().endSpec().build(), KafkaTemplates.kafkaEphemeral(kafkaClusterTargetName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationScramSha512()).build()).endKafka().endSpec().build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, testStorage.getTopicName(), 3).build(), KafkaUserTemplates.scramShaUser(kafkaClusterSourceName, kafkaUserSourceName).build(), KafkaUserTemplates.scramShaUser(kafkaClusterTargetName, kafkaUserTargetName).build());
    // Initialize PasswordSecretSource to set this as PasswordSecret in MirrorMaker2 spec
    PasswordSecretSource passwordSecretSource = new PasswordSecretSource();
    passwordSecretSource.setSecretName(kafkaUserSourceName);
    passwordSecretSource.setPassword("password");
    // Initialize PasswordSecretSource to set this as PasswordSecret in MirrorMaker2 spec
    PasswordSecretSource passwordSecretTarget = new PasswordSecretSource();
    passwordSecretTarget.setSecretName(kafkaUserTargetName);
    passwordSecretTarget.setPassword("password");
    // Initialize CertSecretSource with certificate and secret names for source
    CertSecretSource certSecretSource = new CertSecretSource();
    certSecretSource.setCertificate("ca.crt");
    certSecretSource.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterSourceName));
    // Initialize CertSecretSource with certificate and secret names for target
    CertSecretSource certSecretTarget = new CertSecretSource();
    certSecretTarget.setCertificate("ca.crt");
    certSecretTarget.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterTargetName));
    KafkaClients clients = new KafkaClientsBuilder().withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterSourceName)).withUserName(kafkaUserSourceName).withNamespaceName(testStorage.getNamespaceName()).withTopicName(testStorage.getTopicName()).withMessageCount(MESSAGE_COUNT).build();
    // Check brokers availability
    LOGGER.info("Messages exchange - topic {}, cluster {} and message count of {}", testStorage.getTopicName(), kafkaClusterSourceName, MESSAGE_COUNT);
    resourceManager.createResource(extensionContext, clients.producerScramShaTlsStrimzi(kafkaClusterSourceName), clients.consumerScramShaTlsStrimzi(kafkaClusterSourceName));
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    resourceManager.createResource(extensionContext, KafkaMirrorMaker2Templates.kafkaMirrorMaker2(testStorage.getClusterName(), kafkaClusterTargetName, kafkaClusterSourceName, 1, true).editSpec().editMatchingCluster(spec -> spec.getAlias().equals(kafkaClusterSourceName)).withNewKafkaClientAuthenticationScramSha512().withUsername(kafkaUserSourceName).withPasswordSecret(passwordSecretSource).endKafkaClientAuthenticationScramSha512().withNewTls().withTrustedCertificates(certSecretSource).endTls().endCluster().editMatchingCluster(spec -> spec.getAlias().equals(kafkaClusterTargetName)).withNewKafkaClientAuthenticationScramSha512().withUsername(kafkaUserTargetName).withPasswordSecret(passwordSecretTarget).endKafkaClientAuthenticationScramSha512().withNewTls().withTrustedCertificates(certSecretTarget).endTls().endCluster().endSpec().build());
    LOGGER.info("Changing to mirrored topic - topic {}, cluster {}, user {}", sourceMirroredTopicName, kafkaClusterTargetName, kafkaClusterTargetName);
    clients = new KafkaClientsBuilder(clients).withTopicName(sourceMirroredTopicName).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterTargetName)).withUserName(kafkaUserTargetName).build();
    LOGGER.info("Now messages should be mirrored to target topic and cluster");
    resourceManager.createResource(extensionContext, clients.consumerScramShaTlsStrimzi(kafkaClusterTargetName));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Messages successfully mirrored");
    KafkaTopicUtils.waitForKafkaTopicCreation(testStorage.getNamespaceName(), sourceMirroredTopicName);
    KafkaTopic mirroredTopic = KafkaTopicResource.kafkaTopicClient().inNamespace(testStorage.getNamespaceName()).withName(sourceMirroredTopicName).get();
    assertThat(mirroredTopic.getSpec().getPartitions(), is(3));
    assertThat(mirroredTopic.getMetadata().getLabels().get(Labels.STRIMZI_CLUSTER_LABEL), is(kafkaClusterTargetName));
}
Also used : KafkaListenerAuthenticationScramSha512(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationScramSha512) KafkaClientsBuilder(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder) LabelSelector(io.fabric8.kubernetes.api.model.LabelSelector) KafkaResource(io.strimzi.systemtest.resources.crd.KafkaResource) KubeClusterResource.cmdKubeClient(io.strimzi.test.k8s.KubeClusterResource.cmdKubeClient) KafkaTopicUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaTopicUtils) KafkaListenerAuthenticationTls(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationTls) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) KafkaResources(io.strimzi.api.kafka.model.KafkaResources) Ca(io.strimzi.operator.cluster.model.Ca) BeforeAll(org.junit.jupiter.api.BeforeAll) Map(java.util.Map) Tag(org.junit.jupiter.api.Tag) StUtils(io.strimzi.systemtest.utils.StUtils) BeforeAllOnce(io.strimzi.systemtest.BeforeAllOnce) RollingUpdateUtils(io.strimzi.systemtest.utils.RollingUpdateUtils) ACCEPTANCE(io.strimzi.systemtest.Constants.ACCEPTANCE) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) IsolatedSuite(io.strimzi.systemtest.annotations.IsolatedSuite) KafkaClients(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients) KafkaMirrorMaker2Templates(io.strimzi.systemtest.templates.crd.KafkaMirrorMaker2Templates) INTERNAL_CLIENTS_USED(io.strimzi.systemtest.Constants.INTERNAL_CLIENTS_USED) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) PasswordSecretSource(io.strimzi.api.kafka.model.PasswordSecretSource) ClientUtils(io.strimzi.systemtest.utils.ClientUtils) KafkaMirrorMaker2Resources(io.strimzi.api.kafka.model.KafkaMirrorMaker2Resources) KafkaMirrorMaker2(io.strimzi.api.kafka.model.KafkaMirrorMaker2) KafkaMirrorMaker2Status(io.strimzi.api.kafka.model.status.KafkaMirrorMaker2Status) List(java.util.List) Labels(io.strimzi.operator.common.model.Labels) Logger(org.apache.logging.log4j.Logger) KafkaTopicTemplates(io.strimzi.systemtest.templates.crd.KafkaTopicTemplates) KafkaMirrorMaker2Utils(io.strimzi.systemtest.utils.kafkaUtils.KafkaMirrorMaker2Utils) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Secret(io.fabric8.kubernetes.api.model.Secret) Matchers.is(org.hamcrest.Matchers.is) KafkaTopicResource(io.strimzi.systemtest.resources.crd.KafkaTopicResource) SCALABILITY(io.strimzi.systemtest.Constants.SCALABILITY) Matchers.containsString(org.hamcrest.Matchers.containsString) Assertions.assertDoesNotThrow(org.junit.jupiter.api.Assertions.assertDoesNotThrow) AbstractST(io.strimzi.systemtest.AbstractST) Environment(io.strimzi.systemtest.Environment) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) CoreMatchers.equalTo(org.hamcrest.CoreMatchers.equalTo) CoreMatchers.not(org.hamcrest.CoreMatchers.not) HashMap(java.util.HashMap) ExtensionContext(org.junit.jupiter.api.extension.ExtensionContext) TestStorage(io.strimzi.systemtest.storage.TestStorage) SecretUtils(io.strimzi.systemtest.utils.kubeUtils.objects.SecretUtils) PodUtils(io.strimzi.systemtest.utils.kubeUtils.objects.PodUtils) JsonPathMatchers.hasJsonPath(org.valid4j.matchers.jsonpath.JsonPathMatchers.hasJsonPath) DeploymentStrategy(io.strimzi.api.kafka.model.template.DeploymentStrategy) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) TestUtils(io.strimzi.test.TestUtils) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) KafkaListenerAuthenticationScramSha512(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationScramSha512) MIRROR_MAKER2(io.strimzi.systemtest.Constants.MIRROR_MAKER2) KafkaTemplates(io.strimzi.systemtest.templates.crd.KafkaTemplates) Ready(io.strimzi.systemtest.enums.CustomResourceStatus.Ready) KafkaUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaUtils) Constants(io.strimzi.systemtest.Constants) KafkaClientsBuilder(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest) SetupClusterOperator(io.strimzi.systemtest.resources.operator.SetupClusterOperator) KafkaCmdClient(io.strimzi.systemtest.cli.KafkaCmdClient) KubeClusterResource.kubeClient(io.strimzi.test.k8s.KubeClusterResource.kubeClient) KafkaTopic(io.strimzi.api.kafka.model.KafkaTopic) DeploymentUtils(io.strimzi.systemtest.utils.kubeUtils.controllers.DeploymentUtils) Matchers.hasItem(org.hamcrest.Matchers.hasItem) KafkaListenerType(io.strimzi.api.kafka.model.listener.arraylistener.KafkaListenerType) KafkaUserTemplates(io.strimzi.systemtest.templates.crd.KafkaUserTemplates) CONNECT_COMPONENTS(io.strimzi.systemtest.Constants.CONNECT_COMPONENTS) KafkaMirrorMaker2Resource(io.strimzi.systemtest.resources.crd.KafkaMirrorMaker2Resource) SecretBuilder(io.fabric8.kubernetes.api.model.SecretBuilder) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) REGRESSION(io.strimzi.systemtest.Constants.REGRESSION) KafkaClients(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) KafkaTopic(io.strimzi.api.kafka.model.KafkaTopic) PasswordSecretSource(io.strimzi.api.kafka.model.PasswordSecretSource) TestStorage(io.strimzi.systemtest.storage.TestStorage) Matchers.containsString(org.hamcrest.Matchers.containsString) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)

Example 43 with KafkaTopic

use of io.strimzi.api.kafka.model.KafkaTopic in project strimzi-kafka-operator by strimzi.

the class CruiseControlUtils method verifyThatCruiseControlSamplesTopicsArePresent.

public static void verifyThatCruiseControlSamplesTopicsArePresent(String namespaceName, long timeout) {
    final int numberOfPartitionsSamplesTopic = 32;
    final int numberOfReplicasSamplesTopic = 2;
    TestUtils.waitFor("Verify that kafka contains cruise control topics with related configuration.", Constants.GLOBAL_POLL_INTERVAL, timeout, () -> {
        KafkaTopic modelTrainingSamples = KafkaTopicResource.kafkaTopicClient().inNamespace(namespaceName).withName(CRUISE_CONTROL_MODEL_TRAINING_SAMPLES_TOPIC).get();
        KafkaTopic partitionsMetricsSamples = KafkaTopicResource.kafkaTopicClient().inNamespace(namespaceName).withName(CRUISE_CONTROL_PARTITION_METRICS_SAMPLES_TOPIC).get();
        if (modelTrainingSamples != null && partitionsMetricsSamples != null) {
            boolean hasTopicCorrectPartitionsCount = modelTrainingSamples.getSpec().getPartitions() == numberOfPartitionsSamplesTopic && partitionsMetricsSamples.getSpec().getPartitions() == numberOfPartitionsSamplesTopic;
            boolean hasTopicCorrectReplicasCount = modelTrainingSamples.getSpec().getReplicas() == numberOfReplicasSamplesTopic && partitionsMetricsSamples.getSpec().getReplicas() == numberOfReplicasSamplesTopic;
            return hasTopicCorrectPartitionsCount && hasTopicCorrectReplicasCount;
        }
        LOGGER.debug("One of the samples {}, {} topics are not present", CRUISE_CONTROL_MODEL_TRAINING_SAMPLES_TOPIC, CRUISE_CONTROL_PARTITION_METRICS_SAMPLES_TOPIC);
        return false;
    });
}
Also used : KafkaTopic(io.strimzi.api.kafka.model.KafkaTopic)

Example 44 with KafkaTopic

use of io.strimzi.api.kafka.model.KafkaTopic in project strimzi-kafka-operator by strimzi.

the class MirrorMaker2IsolatedST method testMirrorMaker2TlsAndTlsClientAuth.

/**
 * Test mirroring messages by MirrorMaker 2.0 over tls transport using mutual tls auth
 */
@SuppressWarnings({ "checkstyle:MethodLength" })
@ParallelNamespaceTest
@Tag(ACCEPTANCE)
@KRaftNotSupported("UserOperator is not supported by KRaft mode and is used in this test case")
void testMirrorMaker2TlsAndTlsClientAuth(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext, clusterOperator.getDeploymentNamespace());
    String kafkaClusterSourceName = testStorage.getClusterName() + "-source";
    String kafkaClusterTargetName = testStorage.getClusterName() + "-target";
    String sourceMirroredTopicName = kafkaClusterSourceName + "." + testStorage.getTopicName();
    String kafkaUserSourceName = testStorage.getUserName() + "-source";
    String kafkaUserTargetName = testStorage.getUserName() + "-target";
    // Deploy source kafka with tls listener and mutual tls auth
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterSourceName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationTls()).build()).endKafka().endSpec().build());
    // Deploy target kafka with tls listener and mutual tls auth
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterTargetName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationTls()).build()).endKafka().endSpec().build());
    // Deploy topic
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, testStorage.getTopicName(), 3).build(), KafkaUserTemplates.tlsUser(kafkaClusterSourceName, kafkaUserSourceName).build(), KafkaUserTemplates.tlsUser(kafkaClusterTargetName, kafkaUserTargetName).build());
    // Initialize CertSecretSource with certificate and secret names for source
    CertSecretSource certSecretSource = new CertSecretSource();
    certSecretSource.setCertificate("ca.crt");
    certSecretSource.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterSourceName));
    // Initialize CertSecretSource with certificate and secret names for target
    CertSecretSource certSecretTarget = new CertSecretSource();
    certSecretTarget.setCertificate("ca.crt");
    certSecretTarget.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterTargetName));
    KafkaClients clients = new KafkaClientsBuilder().withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterSourceName)).withUserName(kafkaUserSourceName).withNamespaceName(testStorage.getNamespaceName()).withTopicName(testStorage.getTopicName()).withMessageCount(MESSAGE_COUNT).build();
    // Check brokers availability
    LOGGER.info("Messages exchange - topic {}, cluster {} and message count of {}", testStorage.getTopicName(), kafkaClusterSourceName, MESSAGE_COUNT);
    resourceManager.createResource(extensionContext, clients.producerTlsStrimzi(kafkaClusterSourceName), clients.consumerTlsStrimzi(kafkaClusterSourceName));
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    resourceManager.createResource(extensionContext, KafkaMirrorMaker2Templates.kafkaMirrorMaker2(testStorage.getClusterName(), kafkaClusterTargetName, kafkaClusterSourceName, 1, true).editSpec().editMatchingCluster(spec -> spec.getAlias().equals(kafkaClusterSourceName)).withNewKafkaClientAuthenticationTls().withNewCertificateAndKey().withSecretName(kafkaUserSourceName).withCertificate("user.crt").withKey("user.key").endCertificateAndKey().endKafkaClientAuthenticationTls().withNewTls().withTrustedCertificates(certSecretSource).endTls().endCluster().editMatchingCluster(spec -> spec.getAlias().equals(kafkaClusterTargetName)).withNewKafkaClientAuthenticationTls().withNewCertificateAndKey().withSecretName(kafkaUserTargetName).withCertificate("user.crt").withKey("user.key").endCertificateAndKey().endKafkaClientAuthenticationTls().withNewTls().withTrustedCertificates(certSecretTarget).endTls().endCluster().endSpec().build());
    LOGGER.info("Changing to mirrored topic - topic {}, cluster {}, user {}", sourceMirroredTopicName, kafkaClusterTargetName, kafkaClusterTargetName);
    clients = new KafkaClientsBuilder(clients).withTopicName(sourceMirroredTopicName).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterTargetName)).withUserName(kafkaUserTargetName).build();
    LOGGER.info("Now messages should be mirrored to target topic and cluster");
    resourceManager.createResource(extensionContext, clients.consumerTlsStrimzi(kafkaClusterTargetName));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Messages successfully mirrored");
    KafkaTopicUtils.waitForKafkaTopicCreation(testStorage.getNamespaceName(), sourceMirroredTopicName);
    KafkaTopic mirroredTopic = KafkaTopicResource.kafkaTopicClient().inNamespace(testStorage.getNamespaceName()).withName(sourceMirroredTopicName).get();
    assertThat(mirroredTopic.getSpec().getPartitions(), is(3));
    assertThat(mirroredTopic.getMetadata().getLabels().get(Labels.STRIMZI_CLUSTER_LABEL), is(kafkaClusterTargetName));
}
Also used : KafkaClientsBuilder(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder) KafkaListenerAuthenticationTls(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationTls) KafkaClients(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) KafkaTopic(io.strimzi.api.kafka.model.KafkaTopic) TestStorage(io.strimzi.systemtest.storage.TestStorage) Matchers.containsString(org.hamcrest.Matchers.containsString) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest) Tag(org.junit.jupiter.api.Tag)

Example 45 with KafkaTopic

use of io.strimzi.api.kafka.model.KafkaTopic in project strimzi-kafka-operator by strimzi.

the class MirrorMaker2IsolatedST method testStrimziIdentityReplicationPolicy.

/*
     * This test is using the Strimzi Identity Replication policy. This is needed for backwards compatibility for users
     * who might still have it configured.
     *
     * This ST should be deleted once we drop the Strimzi policy completely.
     */
@ParallelNamespaceTest
void testStrimziIdentityReplicationPolicy(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext, clusterOperator.getDeploymentNamespace());
    String kafkaClusterSourceName = testStorage.getClusterName() + "-source";
    String kafkaClusterTargetName = testStorage.getClusterName() + "-target";
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterSourceName, 1, 1).build(), KafkaTemplates.kafkaEphemeral(kafkaClusterTargetName, 1, 1).build());
    // Create topic
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, testStorage.getTopicName(), 3).build());
    resourceManager.createResource(extensionContext, KafkaMirrorMaker2Templates.kafkaMirrorMaker2(testStorage.getClusterName(), kafkaClusterTargetName, kafkaClusterSourceName, 1, false).editSpec().editMirror(0).editSourceConnector().addToConfig("replication.policy.class", "io.strimzi.kafka.connect.mirror.IdentityReplicationPolicy").addToConfig("refresh.topics.interval.seconds", "1").endSourceConnector().endMirror().endSpec().build());
    LOGGER.info("Sending and receiving messages via {}", kafkaClusterSourceName);
    KafkaClients clients = new KafkaClientsBuilder().withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withBootstrapAddress(KafkaResources.plainBootstrapAddress(kafkaClusterSourceName)).withNamespaceName(testStorage.getNamespaceName()).withTopicName(testStorage.getTopicName()).withMessageCount(MESSAGE_COUNT).build();
    resourceManager.createResource(extensionContext, clients.producerStrimzi(), clients.consumerStrimzi());
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Changing to {} and will try to receive messages", kafkaClusterTargetName);
    clients = new KafkaClientsBuilder(clients).withBootstrapAddress(KafkaResources.plainBootstrapAddress(kafkaClusterTargetName)).build();
    resourceManager.createResource(extensionContext, clients.consumerStrimzi());
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    if (!Environment.isKRaftModeEnabled()) {
        LOGGER.info("Checking if the mirrored topic name is same as the original one");
        List<String> kafkaTopics = KafkaCmdClient.listTopicsUsingPodCli(testStorage.getNamespaceName(), kafkaClusterTargetName, 0);
        assertNotNull(kafkaTopics.stream().filter(kafkaTopic -> kafkaTopic.equals(testStorage.getTopicName())).findAny());
        List<String> kafkaTopicSpec = KafkaCmdClient.describeTopicUsingPodCli(testStorage.getNamespaceName(), kafkaClusterTargetName, 0, testStorage.getTopicName());
        assertThat(kafkaTopicSpec.stream().filter(token -> token.startsWith("Topic:")).findFirst().orElse(null), equalTo("Topic:" + testStorage.getTopicName()));
        assertThat(kafkaTopicSpec.stream().filter(token -> token.startsWith("PartitionCount:")).findFirst().orElse(null), equalTo("PartitionCount:3"));
    }
}
Also used : KafkaClientsBuilder(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder) LabelSelector(io.fabric8.kubernetes.api.model.LabelSelector) KafkaResource(io.strimzi.systemtest.resources.crd.KafkaResource) KubeClusterResource.cmdKubeClient(io.strimzi.test.k8s.KubeClusterResource.cmdKubeClient) KafkaTopicUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaTopicUtils) KafkaListenerAuthenticationTls(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationTls) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) KafkaResources(io.strimzi.api.kafka.model.KafkaResources) Ca(io.strimzi.operator.cluster.model.Ca) BeforeAll(org.junit.jupiter.api.BeforeAll) Map(java.util.Map) Tag(org.junit.jupiter.api.Tag) StUtils(io.strimzi.systemtest.utils.StUtils) BeforeAllOnce(io.strimzi.systemtest.BeforeAllOnce) RollingUpdateUtils(io.strimzi.systemtest.utils.RollingUpdateUtils) ACCEPTANCE(io.strimzi.systemtest.Constants.ACCEPTANCE) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) IsolatedSuite(io.strimzi.systemtest.annotations.IsolatedSuite) KafkaClients(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients) KafkaMirrorMaker2Templates(io.strimzi.systemtest.templates.crd.KafkaMirrorMaker2Templates) INTERNAL_CLIENTS_USED(io.strimzi.systemtest.Constants.INTERNAL_CLIENTS_USED) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) PasswordSecretSource(io.strimzi.api.kafka.model.PasswordSecretSource) ClientUtils(io.strimzi.systemtest.utils.ClientUtils) KafkaMirrorMaker2Resources(io.strimzi.api.kafka.model.KafkaMirrorMaker2Resources) KafkaMirrorMaker2(io.strimzi.api.kafka.model.KafkaMirrorMaker2) KafkaMirrorMaker2Status(io.strimzi.api.kafka.model.status.KafkaMirrorMaker2Status) List(java.util.List) Labels(io.strimzi.operator.common.model.Labels) Logger(org.apache.logging.log4j.Logger) KafkaTopicTemplates(io.strimzi.systemtest.templates.crd.KafkaTopicTemplates) KafkaMirrorMaker2Utils(io.strimzi.systemtest.utils.kafkaUtils.KafkaMirrorMaker2Utils) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Secret(io.fabric8.kubernetes.api.model.Secret) Matchers.is(org.hamcrest.Matchers.is) KafkaTopicResource(io.strimzi.systemtest.resources.crd.KafkaTopicResource) SCALABILITY(io.strimzi.systemtest.Constants.SCALABILITY) Matchers.containsString(org.hamcrest.Matchers.containsString) Assertions.assertDoesNotThrow(org.junit.jupiter.api.Assertions.assertDoesNotThrow) AbstractST(io.strimzi.systemtest.AbstractST) Environment(io.strimzi.systemtest.Environment) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) CoreMatchers.equalTo(org.hamcrest.CoreMatchers.equalTo) CoreMatchers.not(org.hamcrest.CoreMatchers.not) HashMap(java.util.HashMap) ExtensionContext(org.junit.jupiter.api.extension.ExtensionContext) TestStorage(io.strimzi.systemtest.storage.TestStorage) SecretUtils(io.strimzi.systemtest.utils.kubeUtils.objects.SecretUtils) PodUtils(io.strimzi.systemtest.utils.kubeUtils.objects.PodUtils) JsonPathMatchers.hasJsonPath(org.valid4j.matchers.jsonpath.JsonPathMatchers.hasJsonPath) DeploymentStrategy(io.strimzi.api.kafka.model.template.DeploymentStrategy) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) TestUtils(io.strimzi.test.TestUtils) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) KafkaListenerAuthenticationScramSha512(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationScramSha512) MIRROR_MAKER2(io.strimzi.systemtest.Constants.MIRROR_MAKER2) KafkaTemplates(io.strimzi.systemtest.templates.crd.KafkaTemplates) Ready(io.strimzi.systemtest.enums.CustomResourceStatus.Ready) KafkaUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaUtils) Constants(io.strimzi.systemtest.Constants) KafkaClientsBuilder(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest) SetupClusterOperator(io.strimzi.systemtest.resources.operator.SetupClusterOperator) KafkaCmdClient(io.strimzi.systemtest.cli.KafkaCmdClient) KubeClusterResource.kubeClient(io.strimzi.test.k8s.KubeClusterResource.kubeClient) KafkaTopic(io.strimzi.api.kafka.model.KafkaTopic) DeploymentUtils(io.strimzi.systemtest.utils.kubeUtils.controllers.DeploymentUtils) Matchers.hasItem(org.hamcrest.Matchers.hasItem) KafkaListenerType(io.strimzi.api.kafka.model.listener.arraylistener.KafkaListenerType) KafkaUserTemplates(io.strimzi.systemtest.templates.crd.KafkaUserTemplates) CONNECT_COMPONENTS(io.strimzi.systemtest.Constants.CONNECT_COMPONENTS) KafkaMirrorMaker2Resource(io.strimzi.systemtest.resources.crd.KafkaMirrorMaker2Resource) SecretBuilder(io.fabric8.kubernetes.api.model.SecretBuilder) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) REGRESSION(io.strimzi.systemtest.Constants.REGRESSION) KafkaClients(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients) TestStorage(io.strimzi.systemtest.storage.TestStorage) Matchers.containsString(org.hamcrest.Matchers.containsString) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)

Aggregations

KafkaTopic (io.strimzi.api.kafka.model.KafkaTopic)187 Test (org.junit.jupiter.api.Test)92 KafkaTopicBuilder (io.strimzi.api.kafka.model.KafkaTopicBuilder)80 Checkpoint (io.vertx.junit5.Checkpoint)46 ObjectMetaBuilder (io.fabric8.kubernetes.api.model.ObjectMetaBuilder)38 HashMap (java.util.HashMap)32 ObjectMeta (io.fabric8.kubernetes.api.model.ObjectMeta)30 CountDownLatch (java.util.concurrent.CountDownLatch)28 NewTopic (org.apache.kafka.clients.admin.NewTopic)28 List (java.util.List)26 Map (java.util.Map)26 MeterRegistry (io.micrometer.core.instrument.MeterRegistry)22 KafkaTopicStatus (io.strimzi.api.kafka.model.status.KafkaTopicStatus)22 AsyncResult (io.vertx.core.AsyncResult)22 MaxAttemptsExceededException (io.strimzi.operator.common.MaxAttemptsExceededException)20 Vertx (io.vertx.core.Vertx)20 Matchers.containsString (org.hamcrest.Matchers.containsString)20 Watcher (io.fabric8.kubernetes.client.Watcher)18 KafkaClients (io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients)18 KafkaClientsBuilder (io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder)18