Search in sources :

Example 26 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi by strimzi.

the class MirrorMaker2IsolatedST method testKMM2RollAfterSecretsCertsUpdateTLS.

@ParallelNamespaceTest
@KRaftNotSupported("UserOperator is not supported by KRaft mode and is used in this test case")
@SuppressWarnings({ "checkstyle:MethodLength" })
void testKMM2RollAfterSecretsCertsUpdateTLS(ExtensionContext extensionContext) {
    TestStorage testStorage = new TestStorage(extensionContext);
    String kafkaClusterSourceName = testStorage.getClusterName() + "-source";
    String kafkaClusterTargetName = testStorage.getClusterName() + "-target";
    String sourceMirroredTopicName = kafkaClusterSourceName + "." + testStorage.getTopicName();
    String kafkaUserSourceName = testStorage.getUserName() + "-source";
    String kafkaUserTargetName = testStorage.getUserName() + "-target";
    // Deploy source kafka with tls listener and mutual tls auth
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(kafkaClusterSourceName, 1).editSpec().editKafka().addToConfig("min.insync.replicas", 1).withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationTls()).build()).endKafka().endSpec().build());
    // Deploy target kafka with tls listener and mutual tls auth
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(kafkaClusterTargetName, 1).editSpec().editKafka().addToConfig("min.insync.replicas", 1).withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationTls()).build()).endKafka().endSpec().build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, testStorage.getTopicName(), 3).build(), KafkaUserTemplates.tlsUser(kafkaClusterSourceName, kafkaUserSourceName).build(), KafkaUserTemplates.tlsUser(kafkaClusterTargetName, kafkaUserTargetName).build());
    // Initialize CertSecretSource with certificate and secret names for source
    CertSecretSource certSecretSource = new CertSecretSource();
    certSecretSource.setCertificate("ca.crt");
    certSecretSource.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterSourceName));
    // Initialize CertSecretSource with certificate and secret names for target
    CertSecretSource certSecretTarget = new CertSecretSource();
    certSecretTarget.setCertificate("ca.crt");
    certSecretTarget.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterTargetName));
    resourceManager.createResource(extensionContext, KafkaMirrorMaker2Templates.kafkaMirrorMaker2(testStorage.getClusterName(), kafkaClusterTargetName, kafkaClusterSourceName, 1, true).editSpec().editMatchingCluster(spec -> spec.getAlias().equals(kafkaClusterSourceName)).withNewKafkaClientAuthenticationTls().withNewCertificateAndKey().withSecretName(kafkaUserSourceName).withCertificate("user.crt").withKey("user.key").endCertificateAndKey().endKafkaClientAuthenticationTls().withNewTls().withTrustedCertificates(certSecretSource).endTls().endCluster().editMatchingCluster(spec -> spec.getAlias().equals(kafkaClusterTargetName)).withNewKafkaClientAuthenticationTls().withNewCertificateAndKey().withSecretName(kafkaUserTargetName).withCertificate("user.crt").withKey("user.key").endCertificateAndKey().endKafkaClientAuthenticationTls().withNewTls().withTrustedCertificates(certSecretTarget).endTls().endCluster().editFirstMirror().editSourceConnector().addToConfig("refresh.topics.interval.seconds", 1).endSourceConnector().endMirror().endSpec().build());
    String mm2DeploymentName = KafkaMirrorMaker2Resources.deploymentName(testStorage.getClusterName());
    Map<String, String> mmSnapshot = DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), mm2DeploymentName);
    KafkaClients clients = new KafkaClientsBuilder().withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterSourceName)).withUserName(kafkaUserSourceName).withNamespaceName(testStorage.getNamespaceName()).withTopicName(testStorage.getTopicName()).withMessageCount(MESSAGE_COUNT).build();
    resourceManager.createResource(extensionContext, clients.producerTlsStrimzi(kafkaClusterSourceName), clients.consumerTlsStrimzi(kafkaClusterSourceName));
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    clients = new KafkaClientsBuilder(clients).withTopicName(sourceMirroredTopicName).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterTargetName)).withUserName(kafkaUserTargetName).build();
    LOGGER.info("Consumer in target cluster and topic should receive {} messages", MESSAGE_COUNT);
    resourceManager.createResource(extensionContext, clients.consumerTlsStrimzi(kafkaClusterTargetName));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Messages successfully mirrored");
    LabelSelector zkSourceSelector = KafkaResource.getLabelSelector(kafkaClusterSourceName, KafkaResources.zookeeperStatefulSetName(kafkaClusterSourceName));
    LabelSelector kafkaSourceSelector = KafkaResource.getLabelSelector(kafkaClusterSourceName, KafkaResources.kafkaStatefulSetName(kafkaClusterSourceName));
    LabelSelector zkTargetSelector = KafkaResource.getLabelSelector(kafkaClusterTargetName, KafkaResources.zookeeperStatefulSetName(kafkaClusterTargetName));
    LabelSelector kafkaTargetSelector = KafkaResource.getLabelSelector(kafkaClusterTargetName, KafkaResources.kafkaStatefulSetName(kafkaClusterTargetName));
    Map<String, String> kafkaSourcePods = PodUtils.podSnapshot(testStorage.getNamespaceName(), kafkaSourceSelector);
    Map<String, String> zkSourcePods = PodUtils.podSnapshot(testStorage.getNamespaceName(), zkSourceSelector);
    Map<String, String> eoSourcePods = DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(kafkaClusterSourceName));
    Map<String, String> kafkaTargetPods = PodUtils.podSnapshot(testStorage.getNamespaceName(), kafkaTargetSelector);
    Map<String, String> zkTargetPods = PodUtils.podSnapshot(testStorage.getNamespaceName(), zkTargetSelector);
    Map<String, String> eoTargetPods = DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(kafkaClusterTargetName));
    LOGGER.info("Renew Clients CA secret for Source cluster via annotation");
    String sourceClientsCaSecretName = KafkaResources.clientsCaCertificateSecretName(kafkaClusterSourceName);
    SecretUtils.annotateSecret(testStorage.getNamespaceName(), sourceClientsCaSecretName, Ca.ANNO_STRIMZI_IO_FORCE_RENEW, "true");
    kafkaSourcePods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), kafkaSourceSelector, 1, kafkaSourcePods);
    mmSnapshot = DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), mm2DeploymentName, 1, mmSnapshot);
    LOGGER.info("Renew Clients CA secret for Target cluster via annotation");
    String targetClientsCaSecretName = KafkaResources.clientsCaCertificateSecretName(kafkaClusterTargetName);
    SecretUtils.annotateSecret(testStorage.getNamespaceName(), targetClientsCaSecretName, Ca.ANNO_STRIMZI_IO_FORCE_RENEW, "true");
    kafkaTargetPods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), kafkaTargetSelector, 1, kafkaTargetPods);
    mmSnapshot = DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), mm2DeploymentName, 1, mmSnapshot);
    LOGGER.info("Send and receive messages after clients certs were removed");
    clients = new KafkaClientsBuilder(clients).withTopicName(testStorage.getTopicName()).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterSourceName)).withUserName(kafkaUserSourceName).build();
    resourceManager.createResource(extensionContext, clients.producerTlsStrimzi(kafkaClusterSourceName));
    ClientUtils.waitForClientSuccess(testStorage.getProducerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Consumer in target cluster and topic should receive {} messages", MESSAGE_COUNT);
    clients = new KafkaClientsBuilder(clients).withTopicName(sourceMirroredTopicName).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterTargetName)).withUserName(kafkaUserTargetName).build();
    resourceManager.createResource(extensionContext, clients.consumerTlsStrimzi(kafkaClusterTargetName));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Messages successfully mirrored");
    LOGGER.info("Renew Cluster CA secret for Source clusters via annotation");
    String sourceClusterCaSecretName = KafkaResources.clusterCaCertificateSecretName(kafkaClusterSourceName);
    SecretUtils.annotateSecret(testStorage.getNamespaceName(), sourceClusterCaSecretName, Ca.ANNO_STRIMZI_IO_FORCE_RENEW, "true");
    RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), zkSourceSelector, 1, zkSourcePods);
    RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), kafkaSourceSelector, 1, kafkaSourcePods);
    DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(kafkaClusterSourceName), 1, eoSourcePods);
    mmSnapshot = DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), mm2DeploymentName, 1, mmSnapshot);
    LOGGER.info("Renew Cluster CA secret for Target clusters via annotation");
    String targetClusterCaSecretName = KafkaResources.clusterCaCertificateSecretName(kafkaClusterTargetName);
    SecretUtils.annotateSecret(testStorage.getNamespaceName(), targetClusterCaSecretName, Ca.ANNO_STRIMZI_IO_FORCE_RENEW, "true");
    RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), zkTargetSelector, 1, zkTargetPods);
    RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), kafkaTargetSelector, 1, kafkaTargetPods);
    DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(kafkaClusterTargetName), 1, eoTargetPods);
    DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), mm2DeploymentName, 1, mmSnapshot);
    LOGGER.info("Send and receive messages after clients certs were removed");
    clients = new KafkaClientsBuilder(clients).withTopicName(testStorage.getTopicName()).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterSourceName)).withUserName(kafkaUserSourceName).build();
    resourceManager.createResource(extensionContext, clients.producerTlsStrimzi(kafkaClusterSourceName));
    ClientUtils.waitForClientSuccess(testStorage.getProducerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Consumer in target cluster and topic should receive {} messages", MESSAGE_COUNT);
    clients = new KafkaClientsBuilder(clients).withTopicName(sourceMirroredTopicName).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterTargetName)).withUserName(kafkaUserTargetName).build();
    resourceManager.createResource(extensionContext, clients.consumerTlsStrimzi(kafkaClusterTargetName));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Messages successfully mirrored");
}
Also used : LabelSelector(io.fabric8.kubernetes.api.model.LabelSelector) KafkaResource(io.strimzi.systemtest.resources.crd.KafkaResource) KubeClusterResource.cmdKubeClient(io.strimzi.test.k8s.KubeClusterResource.cmdKubeClient) KafkaTopicUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaTopicUtils) KafkaListenerAuthenticationTls(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationTls) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) KafkaResources(io.strimzi.api.kafka.model.KafkaResources) Ca(io.strimzi.operator.cluster.model.Ca) BeforeAll(org.junit.jupiter.api.BeforeAll) Map(java.util.Map) Tag(org.junit.jupiter.api.Tag) StUtils(io.strimzi.systemtest.utils.StUtils) BeforeAllOnce(io.strimzi.systemtest.BeforeAllOnce) RollingUpdateUtils(io.strimzi.systemtest.utils.RollingUpdateUtils) ACCEPTANCE(io.strimzi.systemtest.Constants.ACCEPTANCE) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) IsolatedSuite(io.strimzi.systemtest.annotations.IsolatedSuite) KafkaClients(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients) KafkaMirrorMaker2Templates(io.strimzi.systemtest.templates.crd.KafkaMirrorMaker2Templates) INTERNAL_CLIENTS_USED(io.strimzi.systemtest.Constants.INTERNAL_CLIENTS_USED) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) PasswordSecretSource(io.strimzi.api.kafka.model.PasswordSecretSource) ClientUtils(io.strimzi.systemtest.utils.ClientUtils) KafkaMirrorMaker2Resources(io.strimzi.api.kafka.model.KafkaMirrorMaker2Resources) KafkaMirrorMaker2(io.strimzi.api.kafka.model.KafkaMirrorMaker2) KafkaMirrorMaker2Status(io.strimzi.api.kafka.model.status.KafkaMirrorMaker2Status) List(java.util.List) Labels(io.strimzi.operator.common.model.Labels) Logger(org.apache.logging.log4j.Logger) KafkaTopicTemplates(io.strimzi.systemtest.templates.crd.KafkaTopicTemplates) KafkaMirrorMaker2Utils(io.strimzi.systemtest.utils.kafkaUtils.KafkaMirrorMaker2Utils) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Secret(io.fabric8.kubernetes.api.model.Secret) Matchers.is(org.hamcrest.Matchers.is) KafkaTopicResource(io.strimzi.systemtest.resources.crd.KafkaTopicResource) SCALABILITY(io.strimzi.systemtest.Constants.SCALABILITY) Matchers.containsString(org.hamcrest.Matchers.containsString) Assertions.assertDoesNotThrow(org.junit.jupiter.api.Assertions.assertDoesNotThrow) AbstractST(io.strimzi.systemtest.AbstractST) Environment(io.strimzi.systemtest.Environment) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) CoreMatchers.equalTo(org.hamcrest.CoreMatchers.equalTo) CoreMatchers.not(org.hamcrest.CoreMatchers.not) HashMap(java.util.HashMap) ExtensionContext(org.junit.jupiter.api.extension.ExtensionContext) TestStorage(io.strimzi.systemtest.storage.TestStorage) SecretUtils(io.strimzi.systemtest.utils.kubeUtils.objects.SecretUtils) PodUtils(io.strimzi.systemtest.utils.kubeUtils.objects.PodUtils) JsonPathMatchers.hasJsonPath(org.valid4j.matchers.jsonpath.JsonPathMatchers.hasJsonPath) DeploymentStrategy(io.strimzi.api.kafka.model.template.DeploymentStrategy) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) TestUtils(io.strimzi.test.TestUtils) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) KafkaListenerAuthenticationScramSha512(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationScramSha512) MIRROR_MAKER2(io.strimzi.systemtest.Constants.MIRROR_MAKER2) KafkaTemplates(io.strimzi.systemtest.templates.crd.KafkaTemplates) Ready(io.strimzi.systemtest.enums.CustomResourceStatus.Ready) KafkaUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaUtils) Constants(io.strimzi.systemtest.Constants) KafkaClientsBuilder(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest) SetupClusterOperator(io.strimzi.systemtest.resources.operator.SetupClusterOperator) KafkaCmdClient(io.strimzi.systemtest.cli.KafkaCmdClient) KubeClusterResource.kubeClient(io.strimzi.test.k8s.KubeClusterResource.kubeClient) KafkaTopic(io.strimzi.api.kafka.model.KafkaTopic) DeploymentUtils(io.strimzi.systemtest.utils.kubeUtils.controllers.DeploymentUtils) Matchers.hasItem(org.hamcrest.Matchers.hasItem) KafkaListenerType(io.strimzi.api.kafka.model.listener.arraylistener.KafkaListenerType) KafkaUserTemplates(io.strimzi.systemtest.templates.crd.KafkaUserTemplates) CONNECT_COMPONENTS(io.strimzi.systemtest.Constants.CONNECT_COMPONENTS) KafkaMirrorMaker2Resource(io.strimzi.systemtest.resources.crd.KafkaMirrorMaker2Resource) SecretBuilder(io.fabric8.kubernetes.api.model.SecretBuilder) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) REGRESSION(io.strimzi.systemtest.Constants.REGRESSION) KafkaClientsBuilder(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder) KafkaListenerAuthenticationTls(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationTls) KafkaClients(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) LabelSelector(io.fabric8.kubernetes.api.model.LabelSelector) TestStorage(io.strimzi.systemtest.storage.TestStorage) Matchers.containsString(org.hamcrest.Matchers.containsString) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)

Example 27 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi by strimzi.

the class MirrorMaker2IsolatedST method testConfigureDeploymentStrategy.

@ParallelNamespaceTest
void testConfigureDeploymentStrategy(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext, clusterOperator.getDeploymentNamespace());
    String kafkaClusterSourceName = testStorage.getClusterName() + "-source";
    String kafkaClusterTargetName = testStorage.getClusterName() + "-target";
    // Deploy source kafka
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterSourceName, 1, 1).build());
    // Deploy target kafka
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterTargetName, 1, 1).build());
    resourceManager.createResource(extensionContext, KafkaMirrorMaker2Templates.kafkaMirrorMaker2(testStorage.getClusterName(), kafkaClusterTargetName, kafkaClusterSourceName, 1, false).editSpec().editOrNewTemplate().editOrNewDeployment().withDeploymentStrategy(DeploymentStrategy.RECREATE).endDeployment().endTemplate().endSpec().build());
    String mm2DepName = KafkaMirrorMaker2Resources.deploymentName(testStorage.getClusterName());
    LOGGER.info("Adding label to MirrorMaker2 resource, the CR should be recreated");
    KafkaMirrorMaker2Resource.replaceKafkaMirrorMaker2ResourceInSpecificNamespace(testStorage.getClusterName(), mm2 -> mm2.getMetadata().setLabels(Collections.singletonMap("some", "label")), testStorage.getNamespaceName());
    DeploymentUtils.waitForDeploymentAndPodsReady(testStorage.getNamespaceName(), mm2DepName, 1);
    KafkaMirrorMaker2 kmm2 = KafkaMirrorMaker2Resource.kafkaMirrorMaker2Client().inNamespace(testStorage.getNamespaceName()).withName(testStorage.getClusterName()).get();
    LOGGER.info("Checking that observed gen. is still on 1 (recreation) and new label is present");
    assertThat(kmm2.getStatus().getObservedGeneration(), is(1L));
    assertThat(kmm2.getMetadata().getLabels().toString(), containsString("some=label"));
    assertThat(kmm2.getSpec().getTemplate().getDeployment().getDeploymentStrategy(), is(DeploymentStrategy.RECREATE));
    LOGGER.info("Changing deployment strategy to {}", DeploymentStrategy.ROLLING_UPDATE);
    KafkaMirrorMaker2Resource.replaceKafkaMirrorMaker2ResourceInSpecificNamespace(testStorage.getClusterName(), mm2 -> mm2.getSpec().getTemplate().getDeployment().setDeploymentStrategy(DeploymentStrategy.ROLLING_UPDATE), testStorage.getNamespaceName());
    KafkaMirrorMaker2Utils.waitForKafkaMirrorMaker2Ready(testStorage.getNamespaceName(), testStorage.getClusterName());
    LOGGER.info("Adding another label to MirrorMaker2 resource, pods should be rolled");
    KafkaMirrorMaker2Resource.replaceKafkaMirrorMaker2ResourceInSpecificNamespace(testStorage.getClusterName(), mm2 -> mm2.getMetadata().getLabels().put("another", "label"), testStorage.getNamespaceName());
    DeploymentUtils.waitForDeploymentAndPodsReady(testStorage.getNamespaceName(), mm2DepName, 1);
    LOGGER.info("Checking that observed gen. higher (rolling update) and label is changed");
    kmm2 = KafkaMirrorMaker2Resource.kafkaMirrorMaker2Client().inNamespace(testStorage.getNamespaceName()).withName(testStorage.getClusterName()).get();
    assertThat(kmm2.getStatus().getObservedGeneration(), is(2L));
    assertThat(kmm2.getMetadata().getLabels().toString(), containsString("another=label"));
    assertThat(kmm2.getSpec().getTemplate().getDeployment().getDeploymentStrategy(), is(DeploymentStrategy.ROLLING_UPDATE));
}
Also used : TestStorage(io.strimzi.systemtest.storage.TestStorage) Matchers.containsString(org.hamcrest.Matchers.containsString) KafkaMirrorMaker2(io.strimzi.api.kafka.model.KafkaMirrorMaker2) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)

Example 28 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi by strimzi.

the class MirrorMaker2IsolatedST method testMirrorMaker2TlsAndTlsClientAuth.

/**
 * Test mirroring messages by MirrorMaker 2.0 over tls transport using mutual tls auth
 */
@SuppressWarnings({ "checkstyle:MethodLength" })
@ParallelNamespaceTest
@Tag(ACCEPTANCE)
@KRaftNotSupported("UserOperator is not supported by KRaft mode and is used in this test case")
void testMirrorMaker2TlsAndTlsClientAuth(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext, clusterOperator.getDeploymentNamespace());
    String kafkaClusterSourceName = testStorage.getClusterName() + "-source";
    String kafkaClusterTargetName = testStorage.getClusterName() + "-target";
    String sourceMirroredTopicName = kafkaClusterSourceName + "." + testStorage.getTopicName();
    String kafkaUserSourceName = testStorage.getUserName() + "-source";
    String kafkaUserTargetName = testStorage.getUserName() + "-target";
    // Deploy source kafka with tls listener and mutual tls auth
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterSourceName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationTls()).build()).endKafka().endSpec().build());
    // Deploy target kafka with tls listener and mutual tls auth
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterTargetName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationTls()).build()).endKafka().endSpec().build());
    // Deploy topic
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, testStorage.getTopicName(), 3).build(), KafkaUserTemplates.tlsUser(kafkaClusterSourceName, kafkaUserSourceName).build(), KafkaUserTemplates.tlsUser(kafkaClusterTargetName, kafkaUserTargetName).build());
    // Initialize CertSecretSource with certificate and secret names for source
    CertSecretSource certSecretSource = new CertSecretSource();
    certSecretSource.setCertificate("ca.crt");
    certSecretSource.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterSourceName));
    // Initialize CertSecretSource with certificate and secret names for target
    CertSecretSource certSecretTarget = new CertSecretSource();
    certSecretTarget.setCertificate("ca.crt");
    certSecretTarget.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterTargetName));
    KafkaClients clients = new KafkaClientsBuilder().withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterSourceName)).withUserName(kafkaUserSourceName).withNamespaceName(testStorage.getNamespaceName()).withTopicName(testStorage.getTopicName()).withMessageCount(MESSAGE_COUNT).build();
    // Check brokers availability
    LOGGER.info("Messages exchange - topic {}, cluster {} and message count of {}", testStorage.getTopicName(), kafkaClusterSourceName, MESSAGE_COUNT);
    resourceManager.createResource(extensionContext, clients.producerTlsStrimzi(kafkaClusterSourceName), clients.consumerTlsStrimzi(kafkaClusterSourceName));
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    resourceManager.createResource(extensionContext, KafkaMirrorMaker2Templates.kafkaMirrorMaker2(testStorage.getClusterName(), kafkaClusterTargetName, kafkaClusterSourceName, 1, true).editSpec().editMatchingCluster(spec -> spec.getAlias().equals(kafkaClusterSourceName)).withNewKafkaClientAuthenticationTls().withNewCertificateAndKey().withSecretName(kafkaUserSourceName).withCertificate("user.crt").withKey("user.key").endCertificateAndKey().endKafkaClientAuthenticationTls().withNewTls().withTrustedCertificates(certSecretSource).endTls().endCluster().editMatchingCluster(spec -> spec.getAlias().equals(kafkaClusterTargetName)).withNewKafkaClientAuthenticationTls().withNewCertificateAndKey().withSecretName(kafkaUserTargetName).withCertificate("user.crt").withKey("user.key").endCertificateAndKey().endKafkaClientAuthenticationTls().withNewTls().withTrustedCertificates(certSecretTarget).endTls().endCluster().endSpec().build());
    LOGGER.info("Changing to mirrored topic - topic {}, cluster {}, user {}", sourceMirroredTopicName, kafkaClusterTargetName, kafkaClusterTargetName);
    clients = new KafkaClientsBuilder(clients).withTopicName(sourceMirroredTopicName).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterTargetName)).withUserName(kafkaUserTargetName).build();
    LOGGER.info("Now messages should be mirrored to target topic and cluster");
    resourceManager.createResource(extensionContext, clients.consumerTlsStrimzi(kafkaClusterTargetName));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Messages successfully mirrored");
    KafkaTopicUtils.waitForKafkaTopicCreation(testStorage.getNamespaceName(), sourceMirroredTopicName);
    KafkaTopic mirroredTopic = KafkaTopicResource.kafkaTopicClient().inNamespace(testStorage.getNamespaceName()).withName(sourceMirroredTopicName).get();
    assertThat(mirroredTopic.getSpec().getPartitions(), is(3));
    assertThat(mirroredTopic.getMetadata().getLabels().get(Labels.STRIMZI_CLUSTER_LABEL), is(kafkaClusterTargetName));
}
Also used : KafkaClientsBuilder(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder) KafkaListenerAuthenticationTls(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationTls) KafkaClients(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) KafkaTopic(io.strimzi.api.kafka.model.KafkaTopic) TestStorage(io.strimzi.systemtest.storage.TestStorage) Matchers.containsString(org.hamcrest.Matchers.containsString) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest) Tag(org.junit.jupiter.api.Tag)

Example 29 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi by strimzi.

the class MirrorMaker2IsolatedST method testMirrorMaker2TlsAndScramSha512Auth.

/**
 * Test mirroring messages by MirrorMaker 2.0 over tls transport using scram-sha-512 auth
 */
@ParallelNamespaceTest
@KRaftNotSupported("UserOperator is not supported by KRaft mode and is used in this test case")
void testMirrorMaker2TlsAndScramSha512Auth(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext, clusterOperator.getDeploymentNamespace());
    String kafkaClusterSourceName = testStorage.getClusterName() + "-source";
    String kafkaClusterTargetName = testStorage.getClusterName() + "-target";
    String sourceMirroredTopicName = kafkaClusterSourceName + "." + testStorage.getTopicName();
    String kafkaUserSourceName = testStorage.getUserName() + "-source";
    String kafkaUserTargetName = testStorage.getUserName() + "-target";
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterSourceName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationScramSha512()).build()).endKafka().endSpec().build(), KafkaTemplates.kafkaEphemeral(kafkaClusterTargetName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationScramSha512()).build()).endKafka().endSpec().build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, testStorage.getTopicName(), 3).build(), KafkaUserTemplates.scramShaUser(kafkaClusterSourceName, kafkaUserSourceName).build(), KafkaUserTemplates.scramShaUser(kafkaClusterTargetName, kafkaUserTargetName).build());
    // Initialize PasswordSecretSource to set this as PasswordSecret in MirrorMaker2 spec
    PasswordSecretSource passwordSecretSource = new PasswordSecretSource();
    passwordSecretSource.setSecretName(kafkaUserSourceName);
    passwordSecretSource.setPassword("password");
    // Initialize PasswordSecretSource to set this as PasswordSecret in MirrorMaker2 spec
    PasswordSecretSource passwordSecretTarget = new PasswordSecretSource();
    passwordSecretTarget.setSecretName(kafkaUserTargetName);
    passwordSecretTarget.setPassword("password");
    // Initialize CertSecretSource with certificate and secret names for source
    CertSecretSource certSecretSource = new CertSecretSource();
    certSecretSource.setCertificate("ca.crt");
    certSecretSource.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterSourceName));
    // Initialize CertSecretSource with certificate and secret names for target
    CertSecretSource certSecretTarget = new CertSecretSource();
    certSecretTarget.setCertificate("ca.crt");
    certSecretTarget.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterTargetName));
    KafkaClients clients = new KafkaClientsBuilder().withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterSourceName)).withUserName(kafkaUserSourceName).withNamespaceName(testStorage.getNamespaceName()).withTopicName(testStorage.getTopicName()).withMessageCount(MESSAGE_COUNT).build();
    // Check brokers availability
    LOGGER.info("Messages exchange - topic {}, cluster {} and message count of {}", testStorage.getTopicName(), kafkaClusterSourceName, MESSAGE_COUNT);
    resourceManager.createResource(extensionContext, clients.producerScramShaTlsStrimzi(kafkaClusterSourceName), clients.consumerScramShaTlsStrimzi(kafkaClusterSourceName));
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    resourceManager.createResource(extensionContext, KafkaMirrorMaker2Templates.kafkaMirrorMaker2(testStorage.getClusterName(), kafkaClusterTargetName, kafkaClusterSourceName, 1, true).editSpec().editMatchingCluster(spec -> spec.getAlias().equals(kafkaClusterSourceName)).withNewKafkaClientAuthenticationScramSha512().withUsername(kafkaUserSourceName).withPasswordSecret(passwordSecretSource).endKafkaClientAuthenticationScramSha512().withNewTls().withTrustedCertificates(certSecretSource).endTls().endCluster().editMatchingCluster(spec -> spec.getAlias().equals(kafkaClusterTargetName)).withNewKafkaClientAuthenticationScramSha512().withUsername(kafkaUserTargetName).withPasswordSecret(passwordSecretTarget).endKafkaClientAuthenticationScramSha512().withNewTls().withTrustedCertificates(certSecretTarget).endTls().endCluster().endSpec().build());
    LOGGER.info("Changing to mirrored topic - topic {}, cluster {}, user {}", sourceMirroredTopicName, kafkaClusterTargetName, kafkaClusterTargetName);
    clients = new KafkaClientsBuilder(clients).withTopicName(sourceMirroredTopicName).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterTargetName)).withUserName(kafkaUserTargetName).build();
    LOGGER.info("Now messages should be mirrored to target topic and cluster");
    resourceManager.createResource(extensionContext, clients.consumerScramShaTlsStrimzi(kafkaClusterTargetName));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Messages successfully mirrored");
    KafkaTopicUtils.waitForKafkaTopicCreation(testStorage.getNamespaceName(), sourceMirroredTopicName);
    KafkaTopic mirroredTopic = KafkaTopicResource.kafkaTopicClient().inNamespace(testStorage.getNamespaceName()).withName(sourceMirroredTopicName).get();
    assertThat(mirroredTopic.getSpec().getPartitions(), is(3));
    assertThat(mirroredTopic.getMetadata().getLabels().get(Labels.STRIMZI_CLUSTER_LABEL), is(kafkaClusterTargetName));
}
Also used : KafkaListenerAuthenticationScramSha512(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationScramSha512) KafkaClientsBuilder(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder) LabelSelector(io.fabric8.kubernetes.api.model.LabelSelector) KafkaResource(io.strimzi.systemtest.resources.crd.KafkaResource) KubeClusterResource.cmdKubeClient(io.strimzi.test.k8s.KubeClusterResource.cmdKubeClient) KafkaTopicUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaTopicUtils) KafkaListenerAuthenticationTls(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationTls) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) KafkaResources(io.strimzi.api.kafka.model.KafkaResources) Ca(io.strimzi.operator.cluster.model.Ca) BeforeAll(org.junit.jupiter.api.BeforeAll) Map(java.util.Map) Tag(org.junit.jupiter.api.Tag) StUtils(io.strimzi.systemtest.utils.StUtils) BeforeAllOnce(io.strimzi.systemtest.BeforeAllOnce) RollingUpdateUtils(io.strimzi.systemtest.utils.RollingUpdateUtils) ACCEPTANCE(io.strimzi.systemtest.Constants.ACCEPTANCE) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) IsolatedSuite(io.strimzi.systemtest.annotations.IsolatedSuite) KafkaClients(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients) KafkaMirrorMaker2Templates(io.strimzi.systemtest.templates.crd.KafkaMirrorMaker2Templates) INTERNAL_CLIENTS_USED(io.strimzi.systemtest.Constants.INTERNAL_CLIENTS_USED) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) PasswordSecretSource(io.strimzi.api.kafka.model.PasswordSecretSource) ClientUtils(io.strimzi.systemtest.utils.ClientUtils) KafkaMirrorMaker2Resources(io.strimzi.api.kafka.model.KafkaMirrorMaker2Resources) KafkaMirrorMaker2(io.strimzi.api.kafka.model.KafkaMirrorMaker2) KafkaMirrorMaker2Status(io.strimzi.api.kafka.model.status.KafkaMirrorMaker2Status) List(java.util.List) Labels(io.strimzi.operator.common.model.Labels) Logger(org.apache.logging.log4j.Logger) KafkaTopicTemplates(io.strimzi.systemtest.templates.crd.KafkaTopicTemplates) KafkaMirrorMaker2Utils(io.strimzi.systemtest.utils.kafkaUtils.KafkaMirrorMaker2Utils) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Secret(io.fabric8.kubernetes.api.model.Secret) Matchers.is(org.hamcrest.Matchers.is) KafkaTopicResource(io.strimzi.systemtest.resources.crd.KafkaTopicResource) SCALABILITY(io.strimzi.systemtest.Constants.SCALABILITY) Matchers.containsString(org.hamcrest.Matchers.containsString) Assertions.assertDoesNotThrow(org.junit.jupiter.api.Assertions.assertDoesNotThrow) AbstractST(io.strimzi.systemtest.AbstractST) Environment(io.strimzi.systemtest.Environment) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) CoreMatchers.equalTo(org.hamcrest.CoreMatchers.equalTo) CoreMatchers.not(org.hamcrest.CoreMatchers.not) HashMap(java.util.HashMap) ExtensionContext(org.junit.jupiter.api.extension.ExtensionContext) TestStorage(io.strimzi.systemtest.storage.TestStorage) SecretUtils(io.strimzi.systemtest.utils.kubeUtils.objects.SecretUtils) PodUtils(io.strimzi.systemtest.utils.kubeUtils.objects.PodUtils) JsonPathMatchers.hasJsonPath(org.valid4j.matchers.jsonpath.JsonPathMatchers.hasJsonPath) DeploymentStrategy(io.strimzi.api.kafka.model.template.DeploymentStrategy) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) TestUtils(io.strimzi.test.TestUtils) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) KafkaListenerAuthenticationScramSha512(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationScramSha512) MIRROR_MAKER2(io.strimzi.systemtest.Constants.MIRROR_MAKER2) KafkaTemplates(io.strimzi.systemtest.templates.crd.KafkaTemplates) Ready(io.strimzi.systemtest.enums.CustomResourceStatus.Ready) KafkaUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaUtils) Constants(io.strimzi.systemtest.Constants) KafkaClientsBuilder(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest) SetupClusterOperator(io.strimzi.systemtest.resources.operator.SetupClusterOperator) KafkaCmdClient(io.strimzi.systemtest.cli.KafkaCmdClient) KubeClusterResource.kubeClient(io.strimzi.test.k8s.KubeClusterResource.kubeClient) KafkaTopic(io.strimzi.api.kafka.model.KafkaTopic) DeploymentUtils(io.strimzi.systemtest.utils.kubeUtils.controllers.DeploymentUtils) Matchers.hasItem(org.hamcrest.Matchers.hasItem) KafkaListenerType(io.strimzi.api.kafka.model.listener.arraylistener.KafkaListenerType) KafkaUserTemplates(io.strimzi.systemtest.templates.crd.KafkaUserTemplates) CONNECT_COMPONENTS(io.strimzi.systemtest.Constants.CONNECT_COMPONENTS) KafkaMirrorMaker2Resource(io.strimzi.systemtest.resources.crd.KafkaMirrorMaker2Resource) SecretBuilder(io.fabric8.kubernetes.api.model.SecretBuilder) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) REGRESSION(io.strimzi.systemtest.Constants.REGRESSION) KafkaClients(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) KafkaTopic(io.strimzi.api.kafka.model.KafkaTopic) PasswordSecretSource(io.strimzi.api.kafka.model.PasswordSecretSource) TestStorage(io.strimzi.systemtest.storage.TestStorage) Matchers.containsString(org.hamcrest.Matchers.containsString) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)

Example 30 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi by strimzi.

the class MirrorMaker2IsolatedST method testRestoreOffsetsInConsumerGroup.

@ParallelNamespaceTest
@KRaftNotSupported("This the is failing with KRaft and need more investigation")
void testRestoreOffsetsInConsumerGroup(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext, clusterOperator.getDeploymentNamespace());
    final String kafkaClusterSourceName = testStorage.getClusterName() + "-source";
    final String kafkaClusterTargetName = testStorage.getClusterName() + "-target";
    final String sourceMirroredTopicName = kafkaClusterSourceName + "." + testStorage.getTopicName();
    final String consumerGroup = ClientUtils.generateRandomConsumerGroup();
    final String sourceProducerName = testStorage.getProducerName() + "-source";
    final String sourceConsumerName = testStorage.getConsumerName() + "-source";
    final String targetProducerName = testStorage.getProducerName() + "-target";
    final String targetConsumerName = testStorage.getProducerName() + "-target";
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(kafkaClusterSourceName, 1).build(), KafkaTemplates.kafkaPersistent(kafkaClusterTargetName, 1).build());
    Map<String, Object> sourceConnectorConfig = new HashMap<>();
    sourceConnectorConfig.put("refresh.topics.interval.seconds", "1");
    sourceConnectorConfig.put("replication.factor", "1");
    sourceConnectorConfig.put("offset-syncs.topic.replication.factor", "1");
    Map<String, Object> checkpointConnectorConfig = new HashMap<>();
    checkpointConnectorConfig.put("refresh.groups.interval.seconds", "1");
    checkpointConnectorConfig.put("sync.group.offsets.enabled", "true");
    checkpointConnectorConfig.put("sync.group.offsets.interval.seconds", "1");
    checkpointConnectorConfig.put("emit.checkpoints.enabled", "true");
    checkpointConnectorConfig.put("emit.checkpoints.interval.seconds", "1");
    checkpointConnectorConfig.put("checkpoints.topic.replication.factor", "1");
    Map<String, Object> heartbeatConnectorConfig = new HashMap<>();
    heartbeatConnectorConfig.put("heartbeats.topic.replication.factor", "1");
    resourceManager.createResource(extensionContext, KafkaMirrorMaker2Templates.kafkaMirrorMaker2(kafkaClusterSourceName, kafkaClusterTargetName, kafkaClusterSourceName, 1, false).editSpec().editFirstMirror().editSourceConnector().addToConfig(sourceConnectorConfig).endSourceConnector().editCheckpointConnector().addToConfig(checkpointConnectorConfig).endCheckpointConnector().editHeartbeatConnector().addToConfig(heartbeatConnectorConfig).endHeartbeatConnector().withTopicsPattern(".*").withGroupsPattern(".*").endMirror().endSpec().build(), KafkaMirrorMaker2Templates.kafkaMirrorMaker2(kafkaClusterTargetName, kafkaClusterSourceName, kafkaClusterTargetName, 1, false).editSpec().editFirstMirror().editSourceConnector().addToConfig(sourceConnectorConfig).endSourceConnector().editCheckpointConnector().addToConfig(checkpointConnectorConfig).endCheckpointConnector().editHeartbeatConnector().addToConfig(heartbeatConnectorConfig).endHeartbeatConnector().withTopicsPattern(".*").withGroupsPattern(".*").endMirror().endSpec().build(), KafkaTopicTemplates.topic(kafkaClusterSourceName, testStorage.getTopicName(), 3).build());
    KafkaClients initialInternalClientSourceJob = new KafkaClientsBuilder().withProducerName(sourceProducerName).withConsumerName(sourceConsumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(kafkaClusterSourceName)).withTopicName(testStorage.getTopicName()).withMessageCount(MESSAGE_COUNT).withMessage("Producer A").withConsumerGroup(consumerGroup).withNamespaceName(testStorage.getNamespaceName()).build();
    KafkaClients initialInternalClientTargetJob = new KafkaClientsBuilder().withProducerName(targetProducerName).withConsumerName(targetConsumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(kafkaClusterTargetName)).withTopicName(sourceMirroredTopicName).withMessageCount(MESSAGE_COUNT).withConsumerGroup(consumerGroup).withNamespaceName(testStorage.getNamespaceName()).build();
    LOGGER.info("Send & receive {} messages to/from Source cluster.", MESSAGE_COUNT);
    resourceManager.createResource(extensionContext, initialInternalClientSourceJob.producerStrimzi(), initialInternalClientSourceJob.consumerStrimzi());
    ClientUtils.waitForClientsSuccess(sourceProducerName, sourceConsumerName, testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Send {} messages to Source cluster.", MESSAGE_COUNT);
    KafkaClients internalClientSourceJob = new KafkaClientsBuilder(initialInternalClientSourceJob).withMessage("Producer B").build();
    resourceManager.createResource(extensionContext, internalClientSourceJob.producerStrimzi());
    ClientUtils.waitForClientSuccess(sourceProducerName, testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Wait 1 second as 'sync.group.offsets.interval.seconds=1'. As this is insignificant wait, we're skipping it");
    LOGGER.info("Receive {} messages from mirrored topic on Target cluster.", MESSAGE_COUNT);
    resourceManager.createResource(extensionContext, initialInternalClientTargetJob.consumerStrimzi());
    ClientUtils.waitForClientSuccess(targetConsumerName, testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Send 50 messages to Source cluster");
    internalClientSourceJob = new KafkaClientsBuilder(internalClientSourceJob).withMessageCount(50).withMessage("Producer C").build();
    resourceManager.createResource(extensionContext, internalClientSourceJob.producerStrimzi());
    ClientUtils.waitForClientSuccess(sourceProducerName, testStorage.getNamespaceName(), 50);
    LOGGER.info("Wait 1 second as 'sync.group.offsets.interval.seconds=1'. As this is insignificant wait, we're skipping it");
    LOGGER.info("Receive 10 msgs from source cluster");
    internalClientSourceJob = new KafkaClientsBuilder(internalClientSourceJob).withMessageCount(10).withAdditionalConfig("max.poll.records=10").build();
    resourceManager.createResource(extensionContext, internalClientSourceJob.consumerStrimzi());
    ClientUtils.waitForClientSuccess(sourceConsumerName, testStorage.getNamespaceName(), 10);
    LOGGER.info("Wait 1 second as 'sync.group.offsets.interval.seconds=1'. As this is insignificant wait, we're skipping it");
    LOGGER.info("Receive 40 msgs from mirrored topic on Target cluster");
    KafkaClients internalClientTargetJob = new KafkaClientsBuilder(initialInternalClientTargetJob).withMessageCount(40).build();
    resourceManager.createResource(extensionContext, internalClientTargetJob.consumerStrimzi());
    ClientUtils.waitForClientSuccess(targetConsumerName, testStorage.getNamespaceName(), 40);
    LOGGER.info("There should be no more messages to read. Try to consume at least 1 message. " + "This client job should fail on timeout.");
    resourceManager.createResource(extensionContext, initialInternalClientTargetJob.consumerStrimzi());
    assertDoesNotThrow(() -> ClientUtils.waitForClientTimeout(targetConsumerName, testStorage.getNamespaceName(), 1));
    LOGGER.info("As it's Active-Active MM2 mode, there should be no more messages to read from Source cluster" + " topic. This client job should fail on timeout.");
    resourceManager.createResource(extensionContext, initialInternalClientSourceJob.consumerStrimzi());
    assertDoesNotThrow(() -> ClientUtils.waitForClientTimeout(sourceConsumerName, testStorage.getNamespaceName(), 1));
}
Also used : KafkaClientsBuilder(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder) KafkaClients(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients) HashMap(java.util.HashMap) TestStorage(io.strimzi.systemtest.storage.TestStorage) Matchers.containsString(org.hamcrest.Matchers.containsString) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)

Aggregations

TestStorage (io.strimzi.systemtest.storage.TestStorage)210 KafkaClientsBuilder (io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder)152 ParallelNamespaceTest (io.strimzi.systemtest.annotations.ParallelNamespaceTest)150 KafkaClients (io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients)150 Tag (org.junit.jupiter.api.Tag)128 KRaftNotSupported (io.strimzi.systemtest.annotations.KRaftNotSupported)94 GenericKafkaListenerBuilder (io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder)72 Matchers.containsString (org.hamcrest.Matchers.containsString)70 HashMap (java.util.HashMap)54 Secret (io.fabric8.kubernetes.api.model.Secret)46 CoreMatchers.containsString (org.hamcrest.CoreMatchers.containsString)46 Map (java.util.Map)34 KafkaResources (io.strimzi.api.kafka.model.KafkaResources)32 AbstractST (io.strimzi.systemtest.AbstractST)32 Constants (io.strimzi.systemtest.Constants)32 REGRESSION (io.strimzi.systemtest.Constants.REGRESSION)32 Environment (io.strimzi.systemtest.Environment)32 KafkaTemplates (io.strimzi.systemtest.templates.crd.KafkaTemplates)32 KafkaTopicTemplates (io.strimzi.systemtest.templates.crd.KafkaTopicTemplates)32 ClientUtils (io.strimzi.systemtest.utils.ClientUtils)32