Examples with TestStorage io.strimzi.systemtest.storage.TestStorage used on opensource projects

Example 66 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi-kafka-operator by strimzi.

the class SecurityST method testCertRegeneratedAfterInternalCAisDeleted.

@ParallelNamespaceTest
@Tag(INTERNAL_CLIENTS_USED)
@KRaftNotSupported("UserOperator is not supported by KRaft mode and is used in this test case")
void testCertRegeneratedAfterInternalCAisDeleted(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext, namespace);
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(testStorage.getClusterName(), 3, 1).build());
    Map<String, String> kafkaPods = PodUtils.podSnapshot(testStorage.getNamespaceName(), testStorage.getKafkaSelector());
    resourceManager.createResource(extensionContext, KafkaUserTemplates.tlsUser(testStorage.getClusterName(), testStorage.getUserName()).build(), KafkaTopicTemplates.topic(testStorage.getClusterName(), testStorage.getTopicName()).build());
    KafkaClients kafkaClients = new KafkaClientsBuilder().withTopicName(testStorage.getTopicName()).withMessageCount(MESSAGE_COUNT).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(testStorage.getClusterName())).withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withNamespaceName(testStorage.getNamespaceName()).withUserName(testStorage.getUserName()).build();
    List<Secret> secrets = kubeClient().listSecrets(testStorage.getNamespaceName()).stream().filter(secret -> secret.getMetadata().getName().startsWith(testStorage.getClusterName()) && secret.getMetadata().getName().endsWith("ca-cert")).collect(Collectors.toList());
    for (Secret s : secrets) {
        LOGGER.info("Verifying that secret {} with name {} is present", s, s.getMetadata().getName());
        assertThat(s.getData(), is(notNullValue()));
    }
    for (Secret s : secrets) {
        LOGGER.info("Deleting secret {}", s.getMetadata().getName());
        kubeClient().deleteSecret(testStorage.getNamespaceName(), s.getMetadata().getName());
    }
    PodUtils.verifyThatRunningPodsAreStable(testStorage.getNamespaceName(), KafkaResources.kafkaStatefulSetName(testStorage.getClusterName()));
    RollingUpdateUtils.waitTillComponentHasRolled(testStorage.getNamespaceName(), testStorage.getKafkaSelector(), 3, kafkaPods);
    for (Secret s : secrets) {
        SecretUtils.waitForSecretReady(testStorage.getNamespaceName(), s.getMetadata().getName(), () -> {
        });
    }
    List<Secret> regeneratedSecrets = kubeClient().listSecrets(testStorage.getNamespaceName()).stream().filter(secret -> secret.getMetadata().getName().endsWith("ca-cert")).collect(Collectors.toList());
    for (int i = 0; i < secrets.size(); i++) {
        assertThat("Certificates has different cert UIDs", !secrets.get(i).getData().get("ca.crt").equals(regeneratedSecrets.get(i).getData().get("ca.crt")));
    }
    resourceManager.createResource(extensionContext, kafkaClients.producerTlsStrimzi(testStorage.getClusterName()), kafkaClients.consumerTlsStrimzi(testStorage.getClusterName()));
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
}

Example 67 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi-kafka-operator by strimzi.

the class SecurityST method testCaRenewalBreakInMiddle.

@ParallelNamespaceTest
@Tag(INTERNAL_CLIENTS_USED)
@KRaftNotSupported("UserOperator is not supported by KRaft mode and is used in this test case")
void testCaRenewalBreakInMiddle(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext, namespace);
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(testStorage.getClusterName(), 3, 3).editSpec().withNewClusterCa().withRenewalDays(1).withValidityDays(3).endClusterCa().endSpec().build());
    resourceManager.createResource(extensionContext, KafkaUserTemplates.tlsUser(testStorage.getClusterName(), testStorage.getUserName()).build(), KafkaTopicTemplates.topic(testStorage.getClusterName(), testStorage.getTopicName()).build());
    KafkaClients kafkaClients = new KafkaClientsBuilder().withTopicName(testStorage.getTopicName()).withMessageCount(MESSAGE_COUNT).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(testStorage.getClusterName())).withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withNamespaceName(testStorage.getNamespaceName()).withUserName(testStorage.getUserName()).build();
    resourceManager.createResource(extensionContext, kafkaClients.producerTlsStrimzi(testStorage.getClusterName()), kafkaClients.consumerTlsStrimzi(testStorage.getClusterName()));
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    Map<String, String> zkPods = PodUtils.podSnapshot(testStorage.getNamespaceName(), testStorage.getZookeeperSelector());
    Map<String, String> kafkaPods = PodUtils.podSnapshot(testStorage.getNamespaceName(), testStorage.getKafkaSelector());
    Map<String, String> eoPods = DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(testStorage.getClusterName()));
    InputStream secretInputStream = getClass().getClassLoader().getResourceAsStream("security-st-certs/expired-cluster-ca.crt");
    String clusterCaCert = TestUtils.readResource(secretInputStream);
    SecretUtils.createSecret(testStorage.getNamespaceName(), clusterCaCertificateSecretName(testStorage.getClusterName()), "ca.crt", clusterCaCert);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(testStorage.getClusterName(), k -> {
        k.getSpec().getZookeeper().setResources(new ResourceRequirementsBuilder().addToRequests("cpu", new Quantity("100000m")).build());
        k.getSpec().setClusterCa(new CertificateAuthorityBuilder().withRenewalDays(4).withValidityDays(7).build());
    }, testStorage.getNamespaceName());
    TestUtils.waitFor("Waiting for some kafka pod to be in the pending phase because of selected high cpu resource", Constants.GLOBAL_POLL_INTERVAL, Constants.GLOBAL_TIMEOUT, () -> {
        List<Pod> pendingPods = kubeClient().listPodsByPrefixInName(testStorage.getNamespaceName(), KafkaResources.zookeeperStatefulSetName(testStorage.getClusterName())).stream().filter(pod -> pod.getStatus().getPhase().equals("Pending")).collect(Collectors.toList());
        if (pendingPods.isEmpty()) {
            LOGGER.info("No pods of {} are in desired state", KafkaResources.zookeeperStatefulSetName(testStorage.getClusterName()));
            return false;
        } else {
            LOGGER.info("Pod in 'Pending' state: {}", pendingPods.get(0).getMetadata().getName());
            return true;
        }
    });
    kafkaClients = new KafkaClientsBuilder(kafkaClients).withConsumerGroup(ClientUtils.generateRandomConsumerGroup()).build();
    resourceManager.createResource(extensionContext, kafkaClients.consumerTlsStrimzi(testStorage.getClusterName()));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(testStorage.getClusterName(), k -> {
        k.getSpec().getZookeeper().setResources(new ResourceRequirementsBuilder().addToRequests("cpu", new Quantity("200m")).build());
    }, testStorage.getNamespaceName());
    // Wait until the certificates have been replaced
    SecretUtils.waitForCertToChange(testStorage.getNamespaceName(), clusterCaCert, KafkaResources.clusterCaCertificateSecretName(testStorage.getClusterName()));
    RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), testStorage.getZookeeperSelector(), 3, zkPods);
    RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), testStorage.getKafkaSelector(), 3, kafkaPods);
    DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(testStorage.getClusterName()), 1, eoPods);
    kafkaClients = new KafkaClientsBuilder(kafkaClients).withConsumerGroup(ClientUtils.generateRandomConsumerGroup()).build();
    resourceManager.createResource(extensionContext, kafkaClients.consumerTlsStrimzi(testStorage.getClusterName()));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    // Try to send and receive messages with new certificates
    String topicName = KafkaTopicUtils.generateRandomNameOfTopic();
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(testStorage.getClusterName(), topicName).build());
    kafkaClients = new KafkaClientsBuilder(kafkaClients).withConsumerGroup(ClientUtils.generateRandomConsumerGroup()).withTopicName(topicName).build();
    resourceManager.createResource(extensionContext, kafkaClients.producerTlsStrimzi(testStorage.getClusterName()), kafkaClients.consumerTlsStrimzi(testStorage.getClusterName()));
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
}

Example 68 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi-kafka-operator by strimzi.

the class SecurityST method autoRenewSomeCaCertsTriggeredByAnno.

@SuppressWarnings({ "checkstyle:MethodLength", "checkstyle:NPathComplexity" })
void autoRenewSomeCaCertsTriggeredByAnno(ExtensionContext extensionContext, boolean zkShouldRoll, boolean kafkaShouldRoll, boolean eoShouldRoll, boolean keAndCCShouldRoll) {
    final TestStorage testStorage = new TestStorage(extensionContext, namespace);
    createKafkaCluster(extensionContext, testStorage.getClusterName());
    List<String> secrets;
    // to make it parallel we need decision maker...
    if (extensionContext.getTags().contains("ClusterCaCerts")) {
        secrets = Arrays.asList(clusterCaCertificateSecretName(testStorage.getClusterName()));
    } else if (extensionContext.getTags().contains("ClientsCaCerts")) {
        secrets = Arrays.asList(clientsCaCertificateSecretName(testStorage.getClusterName()));
    } else {
        // AllCaKeys
        secrets = Arrays.asList(clusterCaCertificateSecretName(testStorage.getClusterName()), clientsCaCertificateSecretName(testStorage.getClusterName()));
    }
    resourceManager.createResource(extensionContext, KafkaUserTemplates.tlsUser(testStorage.getClusterName(), testStorage.getUserName()).build(), KafkaTopicTemplates.topic(testStorage.getClusterName(), testStorage.getTopicName()).build());
    KafkaClients kafkaClients = new KafkaClientsBuilder().withTopicName(testStorage.getTopicName()).withMessageCount(MESSAGE_COUNT).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(testStorage.getClusterName())).withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withNamespaceName(testStorage.getNamespaceName()).withUserName(testStorage.getUserName()).build();
    resourceManager.createResource(extensionContext, kafkaClients.producerTlsStrimzi(testStorage.getClusterName()), kafkaClients.consumerTlsStrimzi(testStorage.getClusterName()));
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    // Get all pods, and their resource versions
    Map<String, String> zkPods = PodUtils.podSnapshot(testStorage.getNamespaceName(), testStorage.getZookeeperSelector());
    Map<String, String> kafkaPods = PodUtils.podSnapshot(testStorage.getNamespaceName(), testStorage.getKafkaSelector());
    Map<String, String> eoPod = DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(testStorage.getClusterName()));
    Map<String, String> ccPod = DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), CruiseControlResources.deploymentName(testStorage.getClusterName()));
    Map<String, String> kePod = DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), KafkaExporterResources.deploymentName(testStorage.getClusterName()));
    LOGGER.info("Triggering CA cert renewal by adding the annotation");
    Map<String, String> initialCaCerts = new HashMap<>();
    for (String secretName : secrets) {
        Secret secret = kubeClient().getSecret(testStorage.getNamespaceName(), secretName);
        String value = secret.getData().get("ca.crt");
        assertThat("ca.crt in " + secretName + " should not be null", value, is(notNullValue()));
        initialCaCerts.put(secretName, value);
        Secret annotated = new SecretBuilder(secret).editMetadata().addToAnnotations(Ca.ANNO_STRIMZI_IO_FORCE_RENEW, "true").endMetadata().build();
        LOGGER.info("Patching secret {} with {}", secretName, Ca.ANNO_STRIMZI_IO_FORCE_RENEW);
        kubeClient().patchSecret(testStorage.getNamespaceName(), secretName, annotated);
    }
    if (zkShouldRoll) {
        LOGGER.info("Wait for zk to rolling restart ...");
        RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), testStorage.getZookeeperSelector(), 3, zkPods);
    }
    if (kafkaShouldRoll) {
        LOGGER.info("Wait for kafka to rolling restart ...");
        RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), testStorage.getKafkaSelector(), 3, kafkaPods);
    }
    if (eoShouldRoll) {
        LOGGER.info("Wait for EO to rolling restart ...");
        eoPod = DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(testStorage.getClusterName()), 1, eoPod);
    }
    if (keAndCCShouldRoll) {
        LOGGER.info("Wait for CC and KE to rolling restart ...");
        kePod = DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), KafkaExporterResources.deploymentName(testStorage.getClusterName()), 1, kePod);
        ccPod = DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), CruiseControlResources.deploymentName(testStorage.getClusterName()), 1, ccPod);
    }
    LOGGER.info("Checking the certificates have been replaced");
    for (String secretName : secrets) {
        Secret secret = kubeClient().getSecret(testStorage.getNamespaceName(), secretName);
        assertThat("Secret " + secretName + " should exist", secret, is(notNullValue()));
        assertThat("CA cert in " + secretName + " should have non-null 'data'", is(notNullValue()));
        String value = secret.getData().get("ca.crt");
        assertThat("CA cert in " + secretName + " should have changed", value, is(not(initialCaCerts.get(secretName))));
    }
    kafkaClients = new KafkaClientsBuilder(kafkaClients).withConsumerGroup(ClientUtils.generateRandomConsumerGroup()).build();
    resourceManager.createResource(extensionContext, kafkaClients.consumerTlsStrimzi(testStorage.getClusterName()));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    // Check a new client (signed by new client key) can consume
    String bobUserName = "bob-" + testStorage.getUserName();
    resourceManager.createResource(extensionContext, KafkaUserTemplates.tlsUser(testStorage.getClusterName(), bobUserName).build());
    kafkaClients = new KafkaClientsBuilder(kafkaClients).withConsumerGroup(ClientUtils.generateRandomConsumerGroup()).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(testStorage.getClusterName())).withUserName(bobUserName).build();
    resourceManager.createResource(extensionContext, kafkaClients.consumerTlsStrimzi(testStorage.getClusterName()));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    if (!zkShouldRoll) {
        assertThat("ZK pods should not roll, but did.", PodUtils.podSnapshot(testStorage.getNamespaceName(), testStorage.getZookeeperSelector()), is(zkPods));
    }
    if (!kafkaShouldRoll) {
        assertThat("Kafka pods should not roll, but did.", PodUtils.podSnapshot(testStorage.getNamespaceName(), testStorage.getKafkaSelector()), is(kafkaPods));
    }
    if (!eoShouldRoll) {
        assertThat("EO pod should not roll, but did.", DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(testStorage.getClusterName())), is(eoPod));
    }
    if (!keAndCCShouldRoll) {
        assertThat("CC pod should not roll, but did.", DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), CruiseControlResources.deploymentName(testStorage.getClusterName())), is(ccPod));
        assertThat("KE pod should not roll, but did.", DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), KafkaExporterResources.deploymentName(testStorage.getClusterName())), is(kePod));
    }
}

Example 69 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi-kafka-operator by strimzi.

the class CustomAuthorizerST method testAclWithSuperUser.

@ParallelTest
@Tag(INTERNAL_CLIENTS_USED)
void testAclWithSuperUser(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext, namespace);
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(CLUSTER_NAME, testStorage.getTopicName(), namespace).build());
    KafkaUser adminUser = KafkaUserTemplates.tlsUser(namespace, CLUSTER_NAME, ADMIN).editSpec().withNewKafkaUserAuthorizationSimple().addNewAcl().withNewAclRuleTopicResource().withName(testStorage.getTopicName()).endAclRuleTopicResource().withOperation(AclOperation.WRITE).endAcl().addNewAcl().withNewAclRuleTopicResource().withName(testStorage.getTopicName()).endAclRuleTopicResource().withOperation(AclOperation.DESCRIBE).endAcl().endKafkaUserAuthorizationSimple().endSpec().build();
    resourceManager.createResource(extensionContext, adminUser);
    LOGGER.info("Checking kafka super user:{} that is able to send messages to topic:{}", ADMIN, testStorage.getTopicName());
    KafkaClients kafkaClients = new KafkaClientsBuilder().withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withNamespaceName(testStorage.getNamespaceName()).withMessageCount(MESSAGE_COUNT).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(CLUSTER_NAME)).withTopicName(testStorage.getTopicName()).withUserName(ADMIN).build();
    resourceManager.createResource(extensionContext, kafkaClients.producerTlsStrimzi(CLUSTER_NAME));
    ClientUtils.waitForClientSuccess(testStorage.getProducerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Checking kafka super user:{} that is able to read messages to topic:{} regardless that " + "we configured Acls with only write operation", ADMIN, TOPIC_NAME);
    resourceManager.createResource(extensionContext, kafkaClients.consumerTlsStrimzi(CLUSTER_NAME));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
}

Example 70 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi-kafka-operator by strimzi.

the class CustomCaST method testReplacingCustomClusterKeyPairToInvokeRenewalProcess.

@ParallelNamespaceTest
void testReplacingCustomClusterKeyPairToInvokeRenewalProcess(ExtensionContext extensionContext) throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
    final TestStorage ts = new TestStorage(extensionContext);
    // 0. Generate root and intermediate certificate authority with cluster CA
    SystemTestCertHolder clusterCa = new SystemTestCertHolder("CN=" + extensionContext.getRequiredTestClass().getSimpleName() + "ClusterCA", KafkaResources.clusterCaCertificateSecretName(ts.getClusterName()), KafkaResources.clusterCaKeySecretName(ts.getClusterName()));
    prepareTestCaWithBundleAndKafkaCluster(extensionContext, clusterCa, ts);
    // ------- public key part
    // 4. Update the Secret for the CA certificate.
    // a) Edit the existing secret to add the new CA certificate and update the certificate generation annotation value.
    // b) Rename the current CA certificate to retain it
    final Secret clusterCaCertificateSecret = kubeClient(ts.getNamespaceName()).getSecret(ts.getNamespaceName(), KafkaResources.clusterCaCertificateSecretName(ts.getClusterName()));
    final String oldCaCertName = clusterCa.retrieveOldCertificateName(clusterCaCertificateSecret, "ca.crt");
    // store the old cert
    clusterCaCertificateSecret.getData().put(oldCaCertName, clusterCaCertificateSecret.getData().get("ca.crt"));
    // c) Encode your new CA certificate into base64.
    LOGGER.info("Generating a new custom 'Cluster certificate authority' with `Root` and `Intermediate` for Strimzi and PEM bundles.");
    clusterCa = new SystemTestCertHolder("CN=" + extensionContext.getRequiredTestClass().getSimpleName() + "ClusterCAv2", KafkaResources.clusterCaCertificateSecretName(ts.getClusterName()), KafkaResources.clusterCaKeySecretName(ts.getClusterName()));
    // d) Update the CA certificate.
    clusterCaCertificateSecret.getData().put("ca.crt", Base64.getEncoder().encodeToString(Files.readAllBytes(Paths.get(clusterCa.getBundle().getCertPath()))));
    // e) Increase the value of the CA certificate generation annotation.
    // f) Save the secret with the new CA certificate and certificate generation annotation value.
    SystemTestCertHolder.increaseCertGenerationCounterInSecret(clusterCaCertificateSecret, ts, Ca.ANNO_STRIMZI_IO_CA_CERT_GENERATION);
    // ------- private key part
    // 5. Update the Secret for the CA key used to sign your new CA certificate.
    // a) Edit the existing secret to add the new CA key and update the key generation annotation value.
    final Secret clusterCaKeySecret = kubeClient(ts.getNamespaceName()).getSecret(ts.getNamespaceName(), KafkaResources.clusterCaKeySecretName(ts.getClusterName()));
    // b) Encode the CA key into base64.
    // c) Update the CA key.
    final File strimziKeyPKCS8 = SystemTestCertManager.convertPrivateKeyToPKCS8File(clusterCa.getSystemTestCa().getPrivateKey());
    clusterCaKeySecret.getData().put("ca.key", Base64.getEncoder().encodeToString(Files.readAllBytes(Paths.get(strimziKeyPKCS8.getAbsolutePath()))));
    // d) Increase the value of the CA key generation annotation.
    // 6. Save the secret with the new CA key and key generation annotation value.
    SystemTestCertHolder.increaseCertGenerationCounterInSecret(clusterCaKeySecret, ts, Ca.ANNO_STRIMZI_IO_CA_KEY_GENERATION);
    // --- verification phase (Rolling Update of components)
    // 7. save the current state of the Kafka, ZooKeeper and EntityOperator pods
    Map<String, String> kafkaPods = PodUtils.podSnapshot(ts.getNamespaceName(), ts.getKafkaSelector());
    Map<String, String> zkPods = PodUtils.podSnapshot(ts.getNamespaceName(), ts.getZookeeperSelector());
    Map<String, String> eoPod = DeploymentUtils.depSnapshot(ts.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(ts.getClusterName()));
    // 8. Resume reconciliation from the pause.
    LOGGER.info("Resume the reconciliation of the Kafka custom resource ({}).", KafkaResources.kafkaStatefulSetName(ts.getClusterName()));
    KafkaResource.replaceKafkaResourceInSpecificNamespace(ts.getClusterName(), kafka -> {
        kafka.getMetadata().getAnnotations().remove(Annotations.ANNO_STRIMZI_IO_PAUSE_RECONCILIATION);
    }, ts.getNamespaceName());
    // 9. On the next reconciliation, the Cluster Operator performs a `rolling update`:
    // a) ZooKeeper
    // b) Kafka
    // c) and other components to trust the new CA certificate. (i.e., EntityOperator)
    // When the rolling update is complete, the Cluster Operator
    // will start a new one to generate new server certificates signed by the new CA key.
    zkPods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(ts.getNamespaceName(), ts.getZookeeperSelector(), 3, zkPods);
    kafkaPods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(ts.getNamespaceName(), ts.getKafkaSelector(), 3, kafkaPods);
    eoPod = DeploymentUtils.waitTillDepHasRolled(ts.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(ts.getClusterName()), 1, eoPod);
    // second Rolling update to generate new server certificates signed by the new CA key.
    RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(ts.getNamespaceName(), ts.getZookeeperSelector(), 3, zkPods);
    RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(ts.getNamespaceName(), ts.getKafkaSelector(), 3, kafkaPods);
    DeploymentUtils.waitTillDepHasRolled(ts.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(ts.getClusterName()), 1, eoPod);
    // 10. Try to produce messages
    producerMessages(extensionContext, ts);
}