Examples with TestStorage io.strimzi.systemtest.storage.TestStorage used on opensource projects

Example 96 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi by strimzi.

the class CustomCaST method testClientsCaCertificateRenew.

@ParallelNamespaceTest
void testClientsCaCertificateRenew(ExtensionContext extensionContext) {
    final TestStorage ts = new TestStorage(extensionContext);
    final String testSuite = extensionContext.getRequiredTestClass().getSimpleName();
    final SystemTestCertHolder clientsCa = new SystemTestCertHolder("CN=" + testSuite + "ClientsCA", KafkaResources.clientsCaCertificateSecretName(ts.getClusterName()), KafkaResources.clientsCaKeySecretName(ts.getClusterName()));
    // prepare custom Ca and copy that to the related Secrets
    clientsCa.prepareCustomSecretsFromBundles(ts.getNamespaceName(), ts.getClusterName());
    final X509Certificate clientsCert = SecretUtils.getCertificateFromSecret(kubeClient(ts.getNamespaceName()).getSecret(ts.getNamespaceName(), KafkaResources.clientsCaCertificateSecretName(ts.getClusterName())), "ca.crt");
    checkCustomCaCorrectness(clientsCa, clientsCert);
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(ts.getClusterName(), 3).editOrNewSpec().withNewClientsCa().withRenewalDays(15).withValidityDays(20).withGenerateCertificateAuthority(false).endClientsCa().endSpec().build());
    resourceManager.createResource(extensionContext, KafkaUserTemplates.tlsUser(ts.getClusterName(), ts.getUserName()).build());
    final Map<String, String> entityPods = DeploymentUtils.depSnapshot(ts.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(ts.getClusterName()));
    // Check initial clientsCA validity days
    Secret clientsCASecret = kubeClient(ts.getNamespaceName()).getSecret(ts.getNamespaceName(), KafkaResources.clientsCaCertificateSecretName(ts.getClusterName()));
    X509Certificate cacert = SecretUtils.getCertificateFromSecret(clientsCASecret, "ca.crt");
    final Date initialCertStartTime = cacert.getNotBefore();
    final Date initialCertEndTime = cacert.getNotAfter();
    // Check initial kafkauser validity days
    X509Certificate userCert = SecretUtils.getCertificateFromSecret(kubeClient(ts.getNamespaceName()).getSecret(ts.getNamespaceName(), ts.getUserName()), "user.crt");
    final Date initialKafkaUserCertStartTime = userCert.getNotBefore();
    final Date initialKafkaUserCertEndTime = userCert.getNotAfter();
    LOGGER.info("Change of kafka validity and renewal days - reconciliation should start.");
    final CertificateAuthority newClientsCA = new CertificateAuthority();
    newClientsCA.setRenewalDays(150);
    newClientsCA.setValidityDays(200);
    newClientsCA.setGenerateCertificateAuthority(false);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(ts.getClusterName(), k -> k.getSpec().setClientsCa(newClientsCA), ts.getNamespaceName());
    // Wait for reconciliation and verify certs have been updated
    DeploymentUtils.waitTillDepHasRolled(ts.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(ts.getClusterName()), 1, entityPods);
    // Read renewed secret/certs again
    clientsCASecret = kubeClient(ts.getNamespaceName()).getSecret(ts.getNamespaceName(), KafkaResources.clientsCaCertificateSecretName(ts.getClusterName()));
    cacert = SecretUtils.getCertificateFromSecret(clientsCASecret, "ca.crt");
    final Date changedCertStartTime = cacert.getNotBefore();
    final Date changedCertEndTime = cacert.getNotAfter();
    userCert = SecretUtils.getCertificateFromSecret(kubeClient(ts.getNamespaceName()).getSecret(ts.getNamespaceName(), ts.getUserName()), "user.crt");
    final Date changedKafkaUserCertStartTime = userCert.getNotBefore();
    final Date changedKafkaUserCertEndTime = userCert.getNotAfter();
    LOGGER.info("Initial ClientsCA cert dates: " + initialCertStartTime + " --> " + initialCertEndTime);
    LOGGER.info("Changed ClientsCA cert dates: " + changedCertStartTime + " --> " + changedCertEndTime);
    LOGGER.info("Initial userCert dates: " + initialKafkaUserCertStartTime + " --> " + initialKafkaUserCertEndTime);
    LOGGER.info("Changed userCert dates: " + changedKafkaUserCertStartTime + " --> " + changedKafkaUserCertEndTime);
    assertThat("ClientsCA cert should not have changed.", initialCertEndTime.compareTo(changedCertEndTime) == 0);
    assertThat("UserCert start date has been renewed", initialKafkaUserCertStartTime.compareTo(changedKafkaUserCertStartTime) < 0);
    assertThat("UserCert end date has been renewed", initialKafkaUserCertEndTime.compareTo(changedKafkaUserCertEndTime) < 0);
}

Example 97 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi by strimzi.

the class CustomCaST method testReplacingCustomClusterKeyPairToInvokeRenewalProcess.

@ParallelNamespaceTest
void testReplacingCustomClusterKeyPairToInvokeRenewalProcess(ExtensionContext extensionContext) throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
    final TestStorage ts = new TestStorage(extensionContext);
    // 0. Generate root and intermediate certificate authority with cluster CA
    SystemTestCertHolder clusterCa = new SystemTestCertHolder("CN=" + extensionContext.getRequiredTestClass().getSimpleName() + "ClusterCA", KafkaResources.clusterCaCertificateSecretName(ts.getClusterName()), KafkaResources.clusterCaKeySecretName(ts.getClusterName()));
    prepareTestCaWithBundleAndKafkaCluster(extensionContext, clusterCa, ts);
    // ------- public key part
    // 4. Update the Secret for the CA certificate.
    // a) Edit the existing secret to add the new CA certificate and update the certificate generation annotation value.
    // b) Rename the current CA certificate to retain it
    final Secret clusterCaCertificateSecret = kubeClient(ts.getNamespaceName()).getSecret(ts.getNamespaceName(), KafkaResources.clusterCaCertificateSecretName(ts.getClusterName()));
    final String oldCaCertName = clusterCa.retrieveOldCertificateName(clusterCaCertificateSecret, "ca.crt");
    // store the old cert
    clusterCaCertificateSecret.getData().put(oldCaCertName, clusterCaCertificateSecret.getData().get("ca.crt"));
    // c) Encode your new CA certificate into base64.
    LOGGER.info("Generating a new custom 'Cluster certificate authority' with `Root` and `Intermediate` for Strimzi and PEM bundles.");
    clusterCa = new SystemTestCertHolder("CN=" + extensionContext.getRequiredTestClass().getSimpleName() + "ClusterCAv2", KafkaResources.clusterCaCertificateSecretName(ts.getClusterName()), KafkaResources.clusterCaKeySecretName(ts.getClusterName()));
    // d) Update the CA certificate.
    clusterCaCertificateSecret.getData().put("ca.crt", Base64.getEncoder().encodeToString(Files.readAllBytes(Paths.get(clusterCa.getBundle().getCertPath()))));
    // e) Increase the value of the CA certificate generation annotation.
    // f) Save the secret with the new CA certificate and certificate generation annotation value.
    SystemTestCertHolder.increaseCertGenerationCounterInSecret(clusterCaCertificateSecret, ts, Ca.ANNO_STRIMZI_IO_CA_CERT_GENERATION);
    // ------- private key part
    // 5. Update the Secret for the CA key used to sign your new CA certificate.
    // a) Edit the existing secret to add the new CA key and update the key generation annotation value.
    final Secret clusterCaKeySecret = kubeClient(ts.getNamespaceName()).getSecret(ts.getNamespaceName(), KafkaResources.clusterCaKeySecretName(ts.getClusterName()));
    // b) Encode the CA key into base64.
    // c) Update the CA key.
    final File strimziKeyPKCS8 = SystemTestCertManager.convertPrivateKeyToPKCS8File(clusterCa.getSystemTestCa().getPrivateKey());
    clusterCaKeySecret.getData().put("ca.key", Base64.getEncoder().encodeToString(Files.readAllBytes(Paths.get(strimziKeyPKCS8.getAbsolutePath()))));
    // d) Increase the value of the CA key generation annotation.
    // 6. Save the secret with the new CA key and key generation annotation value.
    SystemTestCertHolder.increaseCertGenerationCounterInSecret(clusterCaKeySecret, ts, Ca.ANNO_STRIMZI_IO_CA_KEY_GENERATION);
    // --- verification phase (Rolling Update of components)
    // 7. save the current state of the Kafka, ZooKeeper and EntityOperator pods
    Map<String, String> kafkaPods = PodUtils.podSnapshot(ts.getNamespaceName(), ts.getKafkaSelector());
    Map<String, String> zkPods = PodUtils.podSnapshot(ts.getNamespaceName(), ts.getZookeeperSelector());
    Map<String, String> eoPod = DeploymentUtils.depSnapshot(ts.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(ts.getClusterName()));
    // 8. Resume reconciliation from the pause.
    LOGGER.info("Resume the reconciliation of the Kafka custom resource ({}).", KafkaResources.kafkaStatefulSetName(ts.getClusterName()));
    KafkaResource.replaceKafkaResourceInSpecificNamespace(ts.getClusterName(), kafka -> {
        kafka.getMetadata().getAnnotations().remove(Annotations.ANNO_STRIMZI_IO_PAUSE_RECONCILIATION);
    }, ts.getNamespaceName());
    // 9. On the next reconciliation, the Cluster Operator performs a `rolling update`:
    // a) ZooKeeper
    // b) Kafka
    // c) and other components to trust the new CA certificate. (i.e., EntityOperator)
    // When the rolling update is complete, the Cluster Operator
    // will start a new one to generate new server certificates signed by the new CA key.
    zkPods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(ts.getNamespaceName(), ts.getZookeeperSelector(), 3, zkPods);
    kafkaPods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(ts.getNamespaceName(), ts.getKafkaSelector(), 3, kafkaPods);
    eoPod = DeploymentUtils.waitTillDepHasRolled(ts.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(ts.getClusterName()), 1, eoPod);
    // second Rolling update to generate new server certificates signed by the new CA key.
    RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(ts.getNamespaceName(), ts.getZookeeperSelector(), 3, zkPods);
    RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(ts.getNamespaceName(), ts.getKafkaSelector(), 3, kafkaPods);
    DeploymentUtils.waitTillDepHasRolled(ts.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(ts.getClusterName()), 1, eoPod);
    // 10. Try to produce messages
    producerMessages(extensionContext, ts);
}

Example 98 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi by strimzi.

the class CustomCaST method testReplacingCustomClientsKeyPairToInvokeRenewalProcess.

@ParallelNamespaceTest
void testReplacingCustomClientsKeyPairToInvokeRenewalProcess(ExtensionContext extensionContext) throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
    final TestStorage ts = new TestStorage(extensionContext);
    // 0. Generate root and intermediate certificate authority with clients CA
    SystemTestCertHolder clientsCa = new SystemTestCertHolder("CN=" + extensionContext.getRequiredTestClass().getSimpleName() + "ClientsCA", KafkaResources.clientsCaCertificateSecretName(ts.getClusterName()), KafkaResources.clientsCaKeySecretName(ts.getClusterName()));
    prepareTestCaWithBundleAndKafkaCluster(extensionContext, clientsCa, ts);
    // ------- public key part
    // 4. Update the Secret for the CA certificate.
    // a) Edit the existing secret to add the new CA certificate and update the certificate generation annotation value.
    // b) Rename the current CA certificate to retain it
    final Secret clientsCaCertificateSecret = kubeClient(ts.getNamespaceName()).getSecret(ts.getNamespaceName(), KafkaResources.clientsCaCertificateSecretName(ts.getClusterName()));
    final String oldCaCertName = clientsCa.retrieveOldCertificateName(clientsCaCertificateSecret, "ca.crt");
    // store the old cert
    clientsCaCertificateSecret.getData().put(oldCaCertName, clientsCaCertificateSecret.getData().get("ca.crt"));
    // c) Encode your new CA certificate into base64.
    LOGGER.info("Generating a new custom 'User certificate authority' with `Root` and `Intermediate` for Strimzi and PEM bundles.");
    clientsCa = new SystemTestCertHolder("CN=" + extensionContext.getRequiredTestClass().getSimpleName() + "ClientsCAv2", KafkaResources.clientsCaCertificateSecretName(ts.getClusterName()), KafkaResources.clientsCaKeySecretName(ts.getClusterName()));
    // d) Update the CA certificate.
    clientsCaCertificateSecret.getData().put("ca.crt", Base64.getEncoder().encodeToString(Files.readAllBytes(Paths.get(clientsCa.getBundle().getCertPath()))));
    // e) Increase the value of the CA certificate generation annotation.
    // f) Save the secret with the new CA certificate and certificate generation annotation value.
    SystemTestCertHolder.increaseCertGenerationCounterInSecret(clientsCaCertificateSecret, ts, Ca.ANNO_STRIMZI_IO_CA_CERT_GENERATION);
    // ------- private key part
    // 5. Update the Secret for the CA key used to sign your new CA certificate.
    // a) Edit the existing secret to add the new CA key and update the key generation annotation value.
    final Secret clientsCaKeySecret = kubeClient(ts.getNamespaceName()).getSecret(ts.getNamespaceName(), KafkaResources.clientsCaKeySecretName(ts.getClusterName()));
    // b) Encode the CA key into base64.
    // c) Update the CA key.
    final File strimziKeyPKCS8 = SystemTestCertManager.convertPrivateKeyToPKCS8File(clientsCa.getSystemTestCa().getPrivateKey());
    clientsCaKeySecret.getData().put("ca.key", Base64.getEncoder().encodeToString(Files.readAllBytes(Paths.get(strimziKeyPKCS8.getAbsolutePath()))));
    // d) Increase the value of the CA key generation annotation.
    // 6. Save the secret with the new CA key and key generation annotation value.
    SystemTestCertHolder.increaseCertGenerationCounterInSecret(clientsCaKeySecret, ts, Ca.ANNO_STRIMZI_IO_CA_KEY_GENERATION);
    // --- verification phase (Rolling Update of components)
    // 7. save the current state of the Kafka, ZooKeeper and EntityOperator pods
    final Map<String, String> kafkaPods = PodUtils.podSnapshot(ts.getNamespaceName(), ts.getKafkaSelector());
    final Map<String, String> zkPods = PodUtils.podSnapshot(ts.getNamespaceName(), ts.getZookeeperSelector());
    final Map<String, String> eoPod = DeploymentUtils.depSnapshot(ts.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(ts.getClusterName()));
    // 8. Resume reconciliation from the pause.
    LOGGER.info("Resume the reconciliation of the Kafka custom resource ({}).", KafkaResources.kafkaStatefulSetName(ts.getClusterName()));
    KafkaResource.replaceKafkaResourceInSpecificNamespace(ts.getClusterName(), kafka -> {
        kafka.getMetadata().getAnnotations().remove(Annotations.ANNO_STRIMZI_IO_PAUSE_RECONCILIATION);
    }, ts.getNamespaceName());
    // 9. On the next reconciliation, the Cluster Operator performs a `rolling update` only for the
    // `Kafka pods`. When the rolling update is complete, the Cluster Operator will start a new one to
    // generate new server certificates signed by the new CA key.
    // a) ZooKeeper must not roll
    RollingUpdateUtils.waitForNoRollingUpdate(ts.getNamespaceName(), ts.getZookeeperSelector(), zkPods);
    assertThat(RollingUpdateUtils.componentHasRolled(ts.getNamespaceName(), ts.getZookeeperSelector(), zkPods), is(Boolean.FALSE));
    // b) Kafka has to roll
    RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(ts.getNamespaceName(), ts.getKafkaSelector(), 3, kafkaPods);
    // c) EO must not roll
    DeploymentUtils.waitForNoRollingUpdate(ts.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(ts.getClusterName()), eoPod);
    // 10. Try to produce messages
    producerMessages(extensionContext, ts);
}

Example 99 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi by strimzi.

the class OauthAuthorizationIsolatedST method testKeycloakAuthorizerToDelegateToSimpleAuthorizer.

@ParallelNamespaceTest
@Order(10)
void testKeycloakAuthorizerToDelegateToSimpleAuthorizer(ExtensionContext extensionContext) {
    TestStorage testStorage = new TestStorage(extensionContext);
    // we have to create keycloak, team-a-client and team-b-client secret from `infra-namespace` to the new namespace
    resourceManager.createResource(extensionContext, kubeClient().getSecret(clusterOperator.getDeploymentNamespace(), KeycloakInstance.KEYCLOAK_SECRET_NAME));
    resourceManager.createResource(extensionContext, kubeClient().getSecret(clusterOperator.getDeploymentNamespace(), TEAM_A_CLIENT_SECRET));
    resourceManager.createResource(extensionContext, kubeClient().getSecret(clusterOperator.getDeploymentNamespace(), TEAM_B_CLIENT_SECRET));
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(testStorage.getClusterName(), 1, 1).editSpec().editKafka().withListeners(OauthAbstractST.BUILD_OAUTH_TLS_LISTENER.apply(keycloakInstance)).withNewKafkaAuthorizationKeycloak().withClientId(KAFKA_CLIENT_ID).withDisableTlsHostnameVerification(true).withDelegateToKafkaAcls(true).withTlsTrustedCertificates(new CertSecretSourceBuilder().withSecretName(KeycloakInstance.KEYCLOAK_SECRET_NAME).withCertificate(KeycloakInstance.KEYCLOAK_SECRET_CERT).build()).withTokenEndpointUri(keycloakInstance.getOauthTokenEndpointUri()).endKafkaAuthorizationKeycloak().endKafka().endSpec().build());
    resourceManager.createResource(extensionContext, KafkaUserTemplates.tlsUser(testStorage.getNamespaceName(), testStorage.getClusterName(), TEAM_A_CLIENT).build());
    resourceManager.createResource(extensionContext, KafkaUserTemplates.tlsUser(testStorage.getNamespaceName(), testStorage.getClusterName(), TEAM_B_CLIENT).build());
    final String teamAProducerName = TEAM_A_PRODUCER_NAME + "-" + testStorage.getClusterName();
    final String teamAConsumerName = TEAM_A_CONSUMER_NAME + "-" + testStorage.getClusterName();
    final String topicName = TOPIC_A + "-" + testStorage.getTopicName();
    final String consumerGroup = "a-consumer_group-" + testStorage.getConsumerName();
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(testStorage.getClusterName(), topicName, testStorage.getNamespaceName()).build());
    KafkaOauthClients teamAOauthClientJob = new KafkaOauthClientsBuilder().withNamespaceName(testStorage.getNamespaceName()).withProducerName(teamAProducerName).withConsumerName(teamAConsumerName).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(testStorage.getClusterName())).withTopicName(topicName).withMessageCount(MESSAGE_COUNT).withConsumerGroup(consumerGroup).withOauthClientId(TEAM_A_CLIENT).withOauthClientSecret(TEAM_A_CLIENT_SECRET).withOauthTokenEndpointUri(keycloakInstance.getOauthTokenEndpointUri()).build();
    resourceManager.createResource(extensionContext, teamAOauthClientJob.producerStrimziOauthTls(testStorage.getClusterName()));
    ClientUtils.waitForClientSuccess(teamAProducerName, testStorage.getNamespaceName(), MESSAGE_COUNT);
    resourceManager.createResource(extensionContext, teamAOauthClientJob.consumerStrimziOauthTls(testStorage.getClusterName()));
    ClientUtils.waitForClientSuccess(teamAConsumerName, testStorage.getNamespaceName(), MESSAGE_COUNT);
}

Example 100 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi by strimzi.

the class DrainCleanerIsolatedST method testDrainCleanerWithComponentsDuringNodeDraining.

@IsolatedTest
// We refer to 6 worker nodes to have always 2 nodes with same labels to properly evacuate pods from one node to another
@MultiNodeClusterOnly(workerNodeCount = 6)
void testDrainCleanerWithComponentsDuringNodeDraining(ExtensionContext extensionContext) {
    TestStorage testStorage = new TestStorage(extensionContext, Constants.DRAIN_CLEANER_NAMESPACE);
    String rackKey = "rack-key";
    final int replicas = 3;
    int size = 5;
    List<String> topicNames = IntStream.range(0, size).boxed().map(i -> testStorage.getTopicName() + "-" + i).collect(Collectors.toList());
    List<String> producerNames = IntStream.range(0, size).boxed().map(i -> testStorage.getProducerName() + "-" + i).collect(Collectors.toList());
    List<String> consumerNames = IntStream.range(0, size).boxed().map(i -> testStorage.getConsumerName() + "-" + i).collect(Collectors.toList());
    List<String> continuousConsumerGroups = IntStream.range(0, size).boxed().map(i -> "continuous-consumer-group-" + i).collect(Collectors.toList());
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(testStorage.getClusterName(), replicas).editMetadata().withNamespace(Constants.DRAIN_CLEANER_NAMESPACE).endMetadata().editSpec().editKafka().withNewRack().withTopologyKey(rackKey).endRack().editOrNewTemplate().editOrNewPodDisruptionBudget().withMaxUnavailable(0).endPodDisruptionBudget().withNewPod().withAffinity(new AffinityBuilder().withNewPodAntiAffinity().addNewRequiredDuringSchedulingIgnoredDuringExecution().editOrNewLabelSelector().addNewMatchExpression().withKey(rackKey).withOperator("In").withValues("zone").endMatchExpression().endLabelSelector().withTopologyKey(rackKey).endRequiredDuringSchedulingIgnoredDuringExecution().endPodAntiAffinity().build()).endPod().endTemplate().endKafka().editZookeeper().editOrNewTemplate().editOrNewPodDisruptionBudget().withMaxUnavailable(0).endPodDisruptionBudget().withNewPod().withAffinity(new AffinityBuilder().withNewPodAntiAffinity().addNewRequiredDuringSchedulingIgnoredDuringExecution().editOrNewLabelSelector().addNewMatchExpression().withKey(rackKey).withOperator("In").withValues("zone").endMatchExpression().endLabelSelector().withTopologyKey(rackKey).endRequiredDuringSchedulingIgnoredDuringExecution().endPodAntiAffinity().build()).endPod().endTemplate().endZookeeper().endSpec().build());
    topicNames.forEach(topic -> resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(testStorage.getClusterName(), topic, 3, 3, 2).editMetadata().withNamespace(Constants.DRAIN_CLEANER_NAMESPACE).endMetadata().build()));
    drainCleaner.createDrainCleaner(extensionContext);
    String kafkaName = KafkaResources.kafkaStatefulSetName(testStorage.getClusterName());
    String zkName = KafkaResources.zookeeperStatefulSetName(testStorage.getClusterName());
    Map<String, List<String>> nodesWithPods = NodeUtils.getPodsForEachNodeInNamespace(Constants.DRAIN_CLEANER_NAMESPACE);
    // remove all pods from map, which doesn't contain "kafka" or "zookeeper" in its name
    nodesWithPods.forEach((node, podlist) -> podlist.retainAll(podlist.stream().filter(podName -> (podName.contains("kafka") || podName.contains("zookeeper"))).collect(Collectors.toList())));
    String producerAdditionConfiguration = "delivery.timeout.ms=30000\nrequest.timeout.ms=30000";
    KafkaClients kafkaBasicExampleClients;
    for (int i = 0; i < size; i++) {
        kafkaBasicExampleClients = new KafkaClientsBuilder().withProducerName(producerNames.get(i)).withConsumerName(consumerNames.get(i)).withTopicName(topicNames.get(i)).withConsumerGroup(continuousConsumerGroups.get(i)).withMessageCount(300).withNamespaceName(Constants.DRAIN_CLEANER_NAMESPACE).withBootstrapAddress(KafkaResources.plainBootstrapAddress(testStorage.getClusterName())).withDelayMs(1000).withAdditionalConfig(producerAdditionConfiguration).build();
        resourceManager.createResource(extensionContext, kafkaBasicExampleClients.producerStrimzi(), kafkaBasicExampleClients.consumerStrimzi());
    }
    LOGGER.info("Starting Node drain");
    nodesWithPods.forEach((nodeName, podList) -> {
        String zkPodName = podList.stream().filter(podName -> podName.contains("zookeeper")).findFirst().get();
        String kafkaPodName = podList.stream().filter(podName -> podName.contains("kafka")).findFirst().get();
        Map<String, String> kafkaPod = PodUtils.podSnapshot(Constants.DRAIN_CLEANER_NAMESPACE, testStorage.getKafkaSelector()).entrySet().stream().filter(snapshot -> snapshot.getKey().equals(kafkaPodName)).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
        Map<String, String> zkPod = PodUtils.podSnapshot(Constants.DRAIN_CLEANER_NAMESPACE, testStorage.getZookeeperSelector()).entrySet().stream().filter(snapshot -> snapshot.getKey().equals(zkPodName)).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
        NodeUtils.drainNode(nodeName);
        NodeUtils.cordonNode(nodeName, true);
        RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(Constants.DRAIN_CLEANER_NAMESPACE, testStorage.getZookeeperSelector(), replicas, zkPod);
        RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(Constants.DRAIN_CLEANER_NAMESPACE, testStorage.getKafkaSelector(), replicas, kafkaPod);
    });
    producerNames.forEach(producer -> ClientUtils.waitForClientsSuccess(producer, consumerNames.get(producerNames.indexOf(producer)), Constants.DRAIN_CLEANER_NAMESPACE, 300));
}