Examples with KafkaClients io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients used on opensource projects

Example 11 with KafkaClients

use of io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients in project strimzi-kafka-operator by strimzi.

the class MultipleClusterOperatorsIsolatedST method testMultipleCOsInDifferentNamespaces.

@IsolatedTest
@Tag(CONNECT)
@Tag(CONNECT_COMPONENTS)
void testMultipleCOsInDifferentNamespaces(ExtensionContext extensionContext) {
    // Strimzi is deployed with cluster-wide access in this class STRIMZI_RBAC_SCOPE=NAMESPACE won't work
    assumeFalse(Environment.isNamespaceRbacScope());
    String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    String producerName = "hello-world-producer";
    String consumerName = "hello-world-consumer";
    deployCOInNamespace(extensionContext, FIRST_CO_NAME, FIRST_NAMESPACE, FIRST_CO_SELECTOR_ENV, true);
    deployCOInNamespace(extensionContext, SECOND_CO_NAME, SECOND_NAMESPACE, SECOND_CO_SELECTOR_ENV, true);
    cluster.createNamespace(DEFAULT_NAMESPACE);
    cluster.setNamespace(DEFAULT_NAMESPACE);
    LOGGER.info("Deploying Kafka without CR selector");
    resourceManager.createResource(extensionContext, false, KafkaTemplates.kafkaEphemeral(clusterName, 3, 3).editMetadata().withNamespace(DEFAULT_NAMESPACE).endMetadata().build());
    // checking that no pods with prefix 'clusterName' will be created in some time
    PodUtils.waitUntilPodStabilityReplicasCount(DEFAULT_NAMESPACE, clusterName, 0);
    LOGGER.info("Adding {} selector of {} into Kafka CR", FIRST_CO_SELECTOR, FIRST_CO_NAME);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, kafka -> kafka.getMetadata().setLabels(FIRST_CO_SELECTOR), DEFAULT_NAMESPACE);
    KafkaUtils.waitForKafkaReady(DEFAULT_NAMESPACE, clusterName);
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName).editMetadata().withNamespace(DEFAULT_NAMESPACE).endMetadata().build(), KafkaConnectTemplates.kafkaConnect(extensionContext, clusterName, 1, false).editOrNewMetadata().addToLabels(FIRST_CO_SELECTOR).addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").withNamespace(DEFAULT_NAMESPACE).endMetadata().build());
    String kafkaConnectPodName = kubeClient(DEFAULT_NAMESPACE).listPods(DEFAULT_NAMESPACE, clusterName, Labels.STRIMZI_KIND_LABEL, KafkaConnect.RESOURCE_KIND).get(0).getMetadata().getName();
    LOGGER.info("Deploying KafkaConnector with file sink and CR selector - {} - different than selector in Kafka", SECOND_CO_SELECTOR);
    resourceManager.createResource(extensionContext, KafkaConnectorTemplates.kafkaConnector(clusterName).editOrNewMetadata().addToLabels(SECOND_CO_SELECTOR).withNamespace(DEFAULT_NAMESPACE).endMetadata().editSpec().withClassName("org.apache.kafka.connect.file.FileStreamSinkConnector").addToConfig("file", Constants.DEFAULT_SINK_FILE_PATH).addToConfig("key.converter", "org.apache.kafka.connect.storage.StringConverter").addToConfig("value.converter", "org.apache.kafka.connect.storage.StringConverter").addToConfig("topics", topicName).endSpec().build());
    KafkaClients basicClients = new KafkaClientsBuilder().withNamespaceName(DEFAULT_NAMESPACE).withProducerName(producerName).withConsumerName(consumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(clusterName)).withTopicName(topicName).withMessageCount(MESSAGE_COUNT).build();
    resourceManager.createResource(extensionContext, basicClients.producerStrimzi());
    ClientUtils.waitForClientSuccess(producerName, DEFAULT_NAMESPACE, MESSAGE_COUNT);
    KafkaConnectUtils.waitForMessagesInKafkaConnectFileSink(kafkaConnectPodName, Constants.DEFAULT_SINK_FILE_PATH, "Hello-world - 99");
}

Example 12 with KafkaClients

use of io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients in project strimzi-kafka-operator by strimzi.

the class AlternativeReconcileTriggersST method testManualTriggeringRollingUpdate.

@ParallelNamespaceTest
@SuppressWarnings("checkstyle:MethodLength")
void testManualTriggeringRollingUpdate(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    final String continuousTopicName = "continuous-topic";
    // 500 messages will take 500 seconds in that case
    final int continuousClientsMessageCount = 500;
    final String producerName = "hello-world-producer";
    final String consumerName = "hello-world-consumer";
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(clusterName, 3, 3).build());
    String kafkaName = KafkaResources.kafkaStatefulSetName(clusterName);
    String zkName = KafkaResources.zookeeperStatefulSetName(clusterName);
    LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, kafkaName);
    LabelSelector zkSelector = KafkaResource.getLabelSelector(clusterName, zkName);
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
    Map<String, String> zkPods = PodUtils.podSnapshot(namespaceName, zkSelector);
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName, namespaceName).build());
    // ##############################
    // Attach clients which will continuously produce/consume messages to/from Kafka brokers during rolling update
    // ##############################
    // Setup topic, which has 3 replicas and 2 min.isr to see if producer will be able to work during rolling update
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, continuousTopicName, 3, 3, 2).build());
    String producerAdditionConfiguration = "delivery.timeout.ms=20000\nrequest.timeout.ms=20000";
    // Add transactional id to make producer transactional
    producerAdditionConfiguration = producerAdditionConfiguration.concat("\ntransactional.id=" + continuousTopicName + ".1");
    producerAdditionConfiguration = producerAdditionConfiguration.concat("\nenable.idempotence=true");
    KafkaClients kafkaBasicClientJob = new KafkaClientsBuilder().withProducerName(producerName).withConsumerName(consumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(clusterName)).withTopicName(continuousTopicName).withMessageCount(continuousClientsMessageCount).withAdditionalConfig(producerAdditionConfiguration).withDelayMs(1000).withNamespaceName(namespaceName).build();
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.producerStrimzi());
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.consumerStrimzi());
    // ##############################
    String userName = KafkaUserUtils.generateRandomNameOfKafkaUser();
    KafkaUser user = KafkaUserTemplates.tlsUser(clusterName, userName).build();
    resourceManager.createResource(extensionContext, user);
    resourceManager.createResource(extensionContext, false, KafkaClientsTemplates.kafkaClients(true, kafkaClientsName, user).build());
    final String kafkaClientsPodName = PodUtils.getPodsByPrefixInNameWithDynamicWait(namespaceName, kafkaClientsName).get(0).getMetadata().getName();
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(kafkaClientsPodName).withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withKafkaUsername(userName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    internalKafkaClient.produceTlsMessagesUntilOperationIsSuccessful(MESSAGE_COUNT);
    // rolling update for kafka
    // set annotation to trigger Kafka rolling update
    LOGGER.info("Annotate Kafka {} {} with manual rolling update annotation", Environment.isStrimziPodSetEnabled() ? StrimziPodSet.RESOURCE_KIND : Constants.STATEFUL_SET, kafkaName);
    StUtils.annotateStatefulSetOrStrimziPodSet(namespaceName, kafkaName, Collections.singletonMap(Annotations.ANNO_STRIMZI_IO_MANUAL_ROLLING_UPDATE, "true"));
    // check annotation to trigger rolling update
    assertThat(Boolean.parseBoolean(StUtils.getAnnotationsOfStatefulSetOrStrimziPodSet(namespaceName, kafkaName).get(Annotations.ANNO_STRIMZI_IO_MANUAL_ROLLING_UPDATE)), is(true));
    RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, 3, kafkaPods);
    // wait when annotation will be removed
    TestUtils.waitFor("CO removes rolling update annotation", Constants.WAIT_FOR_ROLLING_UPDATE_INTERVAL, Constants.GLOBAL_TIMEOUT, () -> StUtils.getAnnotationsOfStatefulSetOrStrimziPodSet(namespaceName, zkName) == null || !StUtils.getAnnotationsOfStatefulSetOrStrimziPodSet(namespaceName, zkName).containsKey(Annotations.ANNO_STRIMZI_IO_MANUAL_ROLLING_UPDATE));
    int received = internalKafkaClient.receiveMessagesTls();
    assertThat(received, is(MESSAGE_COUNT));
    // rolling update for zookeeper
    // set annotation to trigger Zookeeper rolling update
    LOGGER.info("Annotate Zookeeper {} {} with manual rolling update annotation", Environment.isStrimziPodSetEnabled() ? StrimziPodSet.RESOURCE_KIND : Constants.STATEFUL_SET, zkName);
    StUtils.annotateStatefulSetOrStrimziPodSet(namespaceName, zkName, Collections.singletonMap(Annotations.ANNO_STRIMZI_IO_MANUAL_ROLLING_UPDATE, "true"));
    // check annotation to trigger rolling update
    assertThat(Boolean.parseBoolean(StUtils.getAnnotationsOfStatefulSetOrStrimziPodSet(namespaceName, zkName).get(Annotations.ANNO_STRIMZI_IO_MANUAL_ROLLING_UPDATE)), is(true));
    RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, zkSelector, 3, zkPods);
    // wait when annotation will be removed
    TestUtils.waitFor("CO removes rolling update annotation", Constants.WAIT_FOR_ROLLING_UPDATE_INTERVAL, Constants.GLOBAL_TIMEOUT, () -> StUtils.getAnnotationsOfStatefulSetOrStrimziPodSet(namespaceName, zkName) == null || !StUtils.getAnnotationsOfStatefulSetOrStrimziPodSet(namespaceName, zkName).containsKey(Annotations.ANNO_STRIMZI_IO_MANUAL_ROLLING_UPDATE));
    internalKafkaClient = internalKafkaClient.toBuilder().withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
    received = internalKafkaClient.receiveMessagesTls();
    assertThat(received, is(MESSAGE_COUNT));
    // Create new topic to ensure, that ZK is working properly
    String newTopicName = KafkaTopicUtils.generateRandomNameOfTopic();
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, newTopicName, 1, 1).build());
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(newTopicName).withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
    int sent = internalKafkaClient.sendMessagesTls();
    assertThat(sent, is(MESSAGE_COUNT));
    received = internalKafkaClient.receiveMessagesTls();
    assertThat(received, is(sent));
    // ##############################
    // Validate that continuous clients finished successfully
    // ##############################
    ClientUtils.waitTillContinuousClientsFinish(producerName, consumerName, namespaceName, continuousClientsMessageCount);
// ##############################
}

Example 13 with KafkaClients

use of io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients in project strimzi-kafka-operator by strimzi.

the class ClusterOperationIsolatedST method testAvailabilityDuringNodeDrain.

@IsolatedTest
@MultiNodeClusterOnly
@RequiredMinKubeApiVersion(version = 1.15)
void testAvailabilityDuringNodeDrain(ExtensionContext extensionContext) {
    String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    int size = 5;
    List<String> topicNames = IntStream.range(0, size).boxed().map(i -> "test-topic-" + i).collect(Collectors.toList());
    List<String> producerNames = IntStream.range(0, size).boxed().map(i -> "hello-world-producer-" + i).collect(Collectors.toList());
    List<String> consumerNames = IntStream.range(0, size).boxed().map(i -> "hello-world-consumer-" + i).collect(Collectors.toList());
    List<String> continuousConsumerGroups = IntStream.range(0, size).boxed().map(i -> "continuous-consumer-group-" + i).collect(Collectors.toList());
    int continuousClientsMessageCount = 300;
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(clusterName, 3, 3).editOrNewSpec().editEntityOperator().editUserOperator().withReconciliationIntervalSeconds(30).endUserOperator().endEntityOperator().endSpec().build());
    topicNames.forEach(topicName -> resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName, 3, 3, 2).build()));
    String producerAdditionConfiguration = "delivery.timeout.ms=20000\nrequest.timeout.ms=20000";
    KafkaClients kafkaBasicClientResource;
    for (int i = 0; i < size; i++) {
        kafkaBasicClientResource = new KafkaClientsBuilder().withProducerName(producerNames.get(i)).withConsumerName(consumerNames.get(i)).withBootstrapAddress(KafkaResources.plainBootstrapAddress(clusterName)).withTopicName(topicNames.get(i)).withMessageCount(continuousClientsMessageCount).withAdditionalConfig(producerAdditionConfiguration).withConsumerGroup(continuousConsumerGroups.get(i)).withDelayMs(1000).build();
        resourceManager.createResource(extensionContext, kafkaBasicClientResource.producerStrimzi());
        resourceManager.createResource(extensionContext, kafkaBasicClientResource.consumerStrimzi());
    }
    // ##############################
    // Nodes draining
    // ##############################
    kubeClient().getClusterWorkers().forEach(node -> {
        NodeUtils.drainNode(node.getMetadata().getName());
        NodeUtils.cordonNode(node.getMetadata().getName(), true);
    });
    producerNames.forEach(producerName -> ClientUtils.waitTillContinuousClientsFinish(producerName, consumerNames.get(producerName.indexOf(producerName)), NAMESPACE, continuousClientsMessageCount));
    producerNames.forEach(producerName -> kubeClient().deleteJob(producerName));
    consumerNames.forEach(consumerName -> kubeClient().deleteJob(consumerName));
}

Example 14 with KafkaClients

use of io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients in project strimzi-kafka-operator by strimzi.

the class SpecificIsolatedST method testRackAware.

@IsolatedTest("UtestRackAwareConnectWrongDeploymentsing more tha one Kafka cluster in one namespace")
@Tag(REGRESSION)
@Tag(INTERNAL_CLIENTS_USED)
void testRackAware(ExtensionContext extensionContext) {
    assumeFalse(Environment.isNamespaceRbacScope());
    String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    String producerName = "hello-world-producer";
    String consumerName = "hello-world-consumer";
    String rackKey = "rack-key";
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(clusterName, 1, 1).editSpec().editKafka().withNewRack().withTopologyKey(rackKey).endRack().endKafka().endSpec().build());
    Affinity kafkaPodSpecAffinity = StUtils.getStatefulSetOrStrimziPodSetAffinity(KafkaResources.kafkaStatefulSetName(clusterName));
    NodeSelectorRequirement kafkaPodNodeSelectorRequirement = kafkaPodSpecAffinity.getNodeAffinity().getRequiredDuringSchedulingIgnoredDuringExecution().getNodeSelectorTerms().get(0).getMatchExpressions().get(0);
    assertThat(kafkaPodNodeSelectorRequirement.getKey(), is(rackKey));
    assertThat(kafkaPodNodeSelectorRequirement.getOperator(), is("Exists"));
    PodAffinityTerm kafkaPodAffinityTerm = kafkaPodSpecAffinity.getPodAntiAffinity().getPreferredDuringSchedulingIgnoredDuringExecution().get(0).getPodAffinityTerm();
    assertThat(kafkaPodAffinityTerm.getTopologyKey(), is(rackKey));
    assertThat(kafkaPodAffinityTerm.getLabelSelector().getMatchLabels(), hasEntry("strimzi.io/cluster", clusterName));
    assertThat(kafkaPodAffinityTerm.getLabelSelector().getMatchLabels(), hasEntry("strimzi.io/name", KafkaResources.kafkaStatefulSetName(clusterName)));
    String rackId = cmdKubeClient().execInPod(KafkaResources.kafkaPodName(clusterName, 0), "/bin/bash", "-c", "cat /opt/kafka/init/rack.id").out();
    assertThat(rackId.trim(), is("zone"));
    String brokerRack = cmdKubeClient().execInPod(KafkaResources.kafkaPodName(clusterName, 0), "/bin/bash", "-c", "cat /tmp/strimzi.properties | grep broker.rack").out();
    assertThat(brokerRack.contains("broker.rack=zone"), is(true));
    String uid = kubeClient().getPodUid(KafkaResources.kafkaPodName(clusterName, 0));
    List<Event> events = kubeClient().listEventsByResourceUid(uid);
    assertThat(events, hasAllOfReasons(Scheduled, Pulled, Created, Started));
    KafkaClients kafkaBasicClientJob = new KafkaClientsBuilder().withProducerName(producerName).withConsumerName(consumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(clusterName)).withTopicName(TOPIC_NAME).withMessageCount(MESSAGE_COUNT).withDelayMs(0).build();
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.producerStrimzi());
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.consumerStrimzi());
}

Example 15 with KafkaClients

use of io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients in project strimzi by strimzi.

the class AlternativeReconcileTriggersST method testAddingAndRemovingJbodVolumes.

/**
 * Adding and removing JBOD volumes requires rolling updates in the sequential order. Otherwise the StatefulSet does
 * not like it. This tests tries to add and remove volume from JBOD to test both of these situations.
 */
@ParallelNamespaceTest
void testAddingAndRemovingJbodVolumes(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    final String continuousTopicName = "continuous-topic";
    // 500 messages will take 500 seconds in that case
    final int continuousClientsMessageCount = 500;
    final String producerName = "hello-world-producer";
    final String consumerName = "hello-world-consumer";
    PersistentClaimStorage vol0 = new PersistentClaimStorageBuilder().withId(0).withSize("1Gi").withDeleteClaim(true).build();
    PersistentClaimStorage vol1 = new PersistentClaimStorageBuilder().withId(1).withSize("1Gi").withDeleteClaim(true).build();
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaJBOD(clusterName, 3, 3, new JbodStorageBuilder().addToVolumes(vol0).build()).build());
    final String kafkaName = KafkaResources.kafkaStatefulSetName(clusterName);
    final LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, kafkaName);
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName).build());
    // ##############################
    // Attach clients which will continuously produce/consume messages to/from Kafka brokers during rolling update
    // ##############################
    // Setup topic, which has 3 replicas and 2 min.isr to see if producer will be able to work during rolling update
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, continuousTopicName, 3, 3, 2).build());
    String producerAdditionConfiguration = "delivery.timeout.ms=20000\nrequest.timeout.ms=20000";
    // Add transactional id to make producer transactional
    producerAdditionConfiguration = producerAdditionConfiguration.concat("\ntransactional.id=" + continuousTopicName + ".1");
    producerAdditionConfiguration = producerAdditionConfiguration.concat("\nenable.idempotence=true");
    KafkaClients kafkaBasicClientJob = new KafkaClientsBuilder().withProducerName(producerName).withConsumerName(consumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(clusterName)).withTopicName(continuousTopicName).withMessageCount(continuousClientsMessageCount).withAdditionalConfig(producerAdditionConfiguration).withDelayMs(1000).withNamespaceName(namespaceName).build();
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.producerStrimzi());
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.consumerStrimzi());
    // ##############################
    String userName = KafkaUserUtils.generateRandomNameOfKafkaUser();
    KafkaUser user = KafkaUserTemplates.tlsUser(clusterName, userName).build();
    resourceManager.createResource(extensionContext, user);
    resourceManager.createResource(extensionContext, false, KafkaClientsTemplates.kafkaClients(true, kafkaClientsName, user).build());
    final String kafkaClientsPodName = PodUtils.getPodsByPrefixInNameWithDynamicWait(namespaceName, kafkaClientsName).get(0).getMetadata().getName();
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(kafkaClientsPodName).withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withKafkaUsername(userName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    internalKafkaClient.produceTlsMessagesUntilOperationIsSuccessful(MESSAGE_COUNT);
    // Add Jbod volume to Kafka => triggers RU
    LOGGER.info("Add JBOD volume to the Kafka cluster {}", kafkaName);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, kafka -> {
        JbodStorage storage = (JbodStorage) kafka.getSpec().getKafka().getStorage();
        storage.getVolumes().add(vol1);
    }, namespaceName);
    // Wait util it rolls
    kafkaPods = RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, 3, kafkaPods);
    // Remove Jbod volume to Kafka => triggers RU
    LOGGER.info("Remove JBOD volume to the Kafka cluster {}", kafkaName);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, kafka -> {
        JbodStorage storage = (JbodStorage) kafka.getSpec().getKafka().getStorage();
        storage.getVolumes().remove(vol1);
    }, namespaceName);
    // Wait util it rolls
    RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, 3, kafkaPods);
    // ##############################
    // Validate that continuous clients finished successfully
    // ##############################
    ClientUtils.waitTillContinuousClientsFinish(producerName, consumerName, namespaceName, continuousClientsMessageCount);
// ##############################
}