Examples with KafkaClientsBuilder io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder used on opensource projects

Example 16 with KafkaClientsBuilder

use of io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder in project strimzi by strimzi.

the class AlternativeReconcileTriggersST method testManualTriggeringRollingUpdate.

@ParallelNamespaceTest
@SuppressWarnings("checkstyle:MethodLength")
void testManualTriggeringRollingUpdate(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    final String continuousTopicName = "continuous-topic";
    // 500 messages will take 500 seconds in that case
    final int continuousClientsMessageCount = 500;
    final String producerName = "hello-world-producer";
    final String consumerName = "hello-world-consumer";
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(clusterName, 3, 3).build());
    String kafkaName = KafkaResources.kafkaStatefulSetName(clusterName);
    String zkName = KafkaResources.zookeeperStatefulSetName(clusterName);
    LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, kafkaName);
    LabelSelector zkSelector = KafkaResource.getLabelSelector(clusterName, zkName);
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
    Map<String, String> zkPods = PodUtils.podSnapshot(namespaceName, zkSelector);
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName, namespaceName).build());
    // ##############################
    // Attach clients which will continuously produce/consume messages to/from Kafka brokers during rolling update
    // ##############################
    // Setup topic, which has 3 replicas and 2 min.isr to see if producer will be able to work during rolling update
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, continuousTopicName, 3, 3, 2).build());
    String producerAdditionConfiguration = "delivery.timeout.ms=20000\nrequest.timeout.ms=20000";
    // Add transactional id to make producer transactional
    producerAdditionConfiguration = producerAdditionConfiguration.concat("\ntransactional.id=" + continuousTopicName + ".1");
    producerAdditionConfiguration = producerAdditionConfiguration.concat("\nenable.idempotence=true");
    KafkaClients kafkaBasicClientJob = new KafkaClientsBuilder().withProducerName(producerName).withConsumerName(consumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(clusterName)).withTopicName(continuousTopicName).withMessageCount(continuousClientsMessageCount).withAdditionalConfig(producerAdditionConfiguration).withDelayMs(1000).withNamespaceName(namespaceName).build();
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.producerStrimzi());
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.consumerStrimzi());
    // ##############################
    String userName = KafkaUserUtils.generateRandomNameOfKafkaUser();
    KafkaUser user = KafkaUserTemplates.tlsUser(clusterName, userName).build();
    resourceManager.createResource(extensionContext, user);
    resourceManager.createResource(extensionContext, false, KafkaClientsTemplates.kafkaClients(true, kafkaClientsName, user).build());
    final String kafkaClientsPodName = PodUtils.getPodsByPrefixInNameWithDynamicWait(namespaceName, kafkaClientsName).get(0).getMetadata().getName();
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(kafkaClientsPodName).withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withKafkaUsername(userName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    internalKafkaClient.produceTlsMessagesUntilOperationIsSuccessful(MESSAGE_COUNT);
    // rolling update for kafka
    // set annotation to trigger Kafka rolling update
    LOGGER.info("Annotate Kafka {} {} with manual rolling update annotation", Environment.isStrimziPodSetEnabled() ? StrimziPodSet.RESOURCE_KIND : Constants.STATEFUL_SET, kafkaName);
    StUtils.annotateStatefulSetOrStrimziPodSet(namespaceName, kafkaName, Collections.singletonMap(Annotations.ANNO_STRIMZI_IO_MANUAL_ROLLING_UPDATE, "true"));
    // check annotation to trigger rolling update
    assertThat(Boolean.parseBoolean(StUtils.getAnnotationsOfStatefulSetOrStrimziPodSet(namespaceName, kafkaName).get(Annotations.ANNO_STRIMZI_IO_MANUAL_ROLLING_UPDATE)), is(true));
    RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, 3, kafkaPods);
    // wait when annotation will be removed
    TestUtils.waitFor("CO removes rolling update annotation", Constants.WAIT_FOR_ROLLING_UPDATE_INTERVAL, Constants.GLOBAL_TIMEOUT, () -> StUtils.getAnnotationsOfStatefulSetOrStrimziPodSet(namespaceName, zkName) == null || !StUtils.getAnnotationsOfStatefulSetOrStrimziPodSet(namespaceName, zkName).containsKey(Annotations.ANNO_STRIMZI_IO_MANUAL_ROLLING_UPDATE));
    int received = internalKafkaClient.receiveMessagesTls();
    assertThat(received, is(MESSAGE_COUNT));
    // rolling update for zookeeper
    // set annotation to trigger Zookeeper rolling update
    LOGGER.info("Annotate Zookeeper {} {} with manual rolling update annotation", Environment.isStrimziPodSetEnabled() ? StrimziPodSet.RESOURCE_KIND : Constants.STATEFUL_SET, zkName);
    StUtils.annotateStatefulSetOrStrimziPodSet(namespaceName, zkName, Collections.singletonMap(Annotations.ANNO_STRIMZI_IO_MANUAL_ROLLING_UPDATE, "true"));
    // check annotation to trigger rolling update
    assertThat(Boolean.parseBoolean(StUtils.getAnnotationsOfStatefulSetOrStrimziPodSet(namespaceName, zkName).get(Annotations.ANNO_STRIMZI_IO_MANUAL_ROLLING_UPDATE)), is(true));
    RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, zkSelector, 3, zkPods);
    // wait when annotation will be removed
    TestUtils.waitFor("CO removes rolling update annotation", Constants.WAIT_FOR_ROLLING_UPDATE_INTERVAL, Constants.GLOBAL_TIMEOUT, () -> StUtils.getAnnotationsOfStatefulSetOrStrimziPodSet(namespaceName, zkName) == null || !StUtils.getAnnotationsOfStatefulSetOrStrimziPodSet(namespaceName, zkName).containsKey(Annotations.ANNO_STRIMZI_IO_MANUAL_ROLLING_UPDATE));
    internalKafkaClient = internalKafkaClient.toBuilder().withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
    received = internalKafkaClient.receiveMessagesTls();
    assertThat(received, is(MESSAGE_COUNT));
    // Create new topic to ensure, that ZK is working properly
    String newTopicName = KafkaTopicUtils.generateRandomNameOfTopic();
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, newTopicName, 1, 1).build());
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(newTopicName).withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
    int sent = internalKafkaClient.sendMessagesTls();
    assertThat(sent, is(MESSAGE_COUNT));
    received = internalKafkaClient.receiveMessagesTls();
    assertThat(received, is(sent));
    // ##############################
    // Validate that continuous clients finished successfully
    // ##############################
    ClientUtils.waitTillContinuousClientsFinish(producerName, consumerName, namespaceName, continuousClientsMessageCount);
// ##############################
}

Example 17 with KafkaClientsBuilder

use of io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder in project strimzi by strimzi.

the class MirrorMaker2IsolatedST method testMirrorMaker2CorrectlyMirrorsHeaders.

@ParallelNamespaceTest
void testMirrorMaker2CorrectlyMirrorsHeaders(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(INFRA_NAMESPACE, extensionContext);
    String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    String kafkaClusterSourceName = clusterName + "-source";
    String kafkaClusterTargetName = clusterName + "-target";
    String sourceProducerName = clusterName + "-source-producer";
    String targetConsumerName = clusterName + "-target-consumer";
    String sourceExampleTopic = clusterName + "-source-example-topic";
    String targetExampleTopic = kafkaClusterSourceName + "." + sourceExampleTopic;
    // Deploy source kafka
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterSourceName, 1, 1).build());
    // Deploy target kafka
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterTargetName, 1, 1).build());
    // Deploy Topic for example clients
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, sourceExampleTopic).build());
    resourceManager.createResource(extensionContext, KafkaMirrorMaker2Templates.kafkaMirrorMaker2(clusterName, kafkaClusterTargetName, kafkaClusterSourceName, 1, false).build());
    // deploying example clients for checking if mm2 will mirror messages with headers
    KafkaClients targetKafkaClientsJob = new KafkaClientsBuilder().withConsumerName(targetConsumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(kafkaClusterTargetName)).withTopicName(targetExampleTopic).withMessageCount(MESSAGE_COUNT).withDelayMs(1000).build();
    resourceManager.createResource(extensionContext, targetKafkaClientsJob.consumerStrimzi());
    KafkaClients sourceKafkaClientsJob = new KafkaClientsBuilder().withProducerName(sourceProducerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(kafkaClusterSourceName)).withTopicName(sourceExampleTopic).withMessageCount(MESSAGE_COUNT).withDelayMs(1000).build();
    resourceManager.createResource(extensionContext, new JobBuilder(sourceKafkaClientsJob.producerStrimzi()).editSpec().editTemplate().editSpec().editContainer(0).addNewEnv().withName("HEADERS").withValue("header_key_one=header_value_one, header_key_two=header_value_two").endEnv().endContainer().endSpec().endTemplate().endSpec().build());
    ClientUtils.waitTillContinuousClientsFinish(sourceProducerName, targetConsumerName, namespaceName, MESSAGE_COUNT);
    LOGGER.info("Checking log of {} job if the headers are correct", targetConsumerName);
    String header1 = "key: header_key_one, value: header_value_one";
    String header2 = "key: header_key_two, value: header_value_two";
    String log = StUtils.getLogFromPodByTime(namespaceName, kubeClient(namespaceName).listPodsByPrefixInName(targetConsumerName).get(0).getMetadata().getName(), "", MESSAGE_COUNT + "s");
    assertThat(log, containsString(header1));
    assertThat(log, containsString(header2));
}

Example 18 with KafkaClientsBuilder

use of io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder in project strimzi by strimzi.

the class FeatureGatesIsolatedST method testControlPlaneListenerFeatureGate.

/**
 * Control Plane Listener
 * https://github.com/strimzi/proposals/blob/main/025-control-plain-listener.md
 */
@IsolatedTest("Feature Gates test for disabled ControlPlainListener")
@Tag(INTERNAL_CLIENTS_USED)
public void testControlPlaneListenerFeatureGate(ExtensionContext extensionContext) {
    assumeFalse(Environment.isOlmInstall() || Environment.isHelmInstall());
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String producerName = "producer-test-" + new Random().nextInt(Integer.MAX_VALUE);
    final String consumerName = "consumer-test-" + new Random().nextInt(Integer.MAX_VALUE);
    final String topicName = KafkaTopicUtils.generateRandomNameOfTopic();
    final LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, KafkaResources.zookeeperStatefulSetName(clusterName));
    int messageCount = 300;
    List<EnvVar> testEnvVars = new ArrayList<>();
    int kafkaReplicas = 3;
    testEnvVars.add(new EnvVar(Environment.STRIMZI_FEATURE_GATES_ENV, "-ControlPlaneListener", null));
    clusterOperator.unInstall();
    clusterOperator = new SetupClusterOperator.SetupClusterOperatorBuilder().withExtensionContext(BeforeAllOnce.getSharedExtensionContext()).withNamespace(INFRA_NAMESPACE).withWatchingNamespaces(Constants.WATCH_ALL_NAMESPACES).withExtraEnvVars(testEnvVars).createInstallation().runInstallation();
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(clusterName, kafkaReplicas).build());
    LOGGER.info("Check for presence of ContainerPort 9090/tcp (tcp-ctrlplane) in first Kafka pod.");
    final Pod kafkaPod = PodUtils.getPodsByPrefixInNameWithDynamicWait(INFRA_NAMESPACE, clusterName + "-kafka-").get(0);
    ContainerPort expectedControlPlaneContainerPort = new ContainerPort(9090, null, null, "tcp-ctrlplane", "TCP");
    List<ContainerPort> kafkaPodPorts = kafkaPod.getSpec().getContainers().get(0).getPorts();
    assertTrue(kafkaPodPorts.contains(expectedControlPlaneContainerPort));
    Map<String, String> kafkaPods = PodUtils.podSnapshot(INFRA_NAMESPACE, kafkaSelector);
    LOGGER.info("Try to send some messages to Kafka over next few minutes.");
    KafkaTopic kafkaTopic = KafkaTopicTemplates.topic(clusterName, topicName).editSpec().withReplicas(kafkaReplicas).withPartitions(kafkaReplicas).endSpec().build();
    resourceManager.createResource(extensionContext, kafkaTopic);
    KafkaClients kafkaBasicClientJob = new KafkaClientsBuilder().withProducerName(producerName).withConsumerName(consumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(clusterName)).withTopicName(topicName).withMessageCount(messageCount).withDelayMs(500).withNamespaceName(INFRA_NAMESPACE).build();
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.producerStrimzi());
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.consumerStrimzi());
    JobUtils.waitForJobRunning(consumerName, INFRA_NAMESPACE);
    LOGGER.info("Delete first found Kafka broker pod.");
    kubeClient(INFRA_NAMESPACE).deletePod(INFRA_NAMESPACE, kafkaPod);
    RollingUpdateUtils.waitForComponentAndPodsReady(kafkaSelector, kafkaReplicas);
    LOGGER.info("Force Rolling Update of Kafka via annotation.");
    kafkaPods.keySet().forEach(podName -> {
        kubeClient(INFRA_NAMESPACE).editPod(podName).edit(pod -> new PodBuilder(pod).editMetadata().addToAnnotations(Annotations.ANNO_STRIMZI_IO_MANUAL_ROLLING_UPDATE, "true").endMetadata().build());
    });
    LOGGER.info("Wait for next reconciliation to happen.");
    RollingUpdateUtils.waitTillComponentHasRolled(INFRA_NAMESPACE, kafkaSelector, kafkaReplicas, kafkaPods);
    LOGGER.info("Waiting for clients to finish sending/receiving messages.");
    ClientUtils.waitForClientSuccess(producerName, INFRA_NAMESPACE, MESSAGE_COUNT);
    ClientUtils.waitForClientSuccess(consumerName, INFRA_NAMESPACE, MESSAGE_COUNT);
}

Example 19 with KafkaClientsBuilder

use of io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder in project strimzi by strimzi.

the class HttpBridgeKafkaExternalListenersST method testWeirdUsername.

@SuppressWarnings({ "checkstyle:MethodLength" })
private void testWeirdUsername(ExtensionContext extensionContext, String weirdUserName, KafkaListenerAuthentication auth, KafkaBridgeSpec spec, SecurityProtocol securityProtocol) {
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(clusterName, 3, 1).editMetadata().withNamespace(namespace).endMetadata().editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(auth).build(), new GenericKafkaListenerBuilder().withName(Constants.EXTERNAL_LISTENER_DEFAULT_NAME).withPort(9094).withType(KafkaListenerType.NODEPORT).withTls(true).withAuth(auth).build()).endKafka().endSpec().build());
    BridgeClients kafkaBridgeClientJob = new BridgeClientsBuilder().withProducerName(clusterName + "-" + producerName).withConsumerName(clusterName + "-" + consumerName).withBootstrapAddress(KafkaBridgeResources.serviceName(clusterName)).withTopicName(topicName).withMessageCount(MESSAGE_COUNT).withPort(Constants.HTTP_BRIDGE_DEFAULT_PORT).withNamespaceName(namespace).build();
    // Create topic
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName).editMetadata().withNamespace(namespace).endMetadata().build());
    // Create user
    if (auth.getType().equals(Constants.TLS_LISTENER_DEFAULT_NAME)) {
        resourceManager.createResource(extensionContext, KafkaUserTemplates.tlsUser(clusterName, weirdUserName).editMetadata().withNamespace(namespace).endMetadata().build());
    } else {
        resourceManager.createResource(extensionContext, KafkaUserTemplates.scramShaUser(clusterName, weirdUserName).editMetadata().withNamespace(namespace).endMetadata().build());
    }
    final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(namespace, true, kafkaClientsName).build());
    // Deploy http bridge
    resourceManager.createResource(extensionContext, KafkaBridgeTemplates.kafkaBridge(clusterName, KafkaResources.tlsBootstrapAddress(clusterName), 1).editMetadata().withNamespace(namespace).endMetadata().withNewSpecLike(spec).withBootstrapServers(KafkaResources.tlsBootstrapAddress(clusterName)).withNewHttp(Constants.HTTP_BRIDGE_DEFAULT_PORT).withNewConsumer().addToConfig(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest").endConsumer().endSpec().build());
    final Service service = KafkaBridgeUtils.createBridgeNodePortService(clusterName, namespace, BRIDGE_EXTERNAL_SERVICE);
    ServiceResource.createServiceResource(extensionContext, service, namespace);
    resourceManager.createResource(extensionContext, kafkaBridgeClientJob.consumerStrimziBridge());
    final String kafkaProducerExternalName = "kafka-producer-external" + new Random().nextInt(Integer.MAX_VALUE);
    final List<ListenerStatus> listenerStatusList = KafkaResource.kafkaClient().inNamespace(namespace).withName(clusterName).get().getStatus().getListeners();
    final String externalBootstrapServers = listenerStatusList.stream().filter(listener -> listener.getType().equals(Constants.EXTERNAL_LISTENER_DEFAULT_NAME)).findFirst().orElseThrow(RuntimeException::new).getBootstrapServers();
    final KafkaClients externalKafkaProducer = new KafkaClientsBuilder().withProducerName(kafkaProducerExternalName).withBootstrapAddress(externalBootstrapServers).withNamespaceName(namespace).withTopicName(topicName).withMessageCount(100).build();
    if (auth.getType().equals(Constants.TLS_LISTENER_DEFAULT_NAME)) {
        // tls producer
        resourceManager.createResource(extensionContext, externalKafkaProducer.producerTlsStrimzi(clusterName, weirdUserName));
    } else {
        // scram-sha producer
        resourceManager.createResource(extensionContext, externalKafkaProducer.producerScramShaStrimzi(clusterName, weirdUserName));
    }
    ClientUtils.waitForClientSuccess(kafkaProducerExternalName, namespace, MESSAGE_COUNT);
    // delete kafka producer job
    JobUtils.deleteJobWithWait(namespace, kafkaProducerExternalName);
    ClientUtils.waitForClientSuccess(clusterName + "-" + consumerName, namespace, MESSAGE_COUNT);
}

Example 20 with KafkaClientsBuilder

use of io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder in project strimzi by strimzi.

the class ConnectBuilderIsolatedST method testBuildPluginUsingMavenCoordinatesArtifacts.

@ParallelTest
void testBuildPluginUsingMavenCoordinatesArtifacts(ExtensionContext extensionContext) {
    final String connectClusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String imageName = getImageNameForTestCase();
    final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    final String connectorName = connectClusterName + "-camel-connector";
    final String consumerName = mapWithKafkaClientNames.get(extensionContext.getDisplayName()) + "-consumer";
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(INFRA_NAMESPACE, topicName).build(), KafkaConnectTemplates.kafkaConnect(extensionContext, connectClusterName, INFRA_NAMESPACE, INFRA_NAMESPACE, 1, false).editMetadata().addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().editOrNewSpec().addToConfig("key.converter.schemas.enable", false).addToConfig("value.converter.schemas.enable", false).addToConfig("key.converter", "org.apache.kafka.connect.storage.StringConverter").addToConfig("value.converter", "org.apache.kafka.connect.storage.StringConverter").withNewBuild().withPlugins(PLUGIN_WITH_MAVEN_TYPE).withNewDockerOutput().withImage(imageName).endDockerOutput().endBuild().endSpec().build());
    Map<String, Object> connectorConfig = new HashMap<>();
    connectorConfig.put("topics", topicName);
    connectorConfig.put("camel.source.path.timerName", "timer");
    resourceManager.createResource(extensionContext, KafkaConnectorTemplates.kafkaConnector(connectorName, connectClusterName).editOrNewSpec().withClassName(CAMEL_CONNECTOR_TIMER_CLASS_NAME).withConfig(connectorConfig).endSpec().build());
    KafkaClients kafkaClient = new KafkaClientsBuilder().withConsumerName(consumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(INFRA_NAMESPACE)).withTopicName(topicName).withMessageCount(MESSAGE_COUNT).withDelayMs(0).build();
    resourceManager.createResource(extensionContext, kafkaClient.consumerStrimzi());
    ClientUtils.waitForClientSuccess(consumerName, INFRA_NAMESPACE, MESSAGE_COUNT);
}