Examples with ExternalKafkaClient io.strimzi.systemtest.kafkaclients.externalClients.ExternalKafkaClient used on opensource projects

Example 6 with ExternalKafkaClient

use of io.strimzi.systemtest.kafkaclients.externalClients.ExternalKafkaClient in project strimzi by strimzi.

the class ListenersST method testCustomSoloCertificatesForLoadBalancer.

@ParallelNamespaceTest
@Tag(LOADBALANCER_SUPPORTED)
@Tag(EXTERNAL_CLIENTS_USED)
@Tag(INTERNAL_CLIENTS_USED)
void testCustomSoloCertificatesForLoadBalancer(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(INFRA_NAMESPACE, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    final String userName = mapWithTestUsers.get(extensionContext.getDisplayName());
    final String clusterCustomCertServer1 = clusterName + "-" + customCertServer1;
    SecretUtils.createCustomSecret(clusterCustomCertServer1, clusterName, namespaceName, STRIMZI_CERT_AND_KEY_1);
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(clusterName, 3).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9107).withType(KafkaListenerType.INTERNAL).withTls(true).withNewConfiguration().withNewBrokerCertChainAndKey().withSecretName(clusterCustomCertServer1).withKey("ca.key").withCertificate("ca.crt").endBrokerCertChainAndKey().endConfiguration().build(), new GenericKafkaListenerBuilder().withName(Constants.EXTERNAL_LISTENER_DEFAULT_NAME).withPort(9108).withType(KafkaListenerType.LOADBALANCER).withTls(true).withNewConfiguration().withNewBrokerCertChainAndKey().withSecretName(clusterCustomCertServer1).withKey("ca.key").withCertificate("ca.crt").endBrokerCertChainAndKey().withFinalizers(LB_FINALIZERS).endConfiguration().build()).endKafka().endSpec().build());
    KafkaUser aliceUser = KafkaUserTemplates.tlsUser(clusterName, userName).build();
    resourceManager.createResource(extensionContext, aliceUser);
    ExternalKafkaClient externalKafkaClient = new ExternalKafkaClient.Builder().withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withKafkaUsername(userName).withMessageCount(MESSAGE_COUNT).withCertificateAuthorityCertificateName(clusterCustomCertServer1).withSecurityProtocol(SecurityProtocol.SSL).withListenerName(Constants.EXTERNAL_LISTENER_DEFAULT_NAME).build();
    externalKafkaClient.verifyProducedAndConsumedMessages(externalKafkaClient.sendMessagesTls(), externalKafkaClient.receiveMessagesTls());
    // Deploy client pod with custom certificates and collect messages from internal TLS listener
    resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(true, clusterName + "-" + Constants.KAFKA_CLIENTS, false, aliceUser).build());
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(kubeClient(namespaceName).listPodsByPrefixInName(namespaceName, clusterName + "-" + Constants.KAFKA_CLIENTS).get(0).getMetadata().getName()).withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withKafkaUsername(userName).withMessageCount(MESSAGE_COUNT).withConsumerGroupName("consumer-group-certs-3").withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    int sent = internalKafkaClient.sendMessagesTls();
    assertThat(sent, is(MESSAGE_COUNT));
    internalKafkaClient.setMessageCount(MESSAGE_COUNT * 2);
    int received = internalKafkaClient.receiveMessagesTls();
    assertThat(received, is(2 * MESSAGE_COUNT));
}

Example 7 with ExternalKafkaClient

use of io.strimzi.systemtest.kafkaclients.externalClients.ExternalKafkaClient in project strimzi by strimzi.

the class SecurityST method testAclRuleReadAndWrite.

@ParallelNamespaceTest
@Tag(NODEPORT_SUPPORTED)
@Tag(EXTERNAL_CLIENTS_USED)
void testAclRuleReadAndWrite(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    final String kafkaUserWrite = "kafka-user-write";
    final String kafkaUserRead = "kafka-user-read";
    final int numberOfMessages = 500;
    final String consumerGroupName = "consumer-group-name-1";
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(clusterName, 3, 1).editSpec().editKafka().withNewKafkaAuthorizationSimple().endKafkaAuthorizationSimple().withListeners(new GenericKafkaListenerBuilder().withName(Constants.EXTERNAL_LISTENER_DEFAULT_NAME).withPort(9094).withType(KafkaListenerType.NODEPORT).withTls(true).withAuth(new KafkaListenerAuthenticationTls()).build()).endKafka().endSpec().build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName).build());
    resourceManager.createResource(extensionContext, KafkaUserTemplates.tlsUser(clusterName, kafkaUserWrite).editSpec().withNewKafkaUserAuthorizationSimple().addNewAcl().withNewAclRuleTopicResource().withName(topicName).endAclRuleTopicResource().withOperation(AclOperation.WRITE).endAcl().addNewAcl().withNewAclRuleTopicResource().withName(topicName).endAclRuleTopicResource().withOperation(// describe is for that user can find out metadata
    AclOperation.DESCRIBE).endAcl().endKafkaUserAuthorizationSimple().endSpec().build());
    LOGGER.info("Checking KafkaUser {} that is able to send messages to topic '{}'", kafkaUserWrite, topicName);
    ExternalKafkaClient externalKafkaClient = new ExternalKafkaClient.Builder().withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withKafkaUsername(kafkaUserWrite).withMessageCount(numberOfMessages).withSecurityProtocol(SecurityProtocol.SSL).withListenerName(Constants.EXTERNAL_LISTENER_DEFAULT_NAME).build();
    assertThat(externalKafkaClient.sendMessagesTls(), is(numberOfMessages));
    assertThrows(GroupAuthorizationException.class, externalKafkaClient::receiveMessagesTls);
    resourceManager.createResource(extensionContext, KafkaUserTemplates.tlsUser(clusterName, kafkaUserRead).editSpec().withNewKafkaUserAuthorizationSimple().addNewAcl().withNewAclRuleTopicResource().withName(topicName).endAclRuleTopicResource().withOperation(AclOperation.READ).endAcl().addNewAcl().withNewAclRuleGroupResource().withName(consumerGroupName).endAclRuleGroupResource().withOperation(AclOperation.READ).endAcl().addNewAcl().withNewAclRuleTopicResource().withName(topicName).endAclRuleTopicResource().withOperation(// s describe is for that user can find out metadata
    AclOperation.DESCRIBE).endAcl().endKafkaUserAuthorizationSimple().endSpec().build());
    ExternalKafkaClient newExternalKafkaClient = externalKafkaClient.toBuilder().withKafkaUsername(kafkaUserRead).withConsumerGroupName(consumerGroupName).build();
    assertThat(newExternalKafkaClient.receiveMessagesTls(), is(numberOfMessages));
    LOGGER.info("Checking KafkaUser {} that is not able to send messages to topic '{}'", kafkaUserRead, topicName);
    assertThrows(Exception.class, newExternalKafkaClient::sendMessagesTls);
}

Example 8 with ExternalKafkaClient

use of io.strimzi.systemtest.kafkaclients.externalClients.ExternalKafkaClient in project strimzi-kafka-operator by strimzi.

the class CustomResourceStatusIsolatedST method testKafkaStatus.

@ParallelTest
@Tag(NODEPORT_SUPPORTED)
@Tag(EXTERNAL_CLIENTS_USED)
void testKafkaStatus(ExtensionContext extensionContext) {
    LOGGER.info("Checking status of deployed kafka cluster");
    KafkaUtils.waitForKafkaReady(Constants.INFRA_NAMESPACE, CUSTOM_RESOURCE_STATUS_CLUSTER_NAME);
    ExternalKafkaClient externalKafkaClient = new ExternalKafkaClient.Builder().withTopicName(TOPIC_NAME).withNamespaceName(Constants.INFRA_NAMESPACE).withClusterName(CUSTOM_RESOURCE_STATUS_CLUSTER_NAME).withMessageCount(MESSAGE_COUNT).withListenerName(Constants.EXTERNAL_LISTENER_DEFAULT_NAME).build();
    externalKafkaClient.verifyProducedAndConsumedMessages(externalKafkaClient.sendMessagesPlain(), externalKafkaClient.receiveMessagesPlain());
    assertKafkaStatus(1, KafkaResources.bootstrapServiceName(CUSTOM_RESOURCE_STATUS_CLUSTER_NAME) + "." + Constants.INFRA_NAMESPACE + ".svc");
    KafkaResource.replaceKafkaResource(CUSTOM_RESOURCE_STATUS_CLUSTER_NAME, k -> {
        k.getSpec().getEntityOperator().getTopicOperator().setResources(new ResourceRequirementsBuilder().addToRequests("cpu", new Quantity("100000m")).build());
    });
    LOGGER.info("Wait until cluster will be in NotReady state ...");
    KafkaUtils.waitForKafkaNotReady(CUSTOM_RESOURCE_STATUS_CLUSTER_NAME);
    LOGGER.info("Recovery cluster to Ready state ...");
    KafkaResource.replaceKafkaResource(CUSTOM_RESOURCE_STATUS_CLUSTER_NAME, k -> {
        k.getSpec().getEntityOperator().getTopicOperator().setResources(new ResourceRequirementsBuilder().addToRequests("cpu", new Quantity("100m")).build());
    });
    KafkaUtils.waitForKafkaReady(Constants.INFRA_NAMESPACE, CUSTOM_RESOURCE_STATUS_CLUSTER_NAME);
    assertKafkaStatus(3, KafkaResources.bootstrapServiceName(CUSTOM_RESOURCE_STATUS_CLUSTER_NAME) + "." + Constants.INFRA_NAMESPACE + ".svc");
}

Example 9 with ExternalKafkaClient

use of io.strimzi.systemtest.kafkaclients.externalClients.ExternalKafkaClient in project strimzi-kafka-operator by strimzi.

the class SpecificIsolatedST method testLoadBalancerIpOverride.

@IsolatedTest("Using more tha one Kafka cluster in one namespace")
@Tag(LOADBALANCER_SUPPORTED)
@Tag(EXTERNAL_CLIENTS_USED)
void testLoadBalancerIpOverride(ExtensionContext extensionContext) {
    String bootstrapOverrideIP = "10.0.0.1";
    String brokerOverrideIP = "10.0.0.2";
    String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(clusterName, 3, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.EXTERNAL_LISTENER_DEFAULT_NAME).withPort(9094).withType(KafkaListenerType.LOADBALANCER).withTls(true).withNewConfiguration().withNewBootstrap().withLoadBalancerIP(brokerOverrideIP).endBootstrap().withBrokers(new GenericKafkaListenerConfigurationBrokerBuilder().withBroker(0).withLoadBalancerIP(brokerOverrideIP).build()).withFinalizers(LB_FINALIZERS).endConfiguration().build()).endKafka().endSpec().build());
    assertThat("Kafka External bootstrap doesn't contain correct loadBalancer address", kubeClient().getService(KafkaResources.externalBootstrapServiceName(clusterName)).getSpec().getLoadBalancerIP(), is(bootstrapOverrideIP));
    assertThat("Kafka Broker-0 service doesn't contain correct loadBalancer address", kubeClient().getService(KafkaResources.brokerSpecificService(clusterName, 0)).getSpec().getLoadBalancerIP(), is(brokerOverrideIP));
    ExternalKafkaClient externalKafkaClient = new ExternalKafkaClient.Builder().withTopicName(TOPIC_NAME).withNamespaceName(INFRA_NAMESPACE).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withListenerName(Constants.EXTERNAL_LISTENER_DEFAULT_NAME).build();
    externalKafkaClient.verifyProducedAndConsumedMessages(externalKafkaClient.sendMessagesPlain(), externalKafkaClient.receiveMessagesPlain());
}

Example 10 with ExternalKafkaClient

use of io.strimzi.systemtest.kafkaclients.externalClients.ExternalKafkaClient in project strimzi-kafka-operator by strimzi.

the class DynamicConfST method testUpdateToExternalListenerCausesRollingRestartUsingExternalClients.

@IsolatedTest("Using more tha one Kafka cluster in one namespace")
@Tag(NODEPORT_SUPPORTED)
@Tag(EXTERNAL_CLIENTS_USED)
@Tag(ROLLING_UPDATE)
void testUpdateToExternalListenerCausesRollingRestartUsingExternalClients(ExtensionContext extensionContext) {
    String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    String userName = mapWithTestUsers.get(extensionContext.getDisplayName());
    Map<String, Object> deepCopyOfShardKafkaConfig = kafkaConfig.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue()));
    LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, KafkaResources.kafkaStatefulSetName(clusterName));
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(clusterName, KAFKA_REPLICAS, 1).editMetadata().withNamespace(namespace).endMetadata().editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.EXTERNAL_LISTENER_DEFAULT_NAME).withPort(9094).withType(KafkaListenerType.NODEPORT).withTls(false).build()).withConfig(deepCopyOfShardKafkaConfig).endKafka().endSpec().build());
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespace, kafkaSelector);
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName, namespace).build());
    resourceManager.createResource(extensionContext, KafkaUserTemplates.tlsUser(namespace, clusterName, userName).build());
    ExternalKafkaClient externalKafkaClientTls = new ExternalKafkaClient.Builder().withTopicName(topicName).withNamespaceName(namespace).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withKafkaUsername(userName).withSecurityProtocol(SecurityProtocol.SSL).withListenerName(Constants.EXTERNAL_LISTENER_DEFAULT_NAME).build();
    ExternalKafkaClient externalKafkaClientPlain = new ExternalKafkaClient.Builder().withTopicName(topicName).withNamespaceName(namespace).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withSecurityProtocol(SecurityProtocol.PLAINTEXT).withListenerName(Constants.EXTERNAL_LISTENER_DEFAULT_NAME).build();
    externalKafkaClientPlain.verifyProducedAndConsumedMessages(externalKafkaClientPlain.sendMessagesPlain(), externalKafkaClientPlain.receiveMessagesPlain());
    assertThrows(Exception.class, () -> {
        externalKafkaClientTls.sendMessagesTls();
        externalKafkaClientTls.receiveMessagesTls();
        LOGGER.error("Producer & Consumer did not send and receive messages because external listener is set to plain communication");
    });
    LOGGER.info("Updating listeners of Kafka cluster");
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, k -> {
        k.getSpec().getKafka().setListeners(Arrays.asList(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).build(), new GenericKafkaListenerBuilder().withName(Constants.EXTERNAL_LISTENER_DEFAULT_NAME).withPort(9094).withType(KafkaListenerType.NODEPORT).withTls(true).withNewKafkaListenerAuthenticationTlsAuth().endKafkaListenerAuthenticationTlsAuth().build()));
    }, namespace);
    // TODO: remove it ?
    kafkaPods = RollingUpdateUtils.waitTillComponentHasRolled(namespace, kafkaSelector, KAFKA_REPLICAS, kafkaPods);
    externalKafkaClientTls.verifyProducedAndConsumedMessages(externalKafkaClientTls.sendMessagesTls() + MESSAGE_COUNT, externalKafkaClientTls.receiveMessagesTls());
    assertThrows(Exception.class, () -> {
        externalKafkaClientPlain.sendMessagesPlain();
        externalKafkaClientPlain.receiveMessagesPlain();
        LOGGER.error("Producer & Consumer did not send and receive messages because external listener is set to tls communication");
    });
    LOGGER.info("Updating listeners of Kafka cluster");
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, k -> {
        k.getSpec().getKafka().setListeners(Collections.singletonList(new GenericKafkaListenerBuilder().withName(Constants.EXTERNAL_LISTENER_DEFAULT_NAME).withPort(9094).withType(KafkaListenerType.NODEPORT).withTls(false).build()));
    }, namespace);
    RollingUpdateUtils.waitTillComponentHasRolled(namespace, kafkaSelector, KAFKA_REPLICAS, kafkaPods);
    assertThrows(Exception.class, () -> {
        externalKafkaClientTls.sendMessagesTls();
        externalKafkaClientTls.receiveMessagesTls();
        LOGGER.error("Producer & Consumer did not send and receive messages because external listener is set to plain communication");
    });
    externalKafkaClientPlain.verifyProducedAndConsumedMessages(externalKafkaClientPlain.sendMessagesPlain() + MESSAGE_COUNT, externalKafkaClientPlain.receiveMessagesPlain());
}