Examples with InternalKafkaClient io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient used on opensource projects

Example 21 with InternalKafkaClient

use of io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient in project strimzi by strimzi.

the class ListenersST method testCustomSoloCertificatesForLoadBalancer.

@ParallelNamespaceTest
@Tag(LOADBALANCER_SUPPORTED)
@Tag(EXTERNAL_CLIENTS_USED)
@Tag(INTERNAL_CLIENTS_USED)
void testCustomSoloCertificatesForLoadBalancer(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(INFRA_NAMESPACE, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    final String userName = mapWithTestUsers.get(extensionContext.getDisplayName());
    final String clusterCustomCertServer1 = clusterName + "-" + customCertServer1;
    SecretUtils.createCustomSecret(clusterCustomCertServer1, clusterName, namespaceName, STRIMZI_CERT_AND_KEY_1);
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(clusterName, 3).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9107).withType(KafkaListenerType.INTERNAL).withTls(true).withNewConfiguration().withNewBrokerCertChainAndKey().withSecretName(clusterCustomCertServer1).withKey("ca.key").withCertificate("ca.crt").endBrokerCertChainAndKey().endConfiguration().build(), new GenericKafkaListenerBuilder().withName(Constants.EXTERNAL_LISTENER_DEFAULT_NAME).withPort(9108).withType(KafkaListenerType.LOADBALANCER).withTls(true).withNewConfiguration().withNewBrokerCertChainAndKey().withSecretName(clusterCustomCertServer1).withKey("ca.key").withCertificate("ca.crt").endBrokerCertChainAndKey().withFinalizers(LB_FINALIZERS).endConfiguration().build()).endKafka().endSpec().build());
    KafkaUser aliceUser = KafkaUserTemplates.tlsUser(clusterName, userName).build();
    resourceManager.createResource(extensionContext, aliceUser);
    ExternalKafkaClient externalKafkaClient = new ExternalKafkaClient.Builder().withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withKafkaUsername(userName).withMessageCount(MESSAGE_COUNT).withCertificateAuthorityCertificateName(clusterCustomCertServer1).withSecurityProtocol(SecurityProtocol.SSL).withListenerName(Constants.EXTERNAL_LISTENER_DEFAULT_NAME).build();
    externalKafkaClient.verifyProducedAndConsumedMessages(externalKafkaClient.sendMessagesTls(), externalKafkaClient.receiveMessagesTls());
    // Deploy client pod with custom certificates and collect messages from internal TLS listener
    resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(true, clusterName + "-" + Constants.KAFKA_CLIENTS, false, aliceUser).build());
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(kubeClient(namespaceName).listPodsByPrefixInName(namespaceName, clusterName + "-" + Constants.KAFKA_CLIENTS).get(0).getMetadata().getName()).withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withKafkaUsername(userName).withMessageCount(MESSAGE_COUNT).withConsumerGroupName("consumer-group-certs-3").withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    int sent = internalKafkaClient.sendMessagesTls();
    assertThat(sent, is(MESSAGE_COUNT));
    internalKafkaClient.setMessageCount(MESSAGE_COUNT * 2);
    int received = internalKafkaClient.receiveMessagesTls();
    assertThat(received, is(2 * MESSAGE_COUNT));
}

Example 22 with InternalKafkaClient

use of io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient in project strimzi by strimzi.

the class SecurityST method testAutoRenewCaCertsTriggerByExpiredCertificate.

@ParallelNamespaceTest
@Tag(INTERNAL_CLIENTS_USED)
void testAutoRenewCaCertsTriggerByExpiredCertificate(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    String userName = mapWithTestUsers.get(extensionContext.getDisplayName());
    // 1. Create the Secrets already, and a certificate that's already expired
    InputStream secretInputStream = getClass().getClassLoader().getResourceAsStream("security-st-certs/expired-cluster-ca.crt");
    String clusterCaCert = TestUtils.readResource(secretInputStream);
    SecretUtils.createSecret(namespaceName, clusterCaCertificateSecretName(clusterName), "ca.crt", clusterCaCert);
    // 2. Now create a cluster
    createKafkaCluster(extensionContext);
    KafkaUser user = KafkaUserTemplates.tlsUser(clusterName, userName).build();
    resourceManager.createResource(extensionContext, user);
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName).build());
    resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(true, clusterName + "-" + Constants.KAFKA_CLIENTS, user).build());
    String defaultKafkaClientsPodName = kubeClient(namespaceName).listPodsByPrefixInName(namespaceName, clusterName + "-" + Constants.KAFKA_CLIENTS).get(0).getMetadata().getName();
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(defaultKafkaClientsPodName).withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withListenerName(Constants.PLAIN_LISTENER_DEFAULT_NAME).build();
    LOGGER.info("Checking produced and consumed messages to pod:{}", defaultKafkaClientsPodName);
    internalKafkaClient.checkProducedAndConsumedMessages(internalKafkaClient.sendMessagesPlain(), internalKafkaClient.receiveMessagesPlain());
    // Wait until the certificates have been replaced
    SecretUtils.waitForCertToChange(namespaceName, clusterCaCert, clusterCaCertificateSecretName(clusterName));
    // Wait until the pods are all up and ready
    KafkaUtils.waitForClusterStability(namespaceName, clusterName);
    LOGGER.info("Checking produced and consumed messages to pod:{}", defaultKafkaClientsPodName);
    internalKafkaClient.checkProducedAndConsumedMessages(internalKafkaClient.sendMessagesPlain(), internalKafkaClient.receiveMessagesPlain());
}

Example 23 with InternalKafkaClient

use of io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient in project strimzi by strimzi.

the class SecurityST method testCertRegeneratedAfterInternalCAisDeleted.

@ParallelNamespaceTest
@Tag(INTERNAL_CLIENTS_USED)
void testCertRegeneratedAfterInternalCAisDeleted(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    final String userName = mapWithTestUsers.get(extensionContext.getDisplayName());
    final LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, KafkaResources.kafkaStatefulSetName(clusterName));
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(clusterName, 3, 1).build());
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
    KafkaUser user = KafkaUserTemplates.tlsUser(namespaceName, clusterName, userName).build();
    resourceManager.createResource(extensionContext, user);
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName).build());
    resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(true, clusterName + "-" + Constants.KAFKA_CLIENTS, user).build());
    String defaultKafkaClientsPodName = kubeClient(namespaceName).listPodsByPrefixInName(namespaceName, clusterName + "-" + Constants.KAFKA_CLIENTS).get(0).getMetadata().getName();
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(defaultKafkaClientsPodName).withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withKafkaUsername(userName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    // TODO
    List<Secret> secrets = kubeClient(namespaceName).listSecrets(namespaceName).stream().filter(secret -> secret.getMetadata().getName().startsWith(clusterName) && secret.getMetadata().getName().endsWith("ca-cert")).collect(Collectors.toList());
    for (Secret s : secrets) {
        LOGGER.info("Verifying that secret {} with name {} is present", s, s.getMetadata().getName());
        assertThat(s.getData(), is(notNullValue()));
    }
    for (Secret s : secrets) {
        LOGGER.info("Deleting secret {}", s.getMetadata().getName());
        kubeClient(namespaceName).deleteSecret(namespaceName, s.getMetadata().getName());
    }
    PodUtils.verifyThatRunningPodsAreStable(namespaceName, KafkaResources.kafkaStatefulSetName(clusterName));
    RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, 3, kafkaPods);
    for (Secret s : secrets) {
        SecretUtils.waitForSecretReady(namespaceName, s.getMetadata().getName(), () -> {
        });
    }
    List<Secret> regeneratedSecrets = kubeClient(namespaceName).listSecrets(namespaceName).stream().filter(secret -> secret.getMetadata().getName().endsWith("ca-cert")).collect(Collectors.toList());
    for (int i = 0; i < secrets.size(); i++) {
        assertThat("Certificates has different cert UIDs", !secrets.get(i).getData().get("ca.crt").equals(regeneratedSecrets.get(i).getData().get("ca.crt")));
    }
    LOGGER.info("Checking consumed messages to pod:{}", defaultKafkaClientsPodName);
    internalKafkaClient.checkProducedAndConsumedMessages(internalKafkaClient.sendMessagesTls(), internalKafkaClient.receiveMessagesTls());
}

Example 24 with InternalKafkaClient

use of io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient in project strimzi by strimzi.

the class SecurityST method testCaRenewalBreakInMiddle.

@ParallelNamespaceTest
@Tag(INTERNAL_CLIENTS_USED)
void testCaRenewalBreakInMiddle(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    final String userName = mapWithTestUsers.get(extensionContext.getDisplayName());
    final LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, KafkaResources.kafkaStatefulSetName(clusterName));
    final LabelSelector zkSelector = KafkaResource.getLabelSelector(clusterName, KafkaResources.zookeeperStatefulSetName(clusterName));
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(clusterName, 3, 3).editSpec().withNewClusterCa().withRenewalDays(1).withValidityDays(3).endClusterCa().endSpec().build());
    KafkaUser user = KafkaUserTemplates.tlsUser(clusterName, userName).build();
    resourceManager.createResource(extensionContext, user);
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName).build());
    resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(true, clusterName + "-" + Constants.KAFKA_CLIENTS, user).build());
    String defaultKafkaClientsPodName = kubeClient(namespaceName).listPodsByPrefixInName(namespaceName, clusterName + "-" + Constants.KAFKA_CLIENTS).get(0).getMetadata().getName();
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(defaultKafkaClientsPodName).withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withKafkaUsername(userName).withMessageCount(MESSAGE_COUNT).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    internalKafkaClient = internalKafkaClient.toBuilder().withUsingPodName(defaultKafkaClientsPodName).build();
    internalKafkaClient.checkProducedAndConsumedMessages(internalKafkaClient.sendMessagesTls(), internalKafkaClient.receiveMessagesTls());
    Map<String, String> zkPods = PodUtils.podSnapshot(namespaceName, zkSelector);
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
    Map<String, String> eoPods = DeploymentUtils.depSnapshot(namespaceName, KafkaResources.entityOperatorDeploymentName(clusterName));
    InputStream secretInputStream = getClass().getClassLoader().getResourceAsStream("security-st-certs/expired-cluster-ca.crt");
    String clusterCaCert = TestUtils.readResource(secretInputStream);
    SecretUtils.createSecret(namespaceName, clusterCaCertificateSecretName(clusterName), "ca.crt", clusterCaCert);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, k -> {
        k.getSpec().getZookeeper().setResources(new ResourceRequirementsBuilder().addToRequests("cpu", new Quantity("100000m")).build());
        k.getSpec().setClusterCa(new CertificateAuthorityBuilder().withRenewalDays(4).withValidityDays(7).build());
    }, namespaceName);
    TestUtils.waitFor("Waiting for some kafka pod to be in the pending phase because of selected high cpu resource", Constants.GLOBAL_POLL_INTERVAL, Constants.GLOBAL_TIMEOUT, () -> {
        List<Pod> pendingPods = kubeClient(namespaceName).listPodsByPrefixInName(namespaceName, KafkaResources.zookeeperStatefulSetName(clusterName)).stream().filter(pod -> pod.getStatus().getPhase().equals("Pending")).collect(Collectors.toList());
        if (pendingPods.isEmpty()) {
            LOGGER.info("No pods of {} are in desired state", KafkaResources.zookeeperStatefulSetName(clusterName));
            return false;
        } else {
            LOGGER.info("Pod in 'Pending' state: {}", pendingPods.get(0).getMetadata().getName());
            return true;
        }
    });
    internalKafkaClient = internalKafkaClient.toBuilder().withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
    int received = internalKafkaClient.receiveMessagesTls();
    assertThat(received, is(MESSAGE_COUNT));
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, k -> {
        k.getSpec().getZookeeper().setResources(new ResourceRequirementsBuilder().addToRequests("cpu", new Quantity("200m")).build());
    }, namespaceName);
    // Wait until the certificates have been replaced
    SecretUtils.waitForCertToChange(namespaceName, clusterCaCert, KafkaResources.clusterCaCertificateSecretName(clusterName));
    RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(namespaceName, zkSelector, 3, zkPods);
    RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(namespaceName, kafkaSelector, 3, kafkaPods);
    DeploymentUtils.waitTillDepHasRolled(namespaceName, KafkaResources.entityOperatorDeploymentName(clusterName), 1, eoPods);
    internalKafkaClient = internalKafkaClient.toBuilder().withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
    LOGGER.info("Checking produced and consumed messages to pod:{}", internalKafkaClient.getPodName());
    received = internalKafkaClient.receiveMessagesTls();
    assertThat(received, is(MESSAGE_COUNT));
    // Try to send and receive messages with new certificates
    topicName = KafkaTopicUtils.generateRandomNameOfTopic();
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName).build());
    internalKafkaClient = internalKafkaClient.toBuilder().withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).withTopicName(topicName).build();
    internalKafkaClient.checkProducedAndConsumedMessages(internalKafkaClient.sendMessagesTls(), internalKafkaClient.receiveMessagesTls());
}

Example 25 with InternalKafkaClient

use of io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient in project strimzi by strimzi.

the class MirrorMakerIsolatedST method testMirrorMakerTlsScramSha.

/**
 * Test mirroring messages by Mirror Maker over tls transport using scram-sha auth
 */
@ParallelNamespaceTest
@SuppressWarnings("checkstyle:methodlength")
void testMirrorMakerTlsScramSha(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(INFRA_NAMESPACE, extensionContext);
    String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    String kafkaClusterSourceName = clusterName + "-source";
    String kafkaClusterTargetName = clusterName + "-target";
    String kafkaUserSource = clusterName + "-my-user-source";
    String kafkaUserTarget = clusterName + "-my-user-target";
    // Deploy source kafka with tls listener and SCRAM-SHA authentication
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterSourceName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationScramSha512()).build()).endKafka().endSpec().build());
    // Deploy target kafka with tls listener and SCRAM-SHA authentication
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterTargetName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationScramSha512()).build()).endKafka().endSpec().build());
    // Deploy topic
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, topicName).build());
    // createAndWaitForReadiness Kafka user for source cluster
    KafkaUser userSource = KafkaUserTemplates.scramShaUser(kafkaClusterSourceName, kafkaUserSource).build();
    // createAndWaitForReadiness Kafka user for target cluster
    KafkaUser userTarget = KafkaUserTemplates.scramShaUser(kafkaClusterTargetName, kafkaUserTarget).build();
    resourceManager.createResource(extensionContext, userSource);
    resourceManager.createResource(extensionContext, userTarget);
    // Initialize PasswordSecretSource to set this as PasswordSecret in Mirror Maker spec
    PasswordSecretSource passwordSecretSource = new PasswordSecretSource();
    passwordSecretSource.setSecretName(kafkaUserSource);
    passwordSecretSource.setPassword("password");
    // Initialize PasswordSecretSource to set this as PasswordSecret in Mirror Maker spec
    PasswordSecretSource passwordSecretTarget = new PasswordSecretSource();
    passwordSecretTarget.setSecretName(kafkaUserTarget);
    passwordSecretTarget.setPassword("password");
    // Initialize CertSecretSource with certificate and secret names for consumer
    CertSecretSource certSecretSource = new CertSecretSource();
    certSecretSource.setCertificate("ca.crt");
    certSecretSource.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterSourceName));
    // Initialize CertSecretSource with certificate and secret names for producer
    CertSecretSource certSecretTarget = new CertSecretSource();
    certSecretTarget.setCertificate("ca.crt");
    certSecretTarget.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterTargetName));
    // Deploy client
    resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(namespaceName, true, clusterName + "-" + Constants.KAFKA_CLIENTS, userSource, userTarget).build());
    final String kafkaClientsPodName = PodUtils.getPodsByPrefixInNameWithDynamicWait(namespaceName, clusterName + "-" + Constants.KAFKA_CLIENTS).get(0).getMetadata().getName();
    String baseTopic = mapWithTestTopics.get(extensionContext.getDisplayName());
    String topicTestName1 = baseTopic + "-test-1";
    String topicTestName2 = baseTopic + "-test-2";
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, topicTestName1).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, topicTestName2).build());
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(kafkaClientsPodName).withTopicName(topicTestName1).withNamespaceName(namespaceName).withClusterName(kafkaClusterSourceName).withKafkaUsername(userSource.getMetadata().getName()).withMessageCount(messagesCount).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    // Check brokers availability
    internalKafkaClient.produceAndConsumesTlsMessagesUntilBothOperationsAreSuccessful();
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicTestName2).withClusterName(kafkaClusterTargetName).withKafkaUsername(userTarget.getMetadata().getName()).build();
    internalKafkaClient.produceAndConsumesTlsMessagesUntilBothOperationsAreSuccessful();
    // Deploy Mirror Maker with TLS and ScramSha512
    resourceManager.createResource(extensionContext, KafkaMirrorMakerTemplates.kafkaMirrorMaker(clusterName, kafkaClusterSourceName, kafkaClusterTargetName, ClientUtils.generateRandomConsumerGroup(), 1, true).editSpec().editConsumer().withNewKafkaClientAuthenticationScramSha512().withUsername(kafkaUserSource).withPasswordSecret(passwordSecretSource).endKafkaClientAuthenticationScramSha512().withNewTls().withTrustedCertificates(certSecretSource).endTls().endConsumer().editProducer().withNewKafkaClientAuthenticationScramSha512().withUsername(kafkaUserTarget).withPasswordSecret(passwordSecretTarget).endKafkaClientAuthenticationScramSha512().withNewTls().withTrustedCertificates(certSecretTarget).endTls().endProducer().endSpec().build());
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicName).withClusterName(kafkaClusterSourceName).withKafkaUsername(userSource.getMetadata().getName()).build();
    internalKafkaClient.produceAndConsumesTlsMessagesUntilBothOperationsAreSuccessful();
    InternalKafkaClient newInternalKafkaClient = internalKafkaClient.toBuilder().withClusterName(kafkaClusterTargetName).withKafkaUsername(userTarget.getMetadata().getName()).build();
    newInternalKafkaClient.consumesTlsMessagesUntilOperationIsSuccessful(internalKafkaClient.getMessageCount());
}