Examples with InternalKafkaClient io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient used on opensource projects

Example 11 with InternalKafkaClient

use of io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient in project strimzi by strimzi.

the class MirrorMaker2IsolatedST method testKMM2RollAfterSecretsCertsUpdateTLS.

@ParallelNamespaceTest
@SuppressWarnings({ "checkstyle:MethodLength" })
void testKMM2RollAfterSecretsCertsUpdateTLS(ExtensionContext extensionContext) {
    TestStorage testStorage = new TestStorage(extensionContext);
    String kafkaClusterSourceName = testStorage.getClusterName() + "-source";
    String kafkaClusterTargetName = testStorage.getClusterName() + "-target";
    String topicSourceNameA = MIRRORMAKER2_TOPIC_NAME + "-a-" + rng.nextInt(Integer.MAX_VALUE);
    String topicSourceNameB = MIRRORMAKER2_TOPIC_NAME + "-b-" + rng.nextInt(Integer.MAX_VALUE);
    String topicTargetNameA = kafkaClusterSourceName + "." + topicSourceNameA;
    String topicTargetNameB = kafkaClusterSourceName + "." + topicSourceNameB;
    String kafkaUserSourceName = testStorage.getClusterName() + "-my-user-source";
    String kafkaUserTargetName = testStorage.getClusterName() + "-my-user-target";
    String kafkaTlsScramListenerName = "tlsscram";
    final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    // Deploy source kafka with tls listener and mutual tls auth
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterSourceName, 3, 3).editSpec().editKafka().addToConfig("min.insync.replicas", 1).withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationTls()).build()).endKafka().endSpec().build());
    // Deploy target kafka with tls listener and mutual tls auth
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterTargetName, 3, 3).editSpec().editKafka().addToConfig("min.insync.replicas", 1).withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationTls()).build()).endKafka().endSpec().build());
    // Deploy topic
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, topicSourceNameA, 3).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, topicSourceNameB, 3).build());
    // Create Kafka user
    KafkaUser userSource = KafkaUserTemplates.tlsUser(kafkaClusterSourceName, kafkaUserSourceName).build();
    KafkaUser userTarget = KafkaUserTemplates.tlsUser(kafkaClusterTargetName, kafkaUserTargetName).build();
    resourceManager.createResource(extensionContext, userSource);
    resourceManager.createResource(extensionContext, userTarget);
    resourceManager.createResource(extensionContext, false, KafkaClientsTemplates.kafkaClients(testStorage.getNamespaceName(), true, kafkaClientsName, userSource, userTarget).build());
    final String kafkaClientsPodName = PodUtils.getPodsByPrefixInNameWithDynamicWait(testStorage.getNamespaceName(), kafkaClientsName).get(0).getMetadata().getName();
    String baseTopic = mapWithTestTopics.get(extensionContext.getDisplayName());
    String topicTestName1 = baseTopic + "-test-1";
    String topicTestName2 = baseTopic + "-test-2";
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, topicTestName1).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterTargetName, topicTestName2).build());
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(kafkaClientsPodName).withTopicName(topicTestName1).withNamespaceName(testStorage.getNamespaceName()).withClusterName(kafkaClusterSourceName).withKafkaUsername(kafkaUserSourceName).withMessageCount(messagesCount).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    internalKafkaClient = internalKafkaClient.toBuilder().withClusterName(kafkaClusterTargetName).withTopicName(topicTestName2).withKafkaUsername(kafkaUserTargetName).build();
    internalKafkaClient.checkProducedAndConsumedMessages(internalKafkaClient.sendMessagesTls(), internalKafkaClient.receiveMessagesTls());
    // Initialize CertSecretSource with certificate and secret names for source
    CertSecretSource certSecretSource = new CertSecretSource();
    certSecretSource.setCertificate("ca.crt");
    certSecretSource.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterSourceName));
    // Initialize CertSecretSource with certificate and secret names for target
    CertSecretSource certSecretTarget = new CertSecretSource();
    certSecretTarget.setCertificate("ca.crt");
    certSecretTarget.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterTargetName));
    // Deploy Mirror Maker 2.0 with tls listener and mutual tls auth
    KafkaMirrorMaker2ClusterSpec sourceClusterWithTlsAuth = new KafkaMirrorMaker2ClusterSpecBuilder().withAlias(kafkaClusterSourceName).withBootstrapServers(KafkaResources.tlsBootstrapAddress(kafkaClusterSourceName)).withNewKafkaClientAuthenticationTls().withNewCertificateAndKey().withSecretName(kafkaUserSourceName).withCertificate("user.crt").withKey("user.key").endCertificateAndKey().endKafkaClientAuthenticationTls().withNewTls().withTrustedCertificates(certSecretSource).endTls().build();
    KafkaMirrorMaker2ClusterSpec targetClusterWithTlsAuth = new KafkaMirrorMaker2ClusterSpecBuilder().withAlias(kafkaClusterTargetName).withBootstrapServers(KafkaResources.tlsBootstrapAddress(kafkaClusterTargetName)).withNewKafkaClientAuthenticationTls().withNewCertificateAndKey().withSecretName(kafkaUserTargetName).withCertificate("user.crt").withKey("user.key").endCertificateAndKey().endKafkaClientAuthenticationTls().withNewTls().withTrustedCertificates(certSecretTarget).endTls().addToConfig("config.storage.replication.factor", -1).addToConfig("offset.storage.replication.factor", -1).addToConfig("status.storage.replication.factor", -1).build();
    resourceManager.createResource(extensionContext, KafkaMirrorMaker2Templates.kafkaMirrorMaker2(testStorage.getClusterName(), kafkaClusterTargetName, kafkaClusterSourceName, 1, true).editSpec().withClusters(targetClusterWithTlsAuth, sourceClusterWithTlsAuth).editFirstMirror().withTopicsPattern(MIRRORMAKER2_TOPIC_NAME + ".*").editSourceConnector().addToConfig("refresh.topics.interval.seconds", 1).endSourceConnector().endMirror().endSpec().build());
    String mm2DeploymentName = KafkaMirrorMaker2Resources.deploymentName(testStorage.getClusterName());
    Map<String, String> mmSnapshot = DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), mm2DeploymentName);
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicSourceNameA).withClusterName(kafkaClusterSourceName).withKafkaUsername(kafkaUserSourceName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    int sent = internalKafkaClient.sendMessagesTls();
    internalKafkaClient.checkProducedAndConsumedMessages(sent, internalKafkaClient.receiveMessagesTls());
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicTargetNameA).withClusterName(kafkaClusterTargetName).withKafkaUsername(kafkaUserTargetName).build();
    LOGGER.info("Consumer in target cluster and topic should receive {} messages", messagesCount);
    internalKafkaClient.checkProducedAndConsumedMessages(sent, internalKafkaClient.receiveMessagesTls());
    LOGGER.info("Messages successfully mirrored");
    LabelSelector zkSourceSelector = KafkaResource.getLabelSelector(kafkaClusterSourceName, KafkaResources.zookeeperStatefulSetName(kafkaClusterSourceName));
    LabelSelector kafkaSourceSelector = KafkaResource.getLabelSelector(kafkaClusterSourceName, KafkaResources.kafkaStatefulSetName(kafkaClusterSourceName));
    LabelSelector zkTargetSelector = KafkaResource.getLabelSelector(kafkaClusterTargetName, KafkaResources.zookeeperStatefulSetName(kafkaClusterTargetName));
    LabelSelector kafkaTargetSelector = KafkaResource.getLabelSelector(kafkaClusterTargetName, KafkaResources.kafkaStatefulSetName(kafkaClusterTargetName));
    Map<String, String> kafkaSourcePods = PodUtils.podSnapshot(testStorage.getNamespaceName(), kafkaSourceSelector);
    Map<String, String> zkSourcePods = PodUtils.podSnapshot(testStorage.getNamespaceName(), zkSourceSelector);
    Map<String, String> eoSourcePods = DeploymentUtils.depSnapshot(KafkaResources.entityOperatorDeploymentName(kafkaClusterSourceName));
    Map<String, String> kafkaTargetPods = PodUtils.podSnapshot(testStorage.getNamespaceName(), kafkaTargetSelector);
    Map<String, String> zkTargetPods = PodUtils.podSnapshot(testStorage.getNamespaceName(), zkTargetSelector);
    Map<String, String> eoTargetPods = DeploymentUtils.depSnapshot(KafkaResources.entityOperatorDeploymentName(kafkaClusterTargetName));
    LOGGER.info("Renew Clients CA secret for Source cluster via annotation");
    String sourceClientsCaSecretName = KafkaResources.clientsCaCertificateSecretName(kafkaClusterSourceName);
    SecretUtils.annotateSecret(testStorage.getNamespaceName(), sourceClientsCaSecretName, Ca.ANNO_STRIMZI_IO_FORCE_RENEW, "true");
    kafkaSourcePods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), kafkaSourceSelector, 3, kafkaSourcePods);
    mmSnapshot = DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), mm2DeploymentName, 1, mmSnapshot);
    LOGGER.info("Renew Clients CA secret for Target cluster via annotation");
    String targetClientsCaSecretName = KafkaResources.clientsCaCertificateSecretName(kafkaClusterTargetName);
    SecretUtils.annotateSecret(testStorage.getNamespaceName(), targetClientsCaSecretName, Ca.ANNO_STRIMZI_IO_FORCE_RENEW, "true");
    kafkaTargetPods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), kafkaTargetSelector, 3, kafkaTargetPods);
    mmSnapshot = DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), mm2DeploymentName, 1, mmSnapshot);
    LOGGER.info("Send and receive messages after clients certs were removed");
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicSourceNameA).withClusterName(kafkaClusterSourceName).withKafkaUsername(kafkaUserSourceName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    sent = internalKafkaClient.sendMessagesTls();
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicTargetNameA).withClusterName(kafkaClusterTargetName).withKafkaUsername(kafkaUserTargetName).withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
    LOGGER.info("Consumer in target cluster and topic should receive {} messages", messagesCount);
    internalKafkaClient.checkProducedAndConsumedMessages(sent, internalKafkaClient.receiveMessagesTls());
    LOGGER.info("Messages successfully mirrored");
    LOGGER.info("Renew Cluster CA secret for Source clusters via annotation");
    String sourceClusterCaSecretName = KafkaResources.clusterCaCertificateSecretName(kafkaClusterSourceName);
    SecretUtils.annotateSecret(testStorage.getNamespaceName(), sourceClusterCaSecretName, Ca.ANNO_STRIMZI_IO_FORCE_RENEW, "true");
    zkSourcePods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), zkSourceSelector, 3, zkSourcePods);
    kafkaSourcePods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), kafkaSourceSelector, 3, kafkaSourcePods);
    eoSourcePods = DeploymentUtils.waitTillDepHasRolled(KafkaResources.entityOperatorDeploymentName(kafkaClusterSourceName), 1, eoSourcePods);
    mmSnapshot = DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), mm2DeploymentName, 1, mmSnapshot);
    LOGGER.info("Renew Cluster CA secret for Target clusters via annotation");
    String targetClusterCaSecretName = KafkaResources.clusterCaCertificateSecretName(kafkaClusterTargetName);
    SecretUtils.annotateSecret(testStorage.getNamespaceName(), targetClusterCaSecretName, Ca.ANNO_STRIMZI_IO_FORCE_RENEW, "true");
    zkTargetPods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), zkTargetSelector, 3, zkTargetPods);
    kafkaTargetPods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), kafkaTargetSelector, 3, kafkaTargetPods);
    eoTargetPods = DeploymentUtils.waitTillDepHasRolled(KafkaResources.entityOperatorDeploymentName(kafkaClusterTargetName), 1, eoTargetPods);
    DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), mm2DeploymentName, 1, mmSnapshot);
    LOGGER.info("Send and receive messages after clients certs were removed");
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicSourceNameB).withClusterName(kafkaClusterSourceName).withKafkaUsername(kafkaUserSourceName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    sent = internalKafkaClient.sendMessagesTls();
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicTargetNameB).withClusterName(kafkaClusterTargetName).withKafkaUsername(kafkaUserTargetName).withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
    LOGGER.info("Consumer in target cluster and topic should receive {} messages", messagesCount);
    internalKafkaClient.checkProducedAndConsumedMessages(sent, internalKafkaClient.receiveMessagesTls());
    LOGGER.info("Messages successfully mirrored");
}

Example 12 with InternalKafkaClient

use of io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient in project strimzi by strimzi.

the class NetworkPoliciesIsolatedST method testNetworkPoliciesWithPlainListener.

@IsolatedTest("Specific cluster operator for test case")
@Tag(INTERNAL_CLIENTS_USED)
void testNetworkPoliciesWithPlainListener(ExtensionContext extensionContext) {
    String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    clusterOperator.unInstall();
    clusterOperator = new SetupClusterOperator.SetupClusterOperatorBuilder().withExtensionContext(BeforeAllOnce.getSharedExtensionContext()).withNamespace(namespace).createInstallation().runInstallation();
    String allowedKafkaClientsName = clusterName + "-" + Constants.KAFKA_CLIENTS + "-allow";
    String deniedKafkaClientsName = clusterName + "-" + Constants.KAFKA_CLIENTS + "-deny";
    Map<String, String> matchLabelForPlain = new HashMap<>();
    matchLabelForPlain.put("app", allowedKafkaClientsName);
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(clusterName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.PLAIN_LISTENER_DEFAULT_NAME).withPort(9092).withType(KafkaListenerType.INTERNAL).withTls(false).withNewKafkaListenerAuthenticationScramSha512Auth().endKafkaListenerAuthenticationScramSha512Auth().withNetworkPolicyPeers(new NetworkPolicyPeerBuilder().withNewPodSelector().withMatchLabels(matchLabelForPlain).endPodSelector().build()).build()).endKafka().withNewKafkaExporter().endKafkaExporter().endSpec().build());
    NetworkPolicyResource.allowNetworkPolicySettingsForKafkaExporter(extensionContext, clusterName);
    String topic0 = "topic-example-0";
    String topic1 = "topic-example-1";
    String userName = "user-example";
    KafkaUser kafkaUser = KafkaUserTemplates.scramShaUser(clusterName, userName).build();
    resourceManager.createResource(extensionContext, kafkaUser);
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topic0).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topic1).build());
    resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(false, allowedKafkaClientsName, kafkaUser).build());
    String allowedKafkaClientsPodName = kubeClient().listPodsByPrefixInName(allowedKafkaClientsName).get(0).getMetadata().getName();
    LOGGER.info("Verifying that {} pod is able to exchange messages", allowedKafkaClientsPodName);
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(allowedKafkaClientsPodName).withTopicName(topic0).withNamespaceName(namespace).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withKafkaUsername(userName).withSecurityProtocol(SecurityProtocol.PLAINTEXT).withListenerName(Constants.PLAIN_LISTENER_DEFAULT_NAME).build();
    internalKafkaClient.checkProducedAndConsumedMessages(internalKafkaClient.sendMessagesPlain(), internalKafkaClient.receiveMessagesPlain());
    resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(false, deniedKafkaClientsName, kafkaUser).build());
    String deniedKafkaClientsPodName = kubeClient().listPodsByPrefixInName(deniedKafkaClientsName).get(0).getMetadata().getName();
    InternalKafkaClient newInternalKafkaClient = internalKafkaClient.toBuilder().withUsingPodName(deniedKafkaClientsPodName).withTopicName(topic1).build();
    LOGGER.info("Verifying that {} pod is not able to exchange messages", deniedKafkaClientsPodName);
    assertThrows(AssertionError.class, () -> {
        newInternalKafkaClient.checkProducedAndConsumedMessages(newInternalKafkaClient.sendMessagesPlain(), newInternalKafkaClient.receiveMessagesPlain());
    });
    LOGGER.info("Check metrics exported by Kafka Exporter");
    MetricsCollector metricsCollector = new MetricsCollector.Builder().withScraperPodName(allowedKafkaClientsPodName).withComponentName(clusterName).withComponentType(ComponentType.KafkaExporter).build();
    Map<String, String> kafkaExporterMetricsData = metricsCollector.collectMetricsFromPods();
    assertThat("Kafka Exporter metrics should be non-empty", kafkaExporterMetricsData.size() > 0);
    for (Map.Entry<String, String> entry : kafkaExporterMetricsData.entrySet()) {
        assertThat("Value from collected metric should be non-empty", !entry.getValue().isEmpty());
        assertThat("Metrics doesn't contain specific values", entry.getValue().contains("kafka_consumergroup_current_offset"));
        assertThat("Metrics doesn't contain specific values", entry.getValue().contains("kafka_topic_partitions{topic=\"" + topic0 + "\"} 1"));
        assertThat("Metrics doesn't contain specific values", entry.getValue().contains("kafka_topic_partitions{topic=\"" + topic1 + "\"} 1"));
    }
}

Example 13 with InternalKafkaClient

use of io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient in project strimzi by strimzi.

the class NetworkPoliciesIsolatedST method testNetworkPoliciesWithTlsListener.

@IsolatedTest("Specific cluster operator for test case")
@Tag(INTERNAL_CLIENTS_USED)
void testNetworkPoliciesWithTlsListener(ExtensionContext extensionContext) {
    String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    clusterOperator.unInstall();
    clusterOperator = new SetupClusterOperator.SetupClusterOperatorBuilder().withExtensionContext(BeforeAllOnce.getSharedExtensionContext()).withNamespace(namespace).createInstallation().runInstallation();
    String allowedKafkaClientsName = clusterName + "-" + Constants.KAFKA_CLIENTS + "-allow";
    String deniedKafkaClientsName = clusterName + "-" + Constants.KAFKA_CLIENTS + "-deny";
    Map<String, String> matchLabelsForTls = new HashMap<>();
    matchLabelsForTls.put("app", allowedKafkaClientsName);
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(clusterName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withNewKafkaListenerAuthenticationScramSha512Auth().endKafkaListenerAuthenticationScramSha512Auth().withNetworkPolicyPeers(new NetworkPolicyPeerBuilder().withNewPodSelector().withMatchLabels(matchLabelsForTls).endPodSelector().build()).build()).endKafka().endSpec().build());
    String topic0 = "topic-example-0";
    String topic1 = "topic-example-1";
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topic0).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topic1).build());
    String userName = "user-example";
    KafkaUser kafkaUser = KafkaUserTemplates.scramShaUser(clusterName, userName).build();
    resourceManager.createResource(extensionContext, kafkaUser);
    resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(true, allowedKafkaClientsName, kafkaUser).build());
    String allowedKafkaClientsPodName = kubeClient().listPodsByPrefixInName(allowedKafkaClientsName).get(0).getMetadata().getName();
    LOGGER.info("Verifying that {} pod is able to exchange messages", allowedKafkaClientsPodName);
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(allowedKafkaClientsPodName).withTopicName(topic0).withNamespaceName(namespace).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withKafkaUsername(userName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    internalKafkaClient.checkProducedAndConsumedMessages(internalKafkaClient.sendMessagesTls(), internalKafkaClient.receiveMessagesTls());
    resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(true, deniedKafkaClientsName, kafkaUser).build());
    String deniedKafkaClientsPodName = kubeClient().listPodsByPrefixInName(deniedKafkaClientsName).get(0).getMetadata().getName();
    InternalKafkaClient newInternalKafkaClient = internalKafkaClient.toBuilder().withUsingPodName(deniedKafkaClientsPodName).withTopicName(topic1).withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
    LOGGER.info("Verifying that {} pod is not able to exchange messages", deniedKafkaClientsPodName);
    assertThrows(AssertionError.class, () -> {
        newInternalKafkaClient.checkProducedAndConsumedMessages(newInternalKafkaClient.sendMessagesTls(), newInternalKafkaClient.receiveMessagesTls());
    });
}

Example 14 with InternalKafkaClient

use of io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient in project strimzi by strimzi.

the class SecurityST method testCertRenewalInMaintenanceWindow.

@ParallelNamespaceTest
@Tag(INTERNAL_CLIENTS_USED)
void testCertRenewalInMaintenanceWindow(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    final String secretName = KafkaResources.clusterCaCertificateSecretName(clusterName);
    final String userName = mapWithTestUsers.get(extensionContext.getDisplayName());
    final LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, KafkaResources.kafkaStatefulSetName(clusterName));
    LocalDateTime maintenanceWindowStart = LocalDateTime.now().withSecond(0);
    long maintenanceWindowDuration = 14;
    maintenanceWindowStart = maintenanceWindowStart.plusMinutes(5);
    final long windowStartMin = maintenanceWindowStart.getMinute();
    final long windowStopMin = windowStartMin + maintenanceWindowDuration > 59 ? windowStartMin + maintenanceWindowDuration - 60 : windowStartMin + maintenanceWindowDuration;
    String maintenanceWindowCron = "* " + windowStartMin + "-" + windowStopMin + " * * * ? *";
    LOGGER.info("Maintenance window is: {}", maintenanceWindowCron);
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(clusterName, 3, 1).editSpec().addToMaintenanceTimeWindows(maintenanceWindowCron).endSpec().build());
    KafkaUser user = KafkaUserTemplates.tlsUser(clusterName, userName).build();
    resourceManager.createResource(extensionContext, user);
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName).build());
    resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(true, clusterName + "-" + Constants.KAFKA_CLIENTS, user).build());
    String defaultKafkaClientsPodName = kubeClient(namespaceName).listPodsByPrefixInName(namespaceName, clusterName + "-" + Constants.KAFKA_CLIENTS).get(0).getMetadata().getName();
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(defaultKafkaClientsPodName).withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withKafkaUsername(user.getMetadata().getName()).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
    LOGGER.info("Annotate secret {} with secret force-renew annotation", secretName);
    Secret secret = new SecretBuilder(kubeClient(namespaceName).getSecret(namespaceName, secretName)).editMetadata().addToAnnotations(Ca.ANNO_STRIMZI_IO_FORCE_RENEW, "true").endMetadata().build();
    kubeClient(namespaceName).patchSecret(namespaceName, secretName, secret);
    LOGGER.info("Wait until maintenance windows starts");
    LocalDateTime finalMaintenanceWindowStart = maintenanceWindowStart;
    TestUtils.waitFor("maintenance window start", Constants.GLOBAL_POLL_INTERVAL, Duration.ofMinutes(maintenanceWindowDuration).toMillis() - 10000, () -> LocalDateTime.now().isAfter(finalMaintenanceWindowStart));
    LOGGER.info("Maintenance window starts");
    assertThat("Rolling update was performed out of maintenance window!", kafkaPods, is(PodUtils.podSnapshot(namespaceName, kafkaSelector)));
    LOGGER.info("Wait until rolling update is triggered during maintenance window");
    RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, 3, kafkaPods);
    assertThat("Rolling update wasn't performed in correct time", LocalDateTime.now().isAfter(maintenanceWindowStart));
    LOGGER.info("Checking consumed messages to pod:{}", defaultKafkaClientsPodName);
    internalKafkaClient.checkProducedAndConsumedMessages(internalKafkaClient.sendMessagesTls(), internalKafkaClient.receiveMessagesTls());
}

Example 15 with InternalKafkaClient

use of io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient in project strimzi by strimzi.

the class MirrorMaker2IsolatedST method testMirrorMaker2TlsAndScramSha512Auth.

/**
 * Test mirroring messages by MirrorMaker 2.0 over tls transport using scram-sha-512 auth
 */
@SuppressWarnings({ "checkstyle:MethodLength" })
@ParallelNamespaceTest
void testMirrorMaker2TlsAndScramSha512Auth(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(INFRA_NAMESPACE, extensionContext);
    final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    String kafkaClusterSourceName = clusterName + "-source";
    String kafkaClusterTargetName = clusterName + "-target";
    String sourceTopicName = "availability-topic-source-" + mapWithTestTopics.get(extensionContext.getDisplayName());
    String targetTopicName = "availability-topic-target-" + mapWithTestTopics.get(extensionContext.getDisplayName());
    String topicSourceNameMirrored = kafkaClusterSourceName + "." + sourceTopicName;
    String topicSourceName = MIRRORMAKER2_TOPIC_NAME + "-" + rng.nextInt(Integer.MAX_VALUE);
    String topicTargetName = kafkaClusterSourceName + "." + topicSourceName;
    String kafkaUserSource = clusterName + "-my-user-source";
    String kafkaUserTarget = clusterName + "-my-user-target";
    // Deploy source kafka with tls listener and SCRAM-SHA authentication
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterSourceName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationScramSha512()).build()).endKafka().endSpec().build());
    // Deploy target kafka with tls listener and SCRAM-SHA authentication
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterTargetName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationScramSha512()).build()).endKafka().endSpec().build());
    // Deploy topic
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, topicSourceName, 3).build());
    // Create Kafka user for source cluster
    KafkaUser userSource = KafkaUserTemplates.scramShaUser(kafkaClusterSourceName, kafkaUserSource).build();
    resourceManager.createResource(extensionContext, userSource);
    // Create Kafka user for target cluster
    KafkaUser userTarget = KafkaUserTemplates.scramShaUser(kafkaClusterTargetName, kafkaUserTarget).build();
    resourceManager.createResource(extensionContext, userTarget);
    // Initialize PasswordSecretSource to set this as PasswordSecret in MirrorMaker2 spec
    PasswordSecretSource passwordSecretSource = new PasswordSecretSource();
    passwordSecretSource.setSecretName(kafkaUserSource);
    passwordSecretSource.setPassword("password");
    // Initialize PasswordSecretSource to set this as PasswordSecret in MirrorMaker2 spec
    PasswordSecretSource passwordSecretTarget = new PasswordSecretSource();
    passwordSecretTarget.setSecretName(kafkaUserTarget);
    passwordSecretTarget.setPassword("password");
    // Initialize CertSecretSource with certificate and secret names for source
    CertSecretSource certSecretSource = new CertSecretSource();
    certSecretSource.setCertificate("ca.crt");
    certSecretSource.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterSourceName));
    // Initialize CertSecretSource with certificate and secret names for target
    CertSecretSource certSecretTarget = new CertSecretSource();
    certSecretTarget.setCertificate("ca.crt");
    certSecretTarget.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterTargetName));
    // Deploy client
    resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(namespaceName, true, kafkaClientsName, userSource, userTarget).build());
    final String kafkaClientsPodName = kubeClient().listPodsByPrefixInName(kafkaClientsName).get(0).getMetadata().getName();
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(kafkaClientsPodName).withTopicName(sourceTopicName).withNamespaceName(namespaceName).withClusterName(kafkaClusterSourceName).withKafkaUsername(userSource.getMetadata().getName()).withMessageCount(messagesCount).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    LOGGER.info("Sending messages to - topic {}, cluster {} and message count of {}", sourceTopicName, kafkaClusterSourceName, messagesCount);
    // Check brokers availability
    internalKafkaClient.checkProducedAndConsumedMessages(internalKafkaClient.sendMessagesTls(), internalKafkaClient.receiveMessagesTls());
    LOGGER.info("Setting topic to {}, cluster to {} and changing user to {}", targetTopicName, kafkaClusterTargetName, userTarget.getMetadata().getName());
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(targetTopicName).withClusterName(kafkaClusterTargetName).withKafkaUsername(userTarget.getMetadata().getName()).build();
    LOGGER.info("Sending messages to - topic {}, cluster {} and message count of {}", targetTopicName, kafkaClusterTargetName, messagesCount);
    internalKafkaClient.checkProducedAndConsumedMessages(internalKafkaClient.sendMessagesTls(), internalKafkaClient.receiveMessagesTls());
    // Deploy Mirror Maker with TLS and ScramSha512
    KafkaMirrorMaker2ClusterSpec sourceClusterWithScramSha512Auth = new KafkaMirrorMaker2ClusterSpecBuilder().withAlias(kafkaClusterSourceName).withBootstrapServers(KafkaResources.tlsBootstrapAddress(kafkaClusterSourceName)).withNewKafkaClientAuthenticationScramSha512().withUsername(kafkaUserSource).withPasswordSecret(passwordSecretSource).endKafkaClientAuthenticationScramSha512().withNewTls().withTrustedCertificates(certSecretSource).endTls().build();
    KafkaMirrorMaker2ClusterSpec targetClusterWithScramSha512Auth = new KafkaMirrorMaker2ClusterSpecBuilder().withAlias(kafkaClusterTargetName).withBootstrapServers(KafkaResources.tlsBootstrapAddress(kafkaClusterTargetName)).withNewKafkaClientAuthenticationScramSha512().withUsername(kafkaUserTarget).withPasswordSecret(passwordSecretTarget).endKafkaClientAuthenticationScramSha512().withNewTls().withTrustedCertificates(certSecretTarget).endTls().addToConfig("config.storage.replication.factor", -1).addToConfig("offset.storage.replication.factor", -1).addToConfig("status.storage.replication.factor", -1).build();
    resourceManager.createResource(extensionContext, KafkaMirrorMaker2Templates.kafkaMirrorMaker2(clusterName, kafkaClusterTargetName, kafkaClusterSourceName, 1, true).editSpec().withClusters(targetClusterWithScramSha512Auth, sourceClusterWithScramSha512Auth).editFirstMirror().withTopicsExcludePattern("availability.*").endMirror().endSpec().build());
    LOGGER.info("Setting topic to {}, cluster to {} and changing user to {}", topicSourceName, kafkaClusterSourceName, userSource.getMetadata().getName());
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicSourceName).withClusterName(kafkaClusterSourceName).withKafkaUsername(userSource.getMetadata().getName()).build();
    LOGGER.info("Sending messages to - topic {}, cluster {} and message count of {}", topicSourceName, kafkaClusterSourceName, messagesCount);
    int sent = internalKafkaClient.sendMessagesTls();
    internalKafkaClient.checkProducedAndConsumedMessages(sent, internalKafkaClient.receiveMessagesTls());
    LOGGER.info("Changing to target - topic {}, cluster {}, user {}", topicTargetName, kafkaClusterTargetName, userTarget.getMetadata().getName());
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicTargetName).withClusterName(kafkaClusterTargetName).withKafkaUsername(userTarget.getMetadata().getName()).build();
    LOGGER.info("Now messages should be mirrored to target topic and cluster");
    internalKafkaClient.checkProducedAndConsumedMessages(sent, internalKafkaClient.receiveMessagesTls());
    LOGGER.info("Messages successfully mirrored");
    KafkaTopicUtils.waitForKafkaTopicCreation(namespaceName, topicTargetName);
    KafkaTopic mirroredTopic = KafkaTopicResource.kafkaTopicClient().inNamespace(namespaceName).withName(topicTargetName).get();
    assertThat(mirroredTopic.getSpec().getPartitions(), is(3));
    assertThat(mirroredTopic.getMetadata().getLabels().get(Labels.STRIMZI_CLUSTER_LABEL), is(kafkaClusterTargetName));
    mirroredTopic = KafkaTopicResource.kafkaTopicClient().inNamespace(namespaceName).withName(topicSourceNameMirrored).get();
    assertThat(mirroredTopic, nullValue());
}