Search in sources :

Example 96 with KRaftNotSupported

use of io.strimzi.systemtest.annotations.KRaftNotSupported in project strimzi by strimzi.

the class LogSettingST method testKafkaLogSetting.

@IsolatedTest("Using shared Kafka")
@KRaftNotSupported("UserOperator is not supported by KRaft mode and is used in this test case")
void testKafkaLogSetting(ExtensionContext extensionContext) {
    String kafkaMap = KafkaResources.kafkaMetricsAndLogConfigMapName(LOG_SETTING_CLUSTER_NAME);
    String zookeeperMap = KafkaResources.zookeeperMetricsAndLogConfigMapName(LOG_SETTING_CLUSTER_NAME);
    String topicOperatorMap = String.format("%s-%s", LOG_SETTING_CLUSTER_NAME, "entity-topic-operator-config");
    String userOperatorMap = String.format("%s-%s", LOG_SETTING_CLUSTER_NAME, "entity-user-operator-config");
    String eoDepName = KafkaResources.entityOperatorDeploymentName(LOG_SETTING_CLUSTER_NAME);
    String kafkaSsName = KafkaResources.kafkaStatefulSetName(LOG_SETTING_CLUSTER_NAME);
    String zkSsName = KafkaResources.zookeeperStatefulSetName(LOG_SETTING_CLUSTER_NAME);
    LabelSelector kafkaSelector = KafkaResource.getLabelSelector(LOG_SETTING_CLUSTER_NAME, kafkaSsName);
    LabelSelector zkSelector = KafkaResource.getLabelSelector(LOG_SETTING_CLUSTER_NAME, zkSsName);
    Map<String, String> eoPods = DeploymentUtils.depSnapshot(namespace, eoDepName);
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespace, kafkaSelector);
    Map<String, String> zkPods = PodUtils.podSnapshot(namespace, zkSelector);
    String userName = mapWithTestUsers.get(extensionContext.getDisplayName());
    String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(LOG_SETTING_CLUSTER_NAME, topicName, namespace).build());
    resourceManager.createResource(extensionContext, KafkaUserTemplates.tlsUser(namespace, LOG_SETTING_CLUSTER_NAME, userName).build());
    LOGGER.info("Checking if Kafka, Zookeeper, TO and UO of cluster:{} has log level set properly", LOG_SETTING_CLUSTER_NAME);
    StUtils.getKafkaConfigurationConfigMaps(LOG_SETTING_CLUSTER_NAME, 3).forEach(cmName -> {
        assertThat("Kafka's log level is set properly", checkLoggersLevel(namespace, KAFKA_LOGGERS, cmName), is(true));
    });
    assertThat("Zookeeper's log level is set properly", checkLoggersLevel(namespace, ZOOKEEPER_LOGGERS, zookeeperMap), is(true));
    assertThat("Topic operator's log level is set properly", checkLoggersLevel(namespace, OPERATORS_LOGGERS, topicOperatorMap), is(true));
    assertThat("User operator's log level is set properly", checkLoggersLevel(namespace, OPERATORS_LOGGERS, userOperatorMap), is(true));
    LOGGER.info("Checking if Kafka, Zookeeper, TO and UO of cluster:{} has GC logging enabled in stateful sets/deployments", LOG_SETTING_CLUSTER_NAME);
    checkGcLoggingPods(namespace, kafkaSelector, true);
    checkGcLoggingPods(namespace, zkSelector, true);
    assertThat("TO GC logging is enabled", checkGcLoggingDeployments(namespace, eoDepName, "topic-operator"), is(true));
    assertThat("UO GC logging is enabled", checkGcLoggingDeployments(namespace, eoDepName, "user-operator"), is(true));
    LOGGER.info("Changing JVM options - setting GC logging to false");
    KafkaResource.replaceKafkaResourceInSpecificNamespace(LOG_SETTING_CLUSTER_NAME, kafka -> {
        kafka.getSpec().getKafka().setJvmOptions(JVM_OPTIONS);
        kafka.getSpec().getZookeeper().setJvmOptions(JVM_OPTIONS);
        kafka.getSpec().getEntityOperator().getTopicOperator().setJvmOptions(JVM_OPTIONS);
        kafka.getSpec().getEntityOperator().getUserOperator().setJvmOptions(JVM_OPTIONS);
    }, namespace);
    RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(namespace, zkSelector, 1, zkPods);
    RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(namespace, kafkaSelector, 3, kafkaPods);
    DeploymentUtils.waitTillDepHasRolled(namespace, eoDepName, 1, eoPods);
    LOGGER.info("Checking if Kafka, Zookeeper, TO and UO of cluster: {} has GC logging disabled in stateful sets/deployments", LOG_SETTING_CLUSTER_NAME);
    checkGcLoggingPods(namespace, kafkaSelector, false);
    checkGcLoggingPods(namespace, zkSelector, false);
    assertThat("TO GC logging is disabled", checkGcLoggingDeployments(namespace, eoDepName, "topic-operator"), is(false));
    assertThat("UO GC logging is disabled", checkGcLoggingDeployments(namespace, eoDepName, "user-operator"), is(false));
    LOGGER.info("Checking if Kafka, Zookeeper, TO and UO of cluster: {} has GC logging disabled in stateful sets/deployments", GC_LOGGING_SET_NAME);
    checkGcLoggingPods(namespace, kafkaSelector, false);
    checkGcLoggingPods(namespace, zkSelector, false);
    assertThat("TO GC logging is enabled", checkGcLoggingDeployments(namespace, eoDepName, "topic-operator"), is(false));
    assertThat("UO GC logging is enabled", checkGcLoggingDeployments(namespace, eoDepName, "user-operator"), is(false));
    kubectlGetStrimziUntilOperationIsSuccessful(namespace, LOG_SETTING_CLUSTER_NAME);
    kubectlGetStrimziUntilOperationIsSuccessful(namespace, GC_LOGGING_SET_NAME);
    checkContainersHaveProcessOneAsTini(namespace, LOG_SETTING_CLUSTER_NAME);
    checkContainersHaveProcessOneAsTini(namespace, GC_LOGGING_SET_NAME);
}
Also used : LabelSelector(io.fabric8.kubernetes.api.model.LabelSelector) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) IsolatedTest(io.strimzi.systemtest.annotations.IsolatedTest)

Example 97 with KRaftNotSupported

use of io.strimzi.systemtest.annotations.KRaftNotSupported in project strimzi by strimzi.

the class MetricsIsolatedST method testReconcileStateMetricInTopicOperator.

@IsolatedTest
@KRaftNotSupported("TopicOperator is not supported by KRaft mode and is used in this test class")
void testReconcileStateMetricInTopicOperator(ExtensionContext extensionContext) {
    cluster.setNamespace(SECOND_NAMESPACE);
    String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    int initialReplicas = 1;
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(SECOND_CLUSTER, topicName, 1, initialReplicas).build());
    String secondClientsPodName = kubeClient(SECOND_NAMESPACE).listPodsByPrefixInName(SECOND_NAMESPACE + "-" + Constants.SCRAPER_NAME).get(0).getMetadata().getName();
    MetricsCollector secondNamespaceCollector = new MetricsCollector.Builder().withNamespaceName(SECOND_NAMESPACE).withScraperPodName(secondClientsPodName).withComponentName(SECOND_CLUSTER).withComponentType(ComponentType.TopicOperator).build();
    Map<String, String> toMetrics = secondNamespaceCollector.collectMetricsFromPods();
    String reasonMessage = "none";
    LOGGER.info("Checking if resource state metric reason message is \"none\" and KafkaTopic is ready");
    assertResourceStateMetricInTopicOperator(topicName, reasonMessage, 1.0, toMetrics);
    LOGGER.info("Changing topic name in spec.topicName");
    KafkaTopicResource.replaceTopicResource(topicName, kafkaTopic -> kafkaTopic.getSpec().setTopicName("some-other-name"));
    KafkaTopicUtils.waitForKafkaTopicNotReady(topicName);
    toMetrics = secondNamespaceCollector.collectMetricsFromPods();
    reasonMessage = "Kafka topics cannot be renamed, but KafkaTopic's spec.topicName has changed.";
    assertResourceStateMetricInTopicOperator(topicName, reasonMessage, 0.0, toMetrics);
    LOGGER.info("Changing back to it's original name and scaling replicas to be higher number");
    KafkaTopicResource.replaceTopicResource(topicName, kafkaTopic -> {
        kafkaTopic.getSpec().setTopicName(topicName);
        kafkaTopic.getSpec().setReplicas(12);
    });
    KafkaTopicUtils.waitForKafkaTopicReplicasChange(SECOND_NAMESPACE, topicName, 12);
    toMetrics = secondNamespaceCollector.collectMetricsFromPods();
    reasonMessage = "Changing 'spec.replicas' is not supported. .*";
    assertResourceStateMetricInTopicOperator(topicName, reasonMessage, 0.0, toMetrics);
    LOGGER.info("Scaling replicas to be higher than before");
    KafkaTopicResource.replaceTopicResource(topicName, kafkaTopic -> kafkaTopic.getSpec().setReplicas(24));
    KafkaTopicUtils.waitForKafkaTopicReplicasChange(SECOND_NAMESPACE, topicName, 24);
    toMetrics = secondNamespaceCollector.collectMetricsFromPods();
    assertResourceStateMetricInTopicOperator(topicName, reasonMessage, 0.0, toMetrics);
    LOGGER.info("Changing KafkaTopic's spec to correct state");
    KafkaTopicResource.replaceTopicResource(topicName, kafkaTopic -> kafkaTopic.getSpec().setReplicas(initialReplicas));
    KafkaTopicUtils.waitForKafkaTopicReady(topicName);
    toMetrics = secondNamespaceCollector.collectMetricsFromPods();
    reasonMessage = "none";
    assertResourceStateMetricInTopicOperator(topicName, reasonMessage, 1.0, toMetrics);
    cluster.setNamespace(clusterOperator.getDeploymentNamespace());
}
Also used : Matchers.emptyString(org.hamcrest.Matchers.emptyString) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) IsolatedTest(io.strimzi.systemtest.annotations.IsolatedTest)

Example 98 with KRaftNotSupported

use of io.strimzi.systemtest.annotations.KRaftNotSupported in project strimzi by strimzi.

the class MirrorMaker2IsolatedST method testKMM2RollAfterSecretsCertsUpdateScramSha.

/**
 * Test mirroring messages by MirrorMaker 2.0 over tls transport using scram-sha-512 auth
 * while user Scram passwords, CA cluster and clients certificates are changed.
 */
@ParallelNamespaceTest
@KRaftNotSupported("UserOperator is not supported by KRaft mode and is used in this test case")
@SuppressWarnings({ "checkstyle:MethodLength" })
void testKMM2RollAfterSecretsCertsUpdateScramSha(ExtensionContext extensionContext) {
    TestStorage testStorage = new TestStorage(extensionContext);
    String kafkaClusterSourceName = testStorage.getClusterName() + "-source";
    String kafkaClusterTargetName = testStorage.getClusterName() + "-target";
    String sourceMirroredTopicName = kafkaClusterSourceName + "." + testStorage.getTopicName();
    String kafkaUserSourceName = testStorage.getUserName() + "-source";
    String kafkaUserTargetName = testStorage.getUserName() + "-target";
    // Deploy source kafka with tls listener and SCRAM-SHA authentication
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(kafkaClusterSourceName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationScramSha512()).build()).endKafka().endSpec().build());
    // Deploy target kafka with tls listeners with tls and SCRAM-SHA authentication
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(kafkaClusterTargetName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationScramSha512()).build()).endKafka().endSpec().build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, testStorage.getTopicName()).build(), KafkaUserTemplates.scramShaUser(kafkaClusterSourceName, kafkaUserSourceName).build(), KafkaUserTemplates.scramShaUser(kafkaClusterTargetName, kafkaUserTargetName).build());
    // Initialize PasswordSecretSource to set this as PasswordSecret in Source/Target MirrorMaker2 spec
    PasswordSecretSource passwordSecretSource = new PasswordSecretSource();
    passwordSecretSource.setSecretName(kafkaUserSourceName);
    passwordSecretSource.setPassword("password");
    PasswordSecretSource passwordSecretTarget = new PasswordSecretSource();
    passwordSecretTarget.setSecretName(kafkaUserTargetName);
    passwordSecretTarget.setPassword("password");
    // Initialize CertSecretSource with certificate and secret names for source
    CertSecretSource certSecretSource = new CertSecretSource();
    certSecretSource.setCertificate("ca.crt");
    certSecretSource.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterSourceName));
    // Initialize CertSecretSource with certificate and secret names for target
    CertSecretSource certSecretTarget = new CertSecretSource();
    certSecretTarget.setCertificate("ca.crt");
    certSecretTarget.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterTargetName));
    resourceManager.createResource(extensionContext, KafkaMirrorMaker2Templates.kafkaMirrorMaker2(testStorage.getClusterName(), kafkaClusterTargetName, kafkaClusterSourceName, 1, true).editSpec().editMatchingCluster(spec -> spec.getAlias().equals(kafkaClusterSourceName)).withNewKafkaClientAuthenticationScramSha512().withUsername(kafkaUserSourceName).withPasswordSecret(passwordSecretSource).endKafkaClientAuthenticationScramSha512().withNewTls().withTrustedCertificates(certSecretSource).endTls().endCluster().editMatchingCluster(spec -> spec.getAlias().equals(kafkaClusterTargetName)).withNewKafkaClientAuthenticationScramSha512().withUsername(kafkaUserTargetName).withPasswordSecret(passwordSecretTarget).endKafkaClientAuthenticationScramSha512().withNewTls().withTrustedCertificates(certSecretTarget).endTls().endCluster().endSpec().build());
    KafkaClients clients = new KafkaClientsBuilder().withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterSourceName)).withUserName(kafkaUserSourceName).withNamespaceName(testStorage.getNamespaceName()).withTopicName(testStorage.getTopicName()).withMessageCount(MESSAGE_COUNT).build();
    resourceManager.createResource(extensionContext, clients.producerScramShaTlsStrimzi(kafkaClusterSourceName), clients.consumerScramShaTlsStrimzi(kafkaClusterSourceName));
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    clients = new KafkaClientsBuilder(clients).withTopicName(sourceMirroredTopicName).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterTargetName)).withUserName(kafkaUserTargetName).build();
    LOGGER.info("Now messages should be mirrored to target topic and cluster");
    resourceManager.createResource(extensionContext, clients.consumerScramShaTlsStrimzi(kafkaClusterTargetName));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Messages successfully mirrored");
    String kmm2DeploymentName = KafkaMirrorMaker2Resources.deploymentName(testStorage.getClusterName());
    Map<String, String> mmSnapshot = DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), kmm2DeploymentName);
    LOGGER.info("Changing KafkaUser sha-password on KMM2 Source and make sure it rolled");
    Secret passwordSource = new SecretBuilder().withNewMetadata().withName(kafkaUserSourceName).endMetadata().addToData("password", "c291cmNlLXBhc3N3b3Jk").build();
    kubeClient().patchSecret(testStorage.getNamespaceName(), kafkaUserSourceName, passwordSource);
    mmSnapshot = DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), kmm2DeploymentName, 1, mmSnapshot);
    LOGGER.info("Changing KafkaUser sha-password on KMM2 Target");
    Secret passwordTarget = new SecretBuilder().withNewMetadata().withName(kafkaUserTargetName).endMetadata().addToData("password", "dGFyZ2V0LXBhc3N3b3Jk").build();
    kubeClient().patchSecret(testStorage.getNamespaceName(), kafkaUserTargetName, passwordTarget);
    DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), kmm2DeploymentName, 1, mmSnapshot);
    clients = new KafkaClientsBuilder(clients).withTopicName(testStorage.getTopicName()).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterSourceName)).withUserName(kafkaUserSourceName).build();
    resourceManager.createResource(extensionContext, clients.producerScramShaTlsStrimzi(kafkaClusterSourceName));
    ClientUtils.waitForClientSuccess(testStorage.getProducerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    clients = new KafkaClientsBuilder(clients).withTopicName(sourceMirroredTopicName).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterTargetName)).withUserName(kafkaUserTargetName).build();
    LOGGER.info("Now messages should be mirrored to target topic and cluster");
    resourceManager.createResource(extensionContext, clients.consumerScramShaTlsStrimzi(kafkaClusterTargetName));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Messages successfully mirrored");
}
Also used : KafkaListenerAuthenticationScramSha512(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationScramSha512) LabelSelector(io.fabric8.kubernetes.api.model.LabelSelector) KafkaResource(io.strimzi.systemtest.resources.crd.KafkaResource) KubeClusterResource.cmdKubeClient(io.strimzi.test.k8s.KubeClusterResource.cmdKubeClient) KafkaTopicUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaTopicUtils) KafkaListenerAuthenticationTls(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationTls) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) KafkaResources(io.strimzi.api.kafka.model.KafkaResources) Ca(io.strimzi.operator.cluster.model.Ca) BeforeAll(org.junit.jupiter.api.BeforeAll) Map(java.util.Map) Tag(org.junit.jupiter.api.Tag) StUtils(io.strimzi.systemtest.utils.StUtils) BeforeAllOnce(io.strimzi.systemtest.BeforeAllOnce) RollingUpdateUtils(io.strimzi.systemtest.utils.RollingUpdateUtils) ACCEPTANCE(io.strimzi.systemtest.Constants.ACCEPTANCE) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) IsolatedSuite(io.strimzi.systemtest.annotations.IsolatedSuite) KafkaClients(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients) KafkaMirrorMaker2Templates(io.strimzi.systemtest.templates.crd.KafkaMirrorMaker2Templates) INTERNAL_CLIENTS_USED(io.strimzi.systemtest.Constants.INTERNAL_CLIENTS_USED) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) PasswordSecretSource(io.strimzi.api.kafka.model.PasswordSecretSource) ClientUtils(io.strimzi.systemtest.utils.ClientUtils) KafkaMirrorMaker2Resources(io.strimzi.api.kafka.model.KafkaMirrorMaker2Resources) KafkaMirrorMaker2(io.strimzi.api.kafka.model.KafkaMirrorMaker2) KafkaMirrorMaker2Status(io.strimzi.api.kafka.model.status.KafkaMirrorMaker2Status) List(java.util.List) Labels(io.strimzi.operator.common.model.Labels) Logger(org.apache.logging.log4j.Logger) KafkaTopicTemplates(io.strimzi.systemtest.templates.crd.KafkaTopicTemplates) KafkaMirrorMaker2Utils(io.strimzi.systemtest.utils.kafkaUtils.KafkaMirrorMaker2Utils) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Secret(io.fabric8.kubernetes.api.model.Secret) Matchers.is(org.hamcrest.Matchers.is) KafkaTopicResource(io.strimzi.systemtest.resources.crd.KafkaTopicResource) SCALABILITY(io.strimzi.systemtest.Constants.SCALABILITY) Matchers.containsString(org.hamcrest.Matchers.containsString) Assertions.assertDoesNotThrow(org.junit.jupiter.api.Assertions.assertDoesNotThrow) AbstractST(io.strimzi.systemtest.AbstractST) Environment(io.strimzi.systemtest.Environment) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) CoreMatchers.equalTo(org.hamcrest.CoreMatchers.equalTo) CoreMatchers.not(org.hamcrest.CoreMatchers.not) HashMap(java.util.HashMap) ExtensionContext(org.junit.jupiter.api.extension.ExtensionContext) TestStorage(io.strimzi.systemtest.storage.TestStorage) SecretUtils(io.strimzi.systemtest.utils.kubeUtils.objects.SecretUtils) PodUtils(io.strimzi.systemtest.utils.kubeUtils.objects.PodUtils) JsonPathMatchers.hasJsonPath(org.valid4j.matchers.jsonpath.JsonPathMatchers.hasJsonPath) DeploymentStrategy(io.strimzi.api.kafka.model.template.DeploymentStrategy) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) TestUtils(io.strimzi.test.TestUtils) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) KafkaListenerAuthenticationScramSha512(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationScramSha512) MIRROR_MAKER2(io.strimzi.systemtest.Constants.MIRROR_MAKER2) KafkaTemplates(io.strimzi.systemtest.templates.crd.KafkaTemplates) Ready(io.strimzi.systemtest.enums.CustomResourceStatus.Ready) KafkaUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaUtils) Constants(io.strimzi.systemtest.Constants) KafkaClientsBuilder(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest) SetupClusterOperator(io.strimzi.systemtest.resources.operator.SetupClusterOperator) KafkaCmdClient(io.strimzi.systemtest.cli.KafkaCmdClient) KubeClusterResource.kubeClient(io.strimzi.test.k8s.KubeClusterResource.kubeClient) KafkaTopic(io.strimzi.api.kafka.model.KafkaTopic) DeploymentUtils(io.strimzi.systemtest.utils.kubeUtils.controllers.DeploymentUtils) Matchers.hasItem(org.hamcrest.Matchers.hasItem) KafkaListenerType(io.strimzi.api.kafka.model.listener.arraylistener.KafkaListenerType) KafkaUserTemplates(io.strimzi.systemtest.templates.crd.KafkaUserTemplates) CONNECT_COMPONENTS(io.strimzi.systemtest.Constants.CONNECT_COMPONENTS) KafkaMirrorMaker2Resource(io.strimzi.systemtest.resources.crd.KafkaMirrorMaker2Resource) SecretBuilder(io.fabric8.kubernetes.api.model.SecretBuilder) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) REGRESSION(io.strimzi.systemtest.Constants.REGRESSION) KafkaClientsBuilder(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder) Secret(io.fabric8.kubernetes.api.model.Secret) SecretBuilder(io.fabric8.kubernetes.api.model.SecretBuilder) KafkaClients(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) PasswordSecretSource(io.strimzi.api.kafka.model.PasswordSecretSource) TestStorage(io.strimzi.systemtest.storage.TestStorage) Matchers.containsString(org.hamcrest.Matchers.containsString) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)

Example 99 with KRaftNotSupported

use of io.strimzi.systemtest.annotations.KRaftNotSupported in project strimzi by strimzi.

the class MirrorMakerIsolatedST method testMirrorMakerTlsAuthenticated.

/**
 * Test mirroring messages by Mirror Maker over tls transport using mutual tls auth
 */
@ParallelNamespaceTest
@Tag(ACCEPTANCE)
@KRaftNotSupported("UserOperator is not supported by KRaft mode and is used in this test case")
@SuppressWarnings({ "checkstyle:MethodLength" })
void testMirrorMakerTlsAuthenticated(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext, clusterOperator.getDeploymentNamespace());
    String kafkaClusterSourceName = testStorage.getClusterName() + "-source";
    String kafkaClusterTargetName = testStorage.getClusterName() + "-target";
    String kafkaSourceUserName = testStorage.getUserName() + "-source";
    String kafkaTargetUserName = testStorage.getUserName() + "-target";
    // Deploy source kafka with tls listener and mutual tls auth
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterSourceName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationTls()).build()).endKafka().endSpec().build());
    // Deploy target kafka with tls listener and mutual tls auth
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterTargetName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationTls()).build()).endKafka().endSpec().build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, testStorage.getTopicName()).build(), KafkaUserTemplates.tlsUser(kafkaClusterSourceName, kafkaSourceUserName).build(), KafkaUserTemplates.tlsUser(kafkaClusterTargetName, kafkaTargetUserName).build());
    // Initialize CertSecretSource with certificate and secret names for consumer
    CertSecretSource certSecretSource = new CertSecretSource();
    certSecretSource.setCertificate("ca.crt");
    certSecretSource.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterSourceName));
    // Initialize CertSecretSource with certificate and secret names for producer
    CertSecretSource certSecretTarget = new CertSecretSource();
    certSecretTarget.setCertificate("ca.crt");
    certSecretTarget.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterTargetName));
    KafkaClients clients = new KafkaClientsBuilder().withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterSourceName)).withNamespaceName(testStorage.getNamespaceName()).withUserName(kafkaSourceUserName).withTopicName(testStorage.getTopicName()).withMessageCount(MESSAGE_COUNT).build();
    resourceManager.createResource(extensionContext, clients.producerTlsStrimzi(kafkaClusterSourceName), clients.consumerTlsStrimzi(kafkaClusterSourceName));
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    // Deploy Mirror Maker with tls listener and mutual tls auth
    resourceManager.createResource(extensionContext, KafkaMirrorMakerTemplates.kafkaMirrorMaker(testStorage.getClusterName(), kafkaClusterSourceName, kafkaClusterTargetName, ClientUtils.generateRandomConsumerGroup(), 1, true).editSpec().editConsumer().withNewTls().withTrustedCertificates(certSecretSource).endTls().withNewKafkaClientAuthenticationTls().withNewCertificateAndKey().withSecretName(kafkaSourceUserName).withCertificate("user.crt").withKey("user.key").endCertificateAndKey().endKafkaClientAuthenticationTls().endConsumer().editProducer().withNewTls().withTrustedCertificates(certSecretTarget).endTls().withNewKafkaClientAuthenticationTls().withNewCertificateAndKey().withSecretName(kafkaTargetUserName).withCertificate("user.crt").withKey("user.key").endCertificateAndKey().endKafkaClientAuthenticationTls().endProducer().endSpec().build());
    clients = new KafkaClientsBuilder(clients).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(kafkaClusterTargetName)).withUserName(kafkaTargetUserName).build();
    resourceManager.createResource(extensionContext, clients.consumerTlsStrimzi(kafkaClusterTargetName));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
}
Also used : KafkaClientsBuilder(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder) KafkaListenerAuthenticationTls(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationTls) KafkaClients(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) TestStorage(io.strimzi.systemtest.storage.TestStorage) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest) Tag(org.junit.jupiter.api.Tag)

Example 100 with KRaftNotSupported

use of io.strimzi.systemtest.annotations.KRaftNotSupported in project strimzi by strimzi.

the class ReconciliationST method testPauseReconciliationInKafkaRebalanceAndTopic.

@ParallelNamespaceTest
@Tag(CRUISE_CONTROL)
@KRaftNotSupported("TopicOperator is not supported by KRaft mode and is used in this test class")
void testPauseReconciliationInKafkaRebalanceAndTopic(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(clusterOperator.getDeploymentNamespace(), extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaWithCruiseControl(clusterName, 3, 1).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName).build());
    LOGGER.info("Adding pause annotation into KafkaTopic resource and changing replication factor");
    KafkaTopicResource.replaceTopicResourceInSpecificNamespace(topicName, topic -> {
        topic.getMetadata().setAnnotations(PAUSE_ANNO);
        topic.getSpec().setPartitions(SCALE_TO);
    }, namespaceName);
    KafkaTopicUtils.waitForKafkaTopicStatus(namespaceName, topicName, CustomResourceStatus.ReconciliationPaused);
    KafkaTopicUtils.waitForKafkaTopicSpecStability(namespaceName, topicName, KafkaResources.kafkaPodName(clusterName, 0), KafkaResources.plainBootstrapAddress(clusterName));
    LOGGER.info("Setting annotation to \"false\", partitions should be scaled to {}", SCALE_TO);
    KafkaTopicResource.replaceTopicResourceInSpecificNamespace(topicName, topic -> topic.getMetadata().getAnnotations().replace(Annotations.ANNO_STRIMZI_IO_PAUSE_RECONCILIATION, "true", "false"), namespaceName);
    KafkaTopicUtils.waitForKafkaTopicPartitionChange(namespaceName, topicName, SCALE_TO);
    resourceManager.createResource(extensionContext, KafkaRebalanceTemplates.kafkaRebalance(clusterName).build());
    LOGGER.info("Waiting for {}, then add pause and rebalance annotation, rebalancing should not be triggered", KafkaRebalanceState.ProposalReady);
    KafkaRebalanceUtils.waitForKafkaRebalanceCustomResourceState(namespaceName, clusterName, KafkaRebalanceState.ProposalReady);
    KafkaRebalanceResource.replaceKafkaRebalanceResourceInSpecificNamespace(clusterName, rebalance -> rebalance.getMetadata().setAnnotations(PAUSE_ANNO), namespaceName);
    KafkaRebalanceUtils.waitForKafkaRebalanceCustomResourceState(namespaceName, clusterName, KafkaRebalanceState.ReconciliationPaused);
    KafkaRebalanceUtils.annotateKafkaRebalanceResource(new Reconciliation("test", KafkaRebalance.RESOURCE_KIND, namespaceName, clusterName), namespaceName, clusterName, KafkaRebalanceAnnotation.approve);
    // unfortunately we don't have any option to check, if something is changed when reconciliations are paused
    // so we will check stability of status
    KafkaRebalanceUtils.waitForRebalanceStatusStability(new Reconciliation("test", KafkaRebalance.RESOURCE_KIND, namespaceName, clusterName), namespaceName, clusterName);
    LOGGER.info("Setting annotation to \"false\" and waiting for KafkaRebalance to be in {} state", KafkaRebalanceState.Ready);
    KafkaRebalanceResource.replaceKafkaRebalanceResourceInSpecificNamespace(clusterName, rebalance -> rebalance.getMetadata().getAnnotations().replace(Annotations.ANNO_STRIMZI_IO_PAUSE_RECONCILIATION, "true", "false"), namespaceName);
    KafkaRebalanceUtils.waitForKafkaRebalanceCustomResourceState(namespaceName, clusterName, KafkaRebalanceState.ProposalReady);
    // because approve annotation wasn't reflected, approving again
    KafkaRebalanceUtils.annotateKafkaRebalanceResource(new Reconciliation("test", KafkaRebalance.RESOURCE_KIND, namespaceName, clusterName), namespaceName, clusterName, KafkaRebalanceAnnotation.approve);
    KafkaRebalanceUtils.waitForKafkaRebalanceCustomResourceState(namespaceName, clusterName, KafkaRebalanceState.Ready);
}
Also used : Reconciliation(io.strimzi.operator.common.Reconciliation) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest) Tag(org.junit.jupiter.api.Tag)

Aggregations

KRaftNotSupported (io.strimzi.systemtest.annotations.KRaftNotSupported)162 ParallelNamespaceTest (io.strimzi.systemtest.annotations.ParallelNamespaceTest)138 Tag (org.junit.jupiter.api.Tag)100 KafkaClientsBuilder (io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder)94 TestStorage (io.strimzi.systemtest.storage.TestStorage)94 KafkaClients (io.strimzi.systemtest.kafkaclients.internalClients.KafkaClients)86 GenericKafkaListenerBuilder (io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder)80 Matchers.containsString (org.hamcrest.Matchers.containsString)74 CoreMatchers.containsString (org.hamcrest.CoreMatchers.containsString)44 LabelSelector (io.fabric8.kubernetes.api.model.LabelSelector)36 ExternalKafkaClient (io.strimzi.systemtest.kafkaclients.externalClients.ExternalKafkaClient)32 Secret (io.fabric8.kubernetes.api.model.Secret)30 HashMap (java.util.HashMap)30 KafkaListenerAuthenticationTls (io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationTls)28 SecretBuilder (io.fabric8.kubernetes.api.model.SecretBuilder)24 KafkaResources (io.strimzi.api.kafka.model.KafkaResources)24 AbstractST (io.strimzi.systemtest.AbstractST)24 Constants (io.strimzi.systemtest.Constants)24 REGRESSION (io.strimzi.systemtest.Constants.REGRESSION)24 KafkaResource (io.strimzi.systemtest.resources.crd.KafkaResource)24