Example 1 with SCALABILITY
use of io.strimzi.systemtest.Constants.SCALABILITY in project strimzi-kafka-operator by strimzi.
the class ConnectIsolatedST method testScaleConnectWithConnectorToZero.
@ParallelNamespaceTest
@Tag(SCALABILITY)
@Tag(CONNECTOR_OPERATOR)
void testScaleConnectWithConnectorToZero(ExtensionContext extensionContext) {
final String namespaceName = StUtils.getNamespaceBasedOnRbac(INFRA_NAMESPACE, extensionContext);
final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(false, kafkaClientsName).build());
resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(clusterName, 3).build());
resourceManager.createResource(extensionContext, KafkaConnectTemplates.kafkaConnect(extensionContext, clusterName, 2).editMetadata().addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().build());
resourceManager.createResource(extensionContext, KafkaConnectorTemplates.kafkaConnector(clusterName).editSpec().withClassName("org.apache.kafka.connect.file.FileStreamSinkConnector").addToConfig("file", Constants.DEFAULT_SINK_FILE_PATH).addToConfig("key.converter", "org.apache.kafka.connect.storage.StringConverter").addToConfig("value.converter", "org.apache.kafka.connect.storage.StringConverter").addToConfig("topics", topicName).endSpec().build());
String connectDeploymentName = KafkaConnectResources.deploymentName(clusterName);
List<Pod> connectPods = kubeClient(namespaceName).listPodsByPrefixInName(namespaceName, KafkaConnectResources.deploymentName(clusterName));
assertThat(connectPods.size(), is(2));
// scale down
LOGGER.info("Scaling KafkaConnect down to zero");
KafkaConnectResource.replaceKafkaConnectResourceInSpecificNamespace(clusterName, kafkaConnect -> kafkaConnect.getSpec().setReplicas(0), namespaceName);
KafkaConnectUtils.waitForConnectReady(namespaceName, clusterName);
PodUtils.waitForPodsReady(namespaceName, kubeClient(namespaceName).getDeploymentSelectors(namespaceName, connectDeploymentName), 0, true);
connectPods = kubeClient(namespaceName).listPodsByPrefixInName(namespaceName, connectDeploymentName);
KafkaConnectStatus connectStatus = KafkaConnectResource.kafkaConnectClient().inNamespace(namespaceName).withName(clusterName).get().getStatus();
KafkaConnectorStatus connectorStatus = KafkaConnectorResource.kafkaConnectorClient().inNamespace(namespaceName).withName(clusterName).get().getStatus();
assertThat(connectPods.size(), is(0));
assertThat(connectStatus.getConditions().get(0).getType(), is(Ready.toString()));
assertThat(connectorStatus.getConditions().stream().anyMatch(condition -> condition.getType().equals(NotReady.toString())), is(true));
assertThat(connectorStatus.getConditions().stream().anyMatch(condition -> condition.getMessage().contains("has 0 replicas")), is(true));
}
Also used :
KafkaConnectStatus(io.strimzi.api.kafka.model.status.KafkaConnectStatus)
Quantity(io.fabric8.kubernetes.api.model.Quantity)
SMOKE(io.strimzi.systemtest.Constants.SMOKE)
ConfigMapVolumeSourceBuilder(io.fabric8.kubernetes.api.model.ConfigMapVolumeSourceBuilder)
KubeClusterResource.cmdKubeClient(io.strimzi.test.k8s.KubeClusterResource.cmdKubeClient)
Matchers.not(org.hamcrest.Matchers.not)
Annotations(io.strimzi.operator.common.Annotations)
INFRA_NAMESPACE(io.strimzi.systemtest.Constants.INFRA_NAMESPACE)
KafkaUser(io.strimzi.api.kafka.model.KafkaUser)
KafkaConnectResource(io.strimzi.systemtest.resources.crd.KafkaConnectResource)
KafkaConnector(io.strimzi.api.kafka.model.KafkaConnector)
SecurityProtocol(org.apache.kafka.common.security.auth.SecurityProtocol)
KafkaListenerAuthenticationTls(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationTls)
KafkaResources(io.strimzi.api.kafka.model.KafkaResources)
KafkaConnectUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaConnectUtils)
KafkaConnectStatus(io.strimzi.api.kafka.model.status.KafkaConnectStatus)
Map(java.util.Map)
JsonObject(io.vertx.core.json.JsonObject)
Tag(org.junit.jupiter.api.Tag)
StUtils(io.strimzi.systemtest.utils.StUtils)
SecretVolumeSourceBuilder(io.fabric8.kubernetes.api.model.SecretVolumeSourceBuilder)
KafkaConnect(io.strimzi.api.kafka.model.KafkaConnect)
ACCEPTANCE(io.strimzi.systemtest.Constants.ACCEPTANCE)
IsolatedSuite(io.strimzi.systemtest.annotations.IsolatedSuite)
INTERNAL_CLIENTS_USED(io.strimzi.systemtest.Constants.INTERNAL_CLIENTS_USED)
GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder)
Base64(java.util.Base64)
List(java.util.List)
CertSecretSourceBuilder(io.strimzi.api.kafka.model.CertSecretSourceBuilder)
Logger(org.apache.logging.log4j.Logger)
KafkaTopicTemplates(io.strimzi.systemtest.templates.crd.KafkaTopicTemplates)
KafkaConnectorStatus(io.strimzi.api.kafka.model.status.KafkaConnectorStatus)
KafkaConnectorTemplates(io.strimzi.systemtest.templates.crd.KafkaConnectorTemplates)
Matchers.equalTo(org.hamcrest.Matchers.equalTo)
PasswordSecretSourceBuilder(io.strimzi.api.kafka.model.PasswordSecretSourceBuilder)
Secret(io.fabric8.kubernetes.api.model.Secret)
Matchers.is(org.hamcrest.Matchers.is)
SCALABILITY(io.strimzi.systemtest.Constants.SCALABILITY)
Matchers.containsString(org.hamcrest.Matchers.containsString)
AbstractST(io.strimzi.systemtest.AbstractST)
Environment(io.strimzi.systemtest.Environment)
Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows)
KafkaClientsTemplates(io.strimzi.systemtest.templates.crd.KafkaClientsTemplates)
KafkaConnectTemplates(io.strimzi.systemtest.templates.crd.KafkaConnectTemplates)
CONNECT(io.strimzi.systemtest.Constants.CONNECT)
ResourceRequirementsBuilder(io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder)
HashMap(java.util.HashMap)
Crds(io.strimzi.api.kafka.Crds)
ExternalKafkaClient(io.strimzi.systemtest.kafkaclients.externalClients.ExternalKafkaClient)
ExtensionContext(org.junit.jupiter.api.extension.ExtensionContext)
LinkedHashMap(java.util.LinkedHashMap)
KafkaConnectorResource(io.strimzi.systemtest.resources.crd.KafkaConnectorResource)
PodUtils(io.strimzi.systemtest.utils.kubeUtils.objects.PodUtils)
JsonPathMatchers.hasJsonPath(org.valid4j.matchers.jsonpath.JsonPathMatchers.hasJsonPath)
DeploymentStrategy(io.strimzi.api.kafka.model.template.DeploymentStrategy)
NotReady(io.strimzi.systemtest.enums.CustomResourceStatus.NotReady)
TestUtils(io.strimzi.test.TestUtils)
MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat)
KafkaListenerAuthenticationScramSha512(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationScramSha512)
NODEPORT_SUPPORTED(io.strimzi.systemtest.Constants.NODEPORT_SUPPORTED)
KafkaTemplates(io.strimzi.systemtest.templates.crd.KafkaTemplates)
Ready(io.strimzi.systemtest.enums.CustomResourceStatus.Ready)
Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo)
InternalKafkaClient(io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient)
Constants(io.strimzi.systemtest.Constants)
Pod(io.fabric8.kubernetes.api.model.Pod)
ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)
EXTERNAL_CLIENTS_USED(io.strimzi.systemtest.Constants.EXTERNAL_CLIENTS_USED)
ConfigMap(io.fabric8.kubernetes.api.model.ConfigMap)
KubeClusterResource.kubeClient(io.strimzi.test.k8s.KubeClusterResource.kubeClient)
ConfigMapBuilder(io.fabric8.kubernetes.api.model.ConfigMapBuilder)
SecretKeySelectorBuilder(io.fabric8.kubernetes.api.model.SecretKeySelectorBuilder)
KafkaConnectorUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaConnectorUtils)
DeploymentUtils(io.strimzi.systemtest.utils.kubeUtils.controllers.DeploymentUtils)
Matchers.hasItem(org.hamcrest.Matchers.hasItem)
KafkaListenerType(io.strimzi.api.kafka.model.listener.arraylistener.KafkaListenerType)
KafkaUserTemplates(io.strimzi.systemtest.templates.crd.KafkaUserTemplates)
CONNECTOR_OPERATOR(io.strimzi.systemtest.Constants.CONNECTOR_OPERATOR)
CONNECT_COMPONENTS(io.strimzi.systemtest.Constants.CONNECT_COMPONENTS)
ConfigMapKeySelectorBuilder(io.fabric8.kubernetes.api.model.ConfigMapKeySelectorBuilder)
SecretBuilder(io.fabric8.kubernetes.api.model.SecretBuilder)
LogManager(org.apache.logging.log4j.LogManager)
Collections(java.util.Collections)
REGRESSION(io.strimzi.systemtest.Constants.REGRESSION)
KafkaConnectResources(io.strimzi.api.kafka.model.KafkaConnectResources)
Pod(io.fabric8.kubernetes.api.model.Pod)
Matchers.containsString(org.hamcrest.Matchers.containsString)
KafkaConnectorStatus(io.strimzi.api.kafka.model.status.KafkaConnectorStatus)
ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)
Tag(org.junit.jupiter.api.Tag)
Example 2 with SCALABILITY
use of io.strimzi.systemtest.Constants.SCALABILITY in project strimzi-kafka-operator by strimzi.
the class RollingUpdateST method testKafkaAndZookeeperScaleUpScaleDown.
@ParallelNamespaceTest
@Tag(ACCEPTANCE)
@Tag(SCALABILITY)
void testKafkaAndZookeeperScaleUpScaleDown(ExtensionContext extensionContext) {
final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
final String userName = mapWithTestUsers.get(extensionContext.getDisplayName());
final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
final LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, KafkaResources.kafkaStatefulSetName(clusterName));
final LabelSelector zkSelector = KafkaResource.getLabelSelector(clusterName, KafkaResources.zookeeperStatefulSetName(clusterName));
resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(clusterName, 3, 3).editSpec().editKafka().addToConfig(singletonMap("default.replication.factor", 1)).addToConfig("auto.create.topics.enable", "false").endKafka().endSpec().build());
Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
KafkaUser user = KafkaUserTemplates.tlsUser(namespaceName, clusterName, userName).build();
resourceManager.createResource(extensionContext, user);
testDockerImagesForKafkaCluster(clusterName, clusterOperator.getDeploymentNamespace(), namespaceName, 3, 1, false);
// kafka cluster already deployed
LOGGER.info("Running kafkaScaleUpScaleDown {}", clusterName);
final int initialReplicas = kubeClient(namespaceName).getClient().pods().inNamespace(namespaceName).withLabelSelector(kafkaSelector).list().getItems().size();
assertEquals(3, initialReplicas);
resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName, 3, initialReplicas, initialReplicas).build());
resourceManager.createResource(extensionContext, false, KafkaClientsTemplates.kafkaClients(true, kafkaClientsName, user).build());
final String defaultKafkaClientsPodName = PodUtils.getPodsByPrefixInNameWithDynamicWait(namespaceName, kafkaClientsName).get(0).getMetadata().getName();
InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(defaultKafkaClientsPodName).withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withKafkaUsername(userName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
internalKafkaClient.produceAndConsumesTlsMessagesUntilBothOperationsAreSuccessful();
// scale up
final int scaleTo = initialReplicas + 4;
LOGGER.info("Scale up Kafka to {}", scaleTo);
// Create snapshot of current cluster
String kafkaStsName = kafkaStatefulSetName(clusterName);
KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, kafka -> {
kafka.getSpec().getKafka().setReplicas(scaleTo);
}, namespaceName);
kafkaPods = RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, scaleTo, kafkaPods);
LOGGER.info("Kafka scale up to {} finished", scaleTo);
internalKafkaClient = internalKafkaClient.toBuilder().withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
internalKafkaClient.consumesTlsMessagesUntilOperationIsSuccessful(MESSAGE_COUNT);
assertThat((int) kubeClient(namespaceName).listPersistentVolumeClaims().stream().filter(pvc -> pvc.getMetadata().getName().contains(KafkaResources.kafkaStatefulSetName(clusterName))).count(), is(scaleTo));
final int zookeeperScaleTo = initialReplicas + 2;
LOGGER.info("Scale up Zookeeper to {}", zookeeperScaleTo);
KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, k -> k.getSpec().getZookeeper().setReplicas(zookeeperScaleTo), namespaceName);
RollingUpdateUtils.waitForComponentAndPodsReady(namespaceName, zkSelector, zookeeperScaleTo);
LOGGER.info("Kafka scale up to {} finished", zookeeperScaleTo);
internalKafkaClient = internalKafkaClient.toBuilder().withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
internalKafkaClient.consumesTlsMessagesUntilOperationIsSuccessful(MESSAGE_COUNT);
// scale down
LOGGER.info("Scale down Kafka to {}", initialReplicas);
KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, k -> k.getSpec().getKafka().setReplicas(initialReplicas), namespaceName);
RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, initialReplicas, kafkaPods);
LOGGER.info("Kafka scale down to {} finished", initialReplicas);
internalKafkaClient = internalKafkaClient.toBuilder().withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
internalKafkaClient.consumesTlsMessagesUntilOperationIsSuccessful(MESSAGE_COUNT);
assertThat(kubeClient(namespaceName).listPersistentVolumeClaims(namespaceName, clusterName).stream().filter(pvc -> pvc.getMetadata().getName().contains("data-" + KafkaResources.kafkaStatefulSetName(clusterName))).collect(Collectors.toList()).size(), is(initialReplicas));
// Create new topic to ensure, that ZK is working properly
String newTopicName = KafkaTopicUtils.generateRandomNameOfTopic();
resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, newTopicName).build());
internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(newTopicName).withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
internalKafkaClient.produceAndConsumesTlsMessagesUntilBothOperationsAreSuccessful();
}
Also used :
Quantity(io.fabric8.kubernetes.api.model.Quantity)
CoreMatchers.is(org.hamcrest.CoreMatchers.is)
LabelSelector(io.fabric8.kubernetes.api.model.LabelSelector)
KafkaResource(io.strimzi.systemtest.resources.crd.KafkaResource)
KafkaTopicUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaTopicUtils)
INFRA_NAMESPACE(io.strimzi.systemtest.Constants.INFRA_NAMESPACE)
KafkaUser(io.strimzi.api.kafka.model.KafkaUser)
KafkaResources.kafkaStatefulSetName(io.strimzi.api.kafka.model.KafkaResources.kafkaStatefulSetName)
MetricsCollector(io.strimzi.systemtest.metrics.MetricsCollector)
ConfigMapKeySelector(io.fabric8.kubernetes.api.model.ConfigMapKeySelector)
KafkaResources(io.strimzi.api.kafka.model.KafkaResources)
Map(java.util.Map)
Killing(io.strimzi.systemtest.k8s.Events.Killing)
Tag(org.junit.jupiter.api.Tag)
StUtils(io.strimzi.systemtest.utils.StUtils)
Matchers.hasAllOfReasons(io.strimzi.systemtest.matchers.Matchers.hasAllOfReasons)
ProbeBuilder(io.strimzi.api.kafka.model.ProbeBuilder)
RollingUpdateUtils(io.strimzi.systemtest.utils.RollingUpdateUtils)
ACCEPTANCE(io.strimzi.systemtest.Constants.ACCEPTANCE)
INTERNAL_CLIENTS_USED(io.strimzi.systemtest.Constants.INTERNAL_CLIENTS_USED)
Collectors(java.util.stream.Collectors)
ClientUtils(io.strimzi.systemtest.utils.ClientUtils)
List(java.util.List)
Logger(org.apache.logging.log4j.Logger)
KafkaTopicTemplates(io.strimzi.systemtest.templates.crd.KafkaTopicTemplates)
Pattern(java.util.regex.Pattern)
SCALABILITY(io.strimzi.systemtest.Constants.SCALABILITY)
AbstractST(io.strimzi.systemtest.AbstractST)
KafkaClientsTemplates(io.strimzi.systemtest.templates.crd.KafkaClientsTemplates)
ExternalLoggingBuilder(io.strimzi.api.kafka.model.ExternalLoggingBuilder)
ParallelSuite(io.strimzi.systemtest.annotations.ParallelSuite)
ResourceRequirementsBuilder(io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder)
HashMap(java.util.HashMap)
ExtensionContext(org.junit.jupiter.api.extension.ExtensionContext)
ArrayList(java.util.ArrayList)
JmxPrometheusExporterMetrics(io.strimzi.api.kafka.model.JmxPrometheusExporterMetrics)
PodUtils(io.strimzi.systemtest.utils.kubeUtils.objects.PodUtils)
JmxPrometheusExporterMetricsBuilder(io.strimzi.api.kafka.model.JmxPrometheusExporterMetricsBuilder)
YAMLFactory(com.fasterxml.jackson.dataformat.yaml.YAMLFactory)
TestUtils(io.strimzi.test.TestUtils)
Collections.singletonMap(java.util.Collections.singletonMap)
MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat)
Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals)
ROLLING_UPDATE(io.strimzi.systemtest.Constants.ROLLING_UPDATE)
KafkaTemplates(io.strimzi.systemtest.templates.crd.KafkaTemplates)
KafkaUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaUtils)
InternalKafkaClient(io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient)
ComponentType(io.strimzi.systemtest.resources.ComponentType)
ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper)
Constants(io.strimzi.systemtest.Constants)
JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException)
ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)
ConfigMap(io.fabric8.kubernetes.api.model.ConfigMap)
KubeClusterResource.kubeClient(io.strimzi.test.k8s.KubeClusterResource.kubeClient)
ConfigMapBuilder(io.fabric8.kubernetes.api.model.ConfigMapBuilder)
IsolatedTest(io.strimzi.systemtest.annotations.IsolatedTest)
ResourceManager(io.strimzi.systemtest.resources.ResourceManager)
KafkaUserTemplates(io.strimzi.systemtest.templates.crd.KafkaUserTemplates)
ConfigMapKeySelectorBuilder(io.fabric8.kubernetes.api.model.ConfigMapKeySelectorBuilder)
LogManager(org.apache.logging.log4j.LogManager)
Collections(java.util.Collections)
REGRESSION(io.strimzi.systemtest.Constants.REGRESSION)
ProbeBuilder(io.strimzi.api.kafka.model.ProbeBuilder)
ExternalLoggingBuilder(io.strimzi.api.kafka.model.ExternalLoggingBuilder)
ResourceRequirementsBuilder(io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder)
JmxPrometheusExporterMetricsBuilder(io.strimzi.api.kafka.model.JmxPrometheusExporterMetricsBuilder)
ConfigMapBuilder(io.fabric8.kubernetes.api.model.ConfigMapBuilder)
ConfigMapKeySelectorBuilder(io.fabric8.kubernetes.api.model.ConfigMapKeySelectorBuilder)
LabelSelector(io.fabric8.kubernetes.api.model.LabelSelector)
InternalKafkaClient(io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient)
KafkaUser(io.strimzi.api.kafka.model.KafkaUser)
ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)
Tag(org.junit.jupiter.api.Tag)
Example 3 with SCALABILITY
use of io.strimzi.systemtest.Constants.SCALABILITY in project strimzi by strimzi.
the class RollingUpdateST method testKafkaAndZookeeperScaleUpScaleDown.
@ParallelNamespaceTest
@Tag(ACCEPTANCE)
@Tag(SCALABILITY)
void testKafkaAndZookeeperScaleUpScaleDown(ExtensionContext extensionContext) {
final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
final String userName = mapWithTestUsers.get(extensionContext.getDisplayName());
final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
final LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, KafkaResources.kafkaStatefulSetName(clusterName));
final LabelSelector zkSelector = KafkaResource.getLabelSelector(clusterName, KafkaResources.zookeeperStatefulSetName(clusterName));
resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(clusterName, 3, 3).editSpec().editKafka().addToConfig(singletonMap("default.replication.factor", 1)).addToConfig("auto.create.topics.enable", "false").endKafka().endSpec().build());
Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
KafkaUser user = KafkaUserTemplates.tlsUser(namespaceName, clusterName, userName).build();
resourceManager.createResource(extensionContext, user);
testDockerImagesForKafkaCluster(clusterName, clusterOperator.getDeploymentNamespace(), namespaceName, 3, 1, false);
// kafka cluster already deployed
LOGGER.info("Running kafkaScaleUpScaleDown {}", clusterName);
final int initialReplicas = kubeClient(namespaceName).getClient().pods().inNamespace(namespaceName).withLabelSelector(kafkaSelector).list().getItems().size();
assertEquals(3, initialReplicas);
resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName, 3, initialReplicas, initialReplicas).build());
resourceManager.createResource(extensionContext, false, KafkaClientsTemplates.kafkaClients(true, kafkaClientsName, user).build());
final String defaultKafkaClientsPodName = PodUtils.getPodsByPrefixInNameWithDynamicWait(namespaceName, kafkaClientsName).get(0).getMetadata().getName();
InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(defaultKafkaClientsPodName).withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withKafkaUsername(userName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
internalKafkaClient.produceAndConsumesTlsMessagesUntilBothOperationsAreSuccessful();
// scale up
final int scaleTo = initialReplicas + 4;
LOGGER.info("Scale up Kafka to {}", scaleTo);
// Create snapshot of current cluster
String kafkaStsName = kafkaStatefulSetName(clusterName);
KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, kafka -> {
kafka.getSpec().getKafka().setReplicas(scaleTo);
}, namespaceName);
kafkaPods = RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, scaleTo, kafkaPods);
LOGGER.info("Kafka scale up to {} finished", scaleTo);
internalKafkaClient = internalKafkaClient.toBuilder().withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
internalKafkaClient.consumesTlsMessagesUntilOperationIsSuccessful(MESSAGE_COUNT);
assertThat((int) kubeClient(namespaceName).listPersistentVolumeClaims().stream().filter(pvc -> pvc.getMetadata().getName().contains(KafkaResources.kafkaStatefulSetName(clusterName))).count(), is(scaleTo));
final int zookeeperScaleTo = initialReplicas + 2;
LOGGER.info("Scale up Zookeeper to {}", zookeeperScaleTo);
KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, k -> k.getSpec().getZookeeper().setReplicas(zookeeperScaleTo), namespaceName);
RollingUpdateUtils.waitForComponentAndPodsReady(namespaceName, zkSelector, zookeeperScaleTo);
LOGGER.info("Kafka scale up to {} finished", zookeeperScaleTo);
internalKafkaClient = internalKafkaClient.toBuilder().withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
internalKafkaClient.consumesTlsMessagesUntilOperationIsSuccessful(MESSAGE_COUNT);
// scale down
LOGGER.info("Scale down Kafka to {}", initialReplicas);
KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, k -> k.getSpec().getKafka().setReplicas(initialReplicas), namespaceName);
RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, initialReplicas, kafkaPods);
LOGGER.info("Kafka scale down to {} finished", initialReplicas);
internalKafkaClient = internalKafkaClient.toBuilder().withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
internalKafkaClient.consumesTlsMessagesUntilOperationIsSuccessful(MESSAGE_COUNT);
assertThat(kubeClient(namespaceName).listPersistentVolumeClaims(namespaceName, clusterName).stream().filter(pvc -> pvc.getMetadata().getName().contains("data-" + KafkaResources.kafkaStatefulSetName(clusterName))).collect(Collectors.toList()).size(), is(initialReplicas));
// Create new topic to ensure, that ZK is working properly
String newTopicName = KafkaTopicUtils.generateRandomNameOfTopic();
resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, newTopicName).build());
internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(newTopicName).withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
internalKafkaClient.produceAndConsumesTlsMessagesUntilBothOperationsAreSuccessful();
}
Also used :
Quantity(io.fabric8.kubernetes.api.model.Quantity)
CoreMatchers.is(org.hamcrest.CoreMatchers.is)
LabelSelector(io.fabric8.kubernetes.api.model.LabelSelector)
KafkaResource(io.strimzi.systemtest.resources.crd.KafkaResource)
KafkaTopicUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaTopicUtils)
INFRA_NAMESPACE(io.strimzi.systemtest.Constants.INFRA_NAMESPACE)
KafkaUser(io.strimzi.api.kafka.model.KafkaUser)
KafkaResources.kafkaStatefulSetName(io.strimzi.api.kafka.model.KafkaResources.kafkaStatefulSetName)
MetricsCollector(io.strimzi.systemtest.metrics.MetricsCollector)
ConfigMapKeySelector(io.fabric8.kubernetes.api.model.ConfigMapKeySelector)
KafkaResources(io.strimzi.api.kafka.model.KafkaResources)
Map(java.util.Map)
Killing(io.strimzi.systemtest.k8s.Events.Killing)
Tag(org.junit.jupiter.api.Tag)
StUtils(io.strimzi.systemtest.utils.StUtils)
Matchers.hasAllOfReasons(io.strimzi.systemtest.matchers.Matchers.hasAllOfReasons)
ProbeBuilder(io.strimzi.api.kafka.model.ProbeBuilder)
RollingUpdateUtils(io.strimzi.systemtest.utils.RollingUpdateUtils)
ACCEPTANCE(io.strimzi.systemtest.Constants.ACCEPTANCE)
INTERNAL_CLIENTS_USED(io.strimzi.systemtest.Constants.INTERNAL_CLIENTS_USED)
Collectors(java.util.stream.Collectors)
ClientUtils(io.strimzi.systemtest.utils.ClientUtils)
List(java.util.List)
Logger(org.apache.logging.log4j.Logger)
KafkaTopicTemplates(io.strimzi.systemtest.templates.crd.KafkaTopicTemplates)
Pattern(java.util.regex.Pattern)
SCALABILITY(io.strimzi.systemtest.Constants.SCALABILITY)
AbstractST(io.strimzi.systemtest.AbstractST)
KafkaClientsTemplates(io.strimzi.systemtest.templates.crd.KafkaClientsTemplates)
ExternalLoggingBuilder(io.strimzi.api.kafka.model.ExternalLoggingBuilder)
ParallelSuite(io.strimzi.systemtest.annotations.ParallelSuite)
ResourceRequirementsBuilder(io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder)
HashMap(java.util.HashMap)
ExtensionContext(org.junit.jupiter.api.extension.ExtensionContext)
ArrayList(java.util.ArrayList)
JmxPrometheusExporterMetrics(io.strimzi.api.kafka.model.JmxPrometheusExporterMetrics)
PodUtils(io.strimzi.systemtest.utils.kubeUtils.objects.PodUtils)
JmxPrometheusExporterMetricsBuilder(io.strimzi.api.kafka.model.JmxPrometheusExporterMetricsBuilder)
YAMLFactory(com.fasterxml.jackson.dataformat.yaml.YAMLFactory)
TestUtils(io.strimzi.test.TestUtils)
Collections.singletonMap(java.util.Collections.singletonMap)
MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat)
Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals)
ROLLING_UPDATE(io.strimzi.systemtest.Constants.ROLLING_UPDATE)
KafkaTemplates(io.strimzi.systemtest.templates.crd.KafkaTemplates)
KafkaUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaUtils)
InternalKafkaClient(io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient)
ComponentType(io.strimzi.systemtest.resources.ComponentType)
ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper)
Constants(io.strimzi.systemtest.Constants)
JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException)
ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)
ConfigMap(io.fabric8.kubernetes.api.model.ConfigMap)
KubeClusterResource.kubeClient(io.strimzi.test.k8s.KubeClusterResource.kubeClient)
ConfigMapBuilder(io.fabric8.kubernetes.api.model.ConfigMapBuilder)
IsolatedTest(io.strimzi.systemtest.annotations.IsolatedTest)
ResourceManager(io.strimzi.systemtest.resources.ResourceManager)
KafkaUserTemplates(io.strimzi.systemtest.templates.crd.KafkaUserTemplates)
ConfigMapKeySelectorBuilder(io.fabric8.kubernetes.api.model.ConfigMapKeySelectorBuilder)
LogManager(org.apache.logging.log4j.LogManager)
Collections(java.util.Collections)
REGRESSION(io.strimzi.systemtest.Constants.REGRESSION)
ProbeBuilder(io.strimzi.api.kafka.model.ProbeBuilder)
ExternalLoggingBuilder(io.strimzi.api.kafka.model.ExternalLoggingBuilder)
ResourceRequirementsBuilder(io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder)
JmxPrometheusExporterMetricsBuilder(io.strimzi.api.kafka.model.JmxPrometheusExporterMetricsBuilder)
ConfigMapBuilder(io.fabric8.kubernetes.api.model.ConfigMapBuilder)
ConfigMapKeySelectorBuilder(io.fabric8.kubernetes.api.model.ConfigMapKeySelectorBuilder)
LabelSelector(io.fabric8.kubernetes.api.model.LabelSelector)
InternalKafkaClient(io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient)
KafkaUser(io.strimzi.api.kafka.model.KafkaUser)
ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)
Tag(org.junit.jupiter.api.Tag)
Example 4 with SCALABILITY
use of io.strimzi.systemtest.Constants.SCALABILITY in project strimzi by strimzi.
the class ConnectIsolatedST method testScaleConnectWithConnectorToZero.
@ParallelNamespaceTest
@Tag(SCALABILITY)
@Tag(CONNECTOR_OPERATOR)
void testScaleConnectWithConnectorToZero(ExtensionContext extensionContext) {
final String namespaceName = StUtils.getNamespaceBasedOnRbac(INFRA_NAMESPACE, extensionContext);
final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(false, kafkaClientsName).build());
resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(clusterName, 3).build());
resourceManager.createResource(extensionContext, KafkaConnectTemplates.kafkaConnect(extensionContext, clusterName, 2).editMetadata().addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().build());
resourceManager.createResource(extensionContext, KafkaConnectorTemplates.kafkaConnector(clusterName).editSpec().withClassName("org.apache.kafka.connect.file.FileStreamSinkConnector").addToConfig("file", Constants.DEFAULT_SINK_FILE_PATH).addToConfig("key.converter", "org.apache.kafka.connect.storage.StringConverter").addToConfig("value.converter", "org.apache.kafka.connect.storage.StringConverter").addToConfig("topics", topicName).endSpec().build());
String connectDeploymentName = KafkaConnectResources.deploymentName(clusterName);
List<Pod> connectPods = kubeClient(namespaceName).listPodsByPrefixInName(namespaceName, KafkaConnectResources.deploymentName(clusterName));
assertThat(connectPods.size(), is(2));
// scale down
LOGGER.info("Scaling KafkaConnect down to zero");
KafkaConnectResource.replaceKafkaConnectResourceInSpecificNamespace(clusterName, kafkaConnect -> kafkaConnect.getSpec().setReplicas(0), namespaceName);
KafkaConnectUtils.waitForConnectReady(namespaceName, clusterName);
PodUtils.waitForPodsReady(namespaceName, kubeClient(namespaceName).getDeploymentSelectors(namespaceName, connectDeploymentName), 0, true);
connectPods = kubeClient(namespaceName).listPodsByPrefixInName(namespaceName, connectDeploymentName);
KafkaConnectStatus connectStatus = KafkaConnectResource.kafkaConnectClient().inNamespace(namespaceName).withName(clusterName).get().getStatus();
KafkaConnectorStatus connectorStatus = KafkaConnectorResource.kafkaConnectorClient().inNamespace(namespaceName).withName(clusterName).get().getStatus();
assertThat(connectPods.size(), is(0));
assertThat(connectStatus.getConditions().get(0).getType(), is(Ready.toString()));
assertThat(connectorStatus.getConditions().stream().anyMatch(condition -> condition.getType().equals(NotReady.toString())), is(true));
assertThat(connectorStatus.getConditions().stream().anyMatch(condition -> condition.getMessage().contains("has 0 replicas")), is(true));
}
Also used :
KafkaConnectStatus(io.strimzi.api.kafka.model.status.KafkaConnectStatus)
Quantity(io.fabric8.kubernetes.api.model.Quantity)
SMOKE(io.strimzi.systemtest.Constants.SMOKE)
ConfigMapVolumeSourceBuilder(io.fabric8.kubernetes.api.model.ConfigMapVolumeSourceBuilder)
KubeClusterResource.cmdKubeClient(io.strimzi.test.k8s.KubeClusterResource.cmdKubeClient)
Matchers.not(org.hamcrest.Matchers.not)
Annotations(io.strimzi.operator.common.Annotations)
INFRA_NAMESPACE(io.strimzi.systemtest.Constants.INFRA_NAMESPACE)
KafkaUser(io.strimzi.api.kafka.model.KafkaUser)
KafkaConnectResource(io.strimzi.systemtest.resources.crd.KafkaConnectResource)
KafkaConnector(io.strimzi.api.kafka.model.KafkaConnector)
SecurityProtocol(org.apache.kafka.common.security.auth.SecurityProtocol)
KafkaListenerAuthenticationTls(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationTls)
KafkaResources(io.strimzi.api.kafka.model.KafkaResources)
KafkaConnectUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaConnectUtils)
KafkaConnectStatus(io.strimzi.api.kafka.model.status.KafkaConnectStatus)
Map(java.util.Map)
JsonObject(io.vertx.core.json.JsonObject)
Tag(org.junit.jupiter.api.Tag)
StUtils(io.strimzi.systemtest.utils.StUtils)
SecretVolumeSourceBuilder(io.fabric8.kubernetes.api.model.SecretVolumeSourceBuilder)
KafkaConnect(io.strimzi.api.kafka.model.KafkaConnect)
ACCEPTANCE(io.strimzi.systemtest.Constants.ACCEPTANCE)
IsolatedSuite(io.strimzi.systemtest.annotations.IsolatedSuite)
INTERNAL_CLIENTS_USED(io.strimzi.systemtest.Constants.INTERNAL_CLIENTS_USED)
GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder)
Base64(java.util.Base64)
List(java.util.List)
CertSecretSourceBuilder(io.strimzi.api.kafka.model.CertSecretSourceBuilder)
Logger(org.apache.logging.log4j.Logger)
KafkaTopicTemplates(io.strimzi.systemtest.templates.crd.KafkaTopicTemplates)
KafkaConnectorStatus(io.strimzi.api.kafka.model.status.KafkaConnectorStatus)
KafkaConnectorTemplates(io.strimzi.systemtest.templates.crd.KafkaConnectorTemplates)
Matchers.equalTo(org.hamcrest.Matchers.equalTo)
PasswordSecretSourceBuilder(io.strimzi.api.kafka.model.PasswordSecretSourceBuilder)
Secret(io.fabric8.kubernetes.api.model.Secret)
Matchers.is(org.hamcrest.Matchers.is)
SCALABILITY(io.strimzi.systemtest.Constants.SCALABILITY)
Matchers.containsString(org.hamcrest.Matchers.containsString)
AbstractST(io.strimzi.systemtest.AbstractST)
Environment(io.strimzi.systemtest.Environment)
Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows)
KafkaClientsTemplates(io.strimzi.systemtest.templates.crd.KafkaClientsTemplates)
KafkaConnectTemplates(io.strimzi.systemtest.templates.crd.KafkaConnectTemplates)
CONNECT(io.strimzi.systemtest.Constants.CONNECT)
ResourceRequirementsBuilder(io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder)
HashMap(java.util.HashMap)
Crds(io.strimzi.api.kafka.Crds)
ExternalKafkaClient(io.strimzi.systemtest.kafkaclients.externalClients.ExternalKafkaClient)
ExtensionContext(org.junit.jupiter.api.extension.ExtensionContext)
LinkedHashMap(java.util.LinkedHashMap)
KafkaConnectorResource(io.strimzi.systemtest.resources.crd.KafkaConnectorResource)
PodUtils(io.strimzi.systemtest.utils.kubeUtils.objects.PodUtils)
JsonPathMatchers.hasJsonPath(org.valid4j.matchers.jsonpath.JsonPathMatchers.hasJsonPath)
DeploymentStrategy(io.strimzi.api.kafka.model.template.DeploymentStrategy)
NotReady(io.strimzi.systemtest.enums.CustomResourceStatus.NotReady)
TestUtils(io.strimzi.test.TestUtils)
MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat)
KafkaListenerAuthenticationScramSha512(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationScramSha512)
NODEPORT_SUPPORTED(io.strimzi.systemtest.Constants.NODEPORT_SUPPORTED)
KafkaTemplates(io.strimzi.systemtest.templates.crd.KafkaTemplates)
Ready(io.strimzi.systemtest.enums.CustomResourceStatus.Ready)
Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo)
InternalKafkaClient(io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient)
Constants(io.strimzi.systemtest.Constants)
Pod(io.fabric8.kubernetes.api.model.Pod)
ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)
EXTERNAL_CLIENTS_USED(io.strimzi.systemtest.Constants.EXTERNAL_CLIENTS_USED)
ConfigMap(io.fabric8.kubernetes.api.model.ConfigMap)
KubeClusterResource.kubeClient(io.strimzi.test.k8s.KubeClusterResource.kubeClient)
ConfigMapBuilder(io.fabric8.kubernetes.api.model.ConfigMapBuilder)
SecretKeySelectorBuilder(io.fabric8.kubernetes.api.model.SecretKeySelectorBuilder)
KafkaConnectorUtils(io.strimzi.systemtest.utils.kafkaUtils.KafkaConnectorUtils)
DeploymentUtils(io.strimzi.systemtest.utils.kubeUtils.controllers.DeploymentUtils)
Matchers.hasItem(org.hamcrest.Matchers.hasItem)
KafkaListenerType(io.strimzi.api.kafka.model.listener.arraylistener.KafkaListenerType)
KafkaUserTemplates(io.strimzi.systemtest.templates.crd.KafkaUserTemplates)
CONNECTOR_OPERATOR(io.strimzi.systemtest.Constants.CONNECTOR_OPERATOR)
CONNECT_COMPONENTS(io.strimzi.systemtest.Constants.CONNECT_COMPONENTS)
ConfigMapKeySelectorBuilder(io.fabric8.kubernetes.api.model.ConfigMapKeySelectorBuilder)
SecretBuilder(io.fabric8.kubernetes.api.model.SecretBuilder)
LogManager(org.apache.logging.log4j.LogManager)
Collections(java.util.Collections)
REGRESSION(io.strimzi.systemtest.Constants.REGRESSION)
KafkaConnectResources(io.strimzi.api.kafka.model.KafkaConnectResources)
Pod(io.fabric8.kubernetes.api.model.Pod)
Matchers.containsString(org.hamcrest.Matchers.containsString)
KafkaConnectorStatus(io.strimzi.api.kafka.model.status.KafkaConnectorStatus)
ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)
Tag(org.junit.jupiter.api.Tag)
Aggregations
ConfigMap (io.fabric8.kubernetes.api.model.ConfigMap)4
ConfigMapBuilder (io.fabric8.kubernetes.api.model.ConfigMapBuilder)4
ConfigMapKeySelectorBuilder (io.fabric8.kubernetes.api.model.ConfigMapKeySelectorBuilder)4
Quantity (io.fabric8.kubernetes.api.model.Quantity)4
ResourceRequirementsBuilder (io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder)4
KafkaResources (io.strimzi.api.kafka.model.KafkaResources)4
KafkaUser (io.strimzi.api.kafka.model.KafkaUser)4
AbstractST (io.strimzi.systemtest.AbstractST)4
Constants (io.strimzi.systemtest.Constants)4
ACCEPTANCE (io.strimzi.systemtest.Constants.ACCEPTANCE)4
INFRA_NAMESPACE (io.strimzi.systemtest.Constants.INFRA_NAMESPACE)4
INTERNAL_CLIENTS_USED (io.strimzi.systemtest.Constants.INTERNAL_CLIENTS_USED)4
REGRESSION (io.strimzi.systemtest.Constants.REGRESSION)4
SCALABILITY (io.strimzi.systemtest.Constants.SCALABILITY)4
ParallelNamespaceTest (io.strimzi.systemtest.annotations.ParallelNamespaceTest)4
InternalKafkaClient (io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient)4
KafkaClientsTemplates (io.strimzi.systemtest.templates.crd.KafkaClientsTemplates)4
KafkaTemplates (io.strimzi.systemtest.templates.crd.KafkaTemplates)4
KafkaTopicTemplates (io.strimzi.systemtest.templates.crd.KafkaTopicTemplates)4
KafkaUserTemplates (io.strimzi.systemtest.templates.crd.KafkaUserTemplates)4
