Examples with TestStorage io.strimzi.systemtest.storage.TestStorage used on opensource projects

Example 86 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi-kafka-operator by strimzi.

the class RackAwarenessST method testKafkaRackAwareness.

@ParallelNamespaceTest
void testKafkaRackAwareness(ExtensionContext extensionContext) {
    Assumptions.assumeFalse(Environment.isNamespaceRbacScope());
    TestStorage storage = storageMap.get(extensionContext);
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(storage.getClusterName(), 1, 1).editSpec().editKafka().withNewRack(TOPOLOGY_KEY).addToConfig("replica.selector.class", "org.apache.kafka.common.replica.RackAwareReplicaSelector").endKafka().endSpec().build());
    LOGGER.info("Kafka cluster deployed successfully");
    String ssName = KafkaResources.kafkaStatefulSetName(storage.getClusterName());
    String podName = PodUtils.getPodNameByPrefix(storage.getNamespaceName(), ssName);
    Pod pod = kubeClient().getPod(storage.getNamespaceName(), podName);
    // check that spec matches the actual pod configuration
    Affinity specAffinity = StUtils.getStatefulSetOrStrimziPodSetAffinity(storage.getNamespaceName(), KafkaResources.kafkaStatefulSetName(storage.getClusterName()));
    NodeSelectorRequirement specNodeRequirement = specAffinity.getNodeAffinity().getRequiredDuringSchedulingIgnoredDuringExecution().getNodeSelectorTerms().get(0).getMatchExpressions().get(0);
    NodeAffinity podAffinity = pod.getSpec().getAffinity().getNodeAffinity();
    NodeSelectorRequirement podNodeRequirement = podAffinity.getRequiredDuringSchedulingIgnoredDuringExecution().getNodeSelectorTerms().get(0).getMatchExpressions().get(0);
    assertThat(podNodeRequirement, is(specNodeRequirement));
    assertThat(specNodeRequirement.getKey(), is(TOPOLOGY_KEY));
    assertThat(specNodeRequirement.getOperator(), is("Exists"));
    PodAffinityTerm specPodAntiAffinityTerm = specAffinity.getPodAntiAffinity().getPreferredDuringSchedulingIgnoredDuringExecution().get(0).getPodAffinityTerm();
    PodAffinityTerm podAntiAffinityTerm = pod.getSpec().getAffinity().getPodAntiAffinity().getPreferredDuringSchedulingIgnoredDuringExecution().get(0).getPodAffinityTerm();
    assertThat(podAntiAffinityTerm, is(specPodAntiAffinityTerm));
    assertThat(specPodAntiAffinityTerm.getTopologyKey(), is(TOPOLOGY_KEY));
    assertThat(specPodAntiAffinityTerm.getLabelSelector().getMatchLabels(), hasEntry("strimzi.io/cluster", storage.getClusterName()));
    assertThat(specPodAntiAffinityTerm.getLabelSelector().getMatchLabels(), hasEntry("strimzi.io/name", KafkaResources.kafkaStatefulSetName(storage.getClusterName())));
    // check Kafka rack awareness configuration
    String podNodeName = pod.getSpec().getNodeName();
    String hostname = podNodeName.contains(".") ? podNodeName.substring(0, podNodeName.indexOf(".")) : podNodeName;
    String rackIdOut = cmdKubeClient(storage.getNamespaceName()).execInPod(KafkaResources.kafkaPodName(storage.getClusterName(), 0), "/bin/bash", "-c", "cat /opt/kafka/init/rack.id").out().trim();
    String brokerRackOut = cmdKubeClient(storage.getNamespaceName()).execInPod(KafkaResources.kafkaPodName(storage.getClusterName(), 0), "/bin/bash", "-c", "cat /tmp/strimzi.properties | grep broker.rack").out().trim();
    assertThat(rackIdOut.trim(), is(hostname));
    assertThat(brokerRackOut.contains("broker.rack=" + hostname), is(true));
}

Example 87 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi-kafka-operator by strimzi.

the class TracingST method createTestResources.

@BeforeEach
void createTestResources(ExtensionContext extensionContext) {
    TestStorage testStorage = new TestStorage(extensionContext, namespace);
    storageMap.put(extensionContext, testStorage);
    deployJaegerInstance(extensionContext, storageMap.get(extensionContext).getNamespaceName());
    resourceManager.createResource(extensionContext, ScraperTemplates.scraperPod(storageMap.get(extensionContext).getNamespaceName(), storageMap.get(extensionContext).getScraperName()).build());
    testStorage.addToTestStorage(Constants.SCRAPER_POD_KEY, kubeClient().listPodsByPrefixInName(storageMap.get(extensionContext).getNamespaceName(), storageMap.get(extensionContext).getScraperName()).get(0).getMetadata().getName());
    storageMap.put(extensionContext, testStorage);
    final KafkaTracingClients kafkaTracingClient = new KafkaTracingClientsBuilder().withNamespaceName(storageMap.get(extensionContext).getNamespaceName()).withProducerName(storageMap.get(extensionContext).getProducerName()).withConsumerName(storageMap.get(extensionContext).getConsumerName()).withBootstrapAddress(KafkaResources.plainBootstrapAddress(storageMap.get(extensionContext).getClusterName())).withTopicName(storageMap.get(extensionContext).getTopicName()).withStreamsTopicTargetName(storageMap.get(extensionContext).retrieveFromTestStorage(Constants.STREAM_TOPIC_KEY).toString()).withMessageCount(MESSAGE_COUNT).withJaegerServiceProducerName(JAEGER_PRODUCER_SERVICE).withJaegerServiceConsumerName(JAEGER_CONSUMER_SERVICE).withJaegerServiceStreamsName(JAEGER_KAFKA_STREAMS_SERVICE).withJaegerServerAgentName(JAEGER_AGENT_NAME).build();
    testStorage.addToTestStorage(Constants.KAFKA_TRACING_CLIENT_KEY, kafkaTracingClient);
    storageMap.put(extensionContext, testStorage);
}

Example 88 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi-kafka-operator by strimzi.

the class AbstractNamespaceST method deployKafkaConnectorWithSink.

void deployKafkaConnectorWithSink(ExtensionContext extensionContext, String clusterName) {
    final TestStorage testStorage = new TestStorage(extensionContext, SECOND_NAMESPACE);
    // Deploy Kafka Connector
    Map<String, Object> connectorConfig = new HashMap<>();
    connectorConfig.put("topics", TOPIC_NAME);
    connectorConfig.put("file", Constants.DEFAULT_SINK_FILE_PATH);
    connectorConfig.put("key.converter", "org.apache.kafka.connect.storage.StringConverter");
    connectorConfig.put("value.converter", "org.apache.kafka.connect.storage.StringConverter");
    resourceManager.createResource(extensionContext, KafkaConnectorTemplates.kafkaConnector(clusterName).editSpec().withClassName("org.apache.kafka.connect.file.FileStreamSinkConnector").withConfig(connectorConfig).endSpec().build());
    KafkaConnectorUtils.waitForConnectorReady(clusterName);
    String kafkaConnectPodName = kubeClient().listPods(clusterName, Labels.STRIMZI_KIND_LABEL, KafkaConnect.RESOURCE_KIND).get(0).getMetadata().getName();
    KafkaConnectUtils.waitUntilKafkaConnectRestApiIsAvailable(kafkaConnectPodName);
    KafkaClients kafkaClients = new KafkaClientsBuilder().withTopicName(TOPIC_NAME).withMessageCount(MESSAGE_COUNT).withBootstrapAddress(KafkaResources.plainBootstrapAddress(SECOND_CLUSTER_NAME)).withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withNamespaceName(testStorage.getNamespaceName()).build();
    resourceManager.createResource(extensionContext, kafkaClients.producerStrimzi(), kafkaClients.consumerStrimzi());
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    KafkaConnectUtils.waitForMessagesInKafkaConnectFileSink(kafkaConnectPodName, Constants.DEFAULT_SINK_FILE_PATH, "99");
}

Example 89 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi-kafka-operator by strimzi.

the class AllNamespaceIsolatedST method testUserInDifferentNamespace.

@IsolatedTest
@KRaftNotSupported("UserOperator is not supported by KRaft mode and is used in this test case")
void testUserInDifferentNamespace(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext, SECOND_NAMESPACE);
    String startingNamespace = cluster.setNamespace(SECOND_NAMESPACE);
    KafkaUser user = KafkaUserTemplates.tlsUser(MAIN_NAMESPACE_CLUSTER_NAME, USER_NAME).build();
    resourceManager.createResource(extensionContext, user);
    Condition kafkaCondition = KafkaUserResource.kafkaUserClient().inNamespace(SECOND_NAMESPACE).withName(USER_NAME).get().getStatus().getConditions().get(0);
    LOGGER.info("KafkaUser condition status: {}", kafkaCondition.getStatus());
    LOGGER.info("KafkaUser condition type: {}", kafkaCondition.getType());
    assertThat(kafkaCondition.getType(), is(Ready.toString()));
    List<Secret> secretsOfSecondNamespace = kubeClient(SECOND_NAMESPACE).listSecrets();
    cluster.setNamespace(THIRD_NAMESPACE);
    for (Secret s : secretsOfSecondNamespace) {
        if (s.getMetadata().getName().equals(USER_NAME)) {
            LOGGER.info("Copying secret {} from namespace {} to namespace {}", s, SECOND_NAMESPACE, THIRD_NAMESPACE);
            copySecret(s, THIRD_NAMESPACE, USER_NAME);
        }
    }
    KafkaClients kafkaClients = new KafkaClientsBuilder().withTopicName(TOPIC_NAME).withMessageCount(MESSAGE_COUNT).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(MAIN_NAMESPACE_CLUSTER_NAME)).withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withNamespaceName(THIRD_NAMESPACE).withUserName(USER_NAME).build();
    resourceManager.createResource(extensionContext, kafkaClients.producerTlsStrimzi(MAIN_NAMESPACE_CLUSTER_NAME), kafkaClients.consumerTlsStrimzi(MAIN_NAMESPACE_CLUSTER_NAME));
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), THIRD_NAMESPACE, MESSAGE_COUNT);
    cluster.setNamespace(startingNamespace);
}

Example 90 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi by strimzi.

the class SecurityST method autoReplaceSomeKeysTriggeredByAnno.

@SuppressWarnings({ "checkstyle:MethodLength", "checkstyle:NPathComplexity" })
void autoReplaceSomeKeysTriggeredByAnno(ExtensionContext extensionContext, int expectedRolls, boolean zkShouldRoll, boolean kafkaShouldRoll, boolean eoShouldRoll, boolean keAndCCShouldRoll) {
    final TestStorage testStorage = new TestStorage(extensionContext, namespace);
    List<String> secrets = null;
    // to make it parallel we need decision maker...
    if (extensionContext.getTags().contains("ClusterCaKeys")) {
        secrets = Arrays.asList(clusterCaKeySecretName(testStorage.getClusterName()));
    } else if (extensionContext.getTags().contains("ClientsCaKeys")) {
        secrets = Arrays.asList(clientsCaKeySecretName(testStorage.getClusterName()));
    } else {
        // AllCaKeys
        secrets = Arrays.asList(clusterCaKeySecretName(testStorage.getClusterName()), clientsCaKeySecretName(testStorage.getClusterName()));
    }
    createKafkaCluster(extensionContext, testStorage.getClusterName());
    resourceManager.createResource(extensionContext, KafkaUserTemplates.tlsUser(testStorage.getClusterName(), testStorage.getUserName()).build(), KafkaTopicTemplates.topic(testStorage.getClusterName(), testStorage.getTopicName()).build());
    KafkaClients kafkaClients = new KafkaClientsBuilder().withTopicName(testStorage.getTopicName()).withMessageCount(MESSAGE_COUNT).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(testStorage.getClusterName())).withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withNamespaceName(testStorage.getNamespaceName()).withUserName(testStorage.getUserName()).build();
    resourceManager.createResource(extensionContext, kafkaClients.producerTlsStrimzi(testStorage.getClusterName()), kafkaClients.consumerTlsStrimzi(testStorage.getClusterName()));
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    // Get all pods, and their resource versions
    Map<String, String> zkPods = PodUtils.podSnapshot(testStorage.getNamespaceName(), testStorage.getZookeeperSelector());
    Map<String, String> kafkaPods = PodUtils.podSnapshot(testStorage.getNamespaceName(), testStorage.getKafkaSelector());
    Map<String, String> eoPod = DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(testStorage.getClusterName()));
    Map<String, String> ccPod = DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), CruiseControlResources.deploymentName(testStorage.getClusterName()));
    Map<String, String> kePod = DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), KafkaExporterResources.deploymentName(testStorage.getClusterName()));
    LOGGER.info("Triggering CA cert renewal by adding the annotation");
    Map<String, String> initialCaKeys = new HashMap<>();
    for (String secretName : secrets) {
        Secret secret = kubeClient().getSecret(testStorage.getNamespaceName(), secretName);
        String value = secret.getData().get("ca.key");
        assertThat("ca.key in " + secretName + " should not be null", value, is(Matchers.notNullValue()));
        initialCaKeys.put(secretName, value);
        Secret annotated = new SecretBuilder(secret).editMetadata().addToAnnotations(Ca.ANNO_STRIMZI_IO_FORCE_REPLACE, "true").endMetadata().build();
        LOGGER.info("Patching secret {} with {}", secretName, Ca.ANNO_STRIMZI_IO_FORCE_REPLACE);
        kubeClient().patchSecret(testStorage.getNamespaceName(), secretName, annotated);
    }
    for (int i = 1; i <= expectedRolls; i++) {
        if (zkShouldRoll) {
            LOGGER.info("Wait for zk to rolling restart ({})...", i);
            zkPods = i < expectedRolls ? RollingUpdateUtils.waitTillComponentHasRolled(testStorage.getNamespaceName(), testStorage.getZookeeperSelector(), zkPods) : RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), testStorage.getZookeeperSelector(), 3, zkPods);
        }
        if (kafkaShouldRoll) {
            LOGGER.info("Wait for kafka to rolling restart ({})...", i);
            kafkaPods = i < expectedRolls ? RollingUpdateUtils.waitTillComponentHasRolled(testStorage.getNamespaceName(), testStorage.getKafkaSelector(), kafkaPods) : RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), testStorage.getKafkaSelector(), 3, kafkaPods);
        }
        if (eoShouldRoll) {
            LOGGER.info("Wait for EO to rolling restart ({})...", i);
            eoPod = i < expectedRolls ? DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(testStorage.getClusterName()), eoPod) : DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(testStorage.getClusterName()), 1, eoPod);
        }
        if (keAndCCShouldRoll) {
            LOGGER.info("Wait for KafkaExporter and CruiseControl to rolling restart ({})...", i);
            kePod = i < expectedRolls ? DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), KafkaExporterResources.deploymentName(testStorage.getClusterName()), kePod) : DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), KafkaExporterResources.deploymentName(testStorage.getClusterName()), 1, kePod);
            ccPod = i < expectedRolls ? DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), CruiseControlResources.deploymentName(testStorage.getClusterName()), ccPod) : DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), CruiseControlResources.deploymentName(testStorage.getClusterName()), 1, ccPod);
        }
    }
    LOGGER.info("Checking the certificates have been replaced");
    for (String secretName : secrets) {
        Secret secret = kubeClient().getSecret(testStorage.getNamespaceName(), secretName);
        assertThat("Secret " + secretName + " should exist", secret, is(notNullValue()));
        assertThat("CA key in " + secretName + " should have non-null 'data'", secret.getData(), is(notNullValue()));
        String value = secret.getData().get("ca.key");
        assertThat("CA key in " + secretName + " should exist", value, is(notNullValue()));
        assertThat("CA key in " + secretName + " should have changed", value, is(not(initialCaKeys.get(secretName))));
    }
    kafkaClients = new KafkaClientsBuilder(kafkaClients).withConsumerGroup(ClientUtils.generateRandomConsumerGroup()).build();
    resourceManager.createResource(extensionContext, kafkaClients.consumerTlsStrimzi(testStorage.getClusterName()));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    // Finally check a new client (signed by new client key) can consume
    final String bobUserName = "bobik-" + testStorage.getUserName();
    resourceManager.createResource(extensionContext, KafkaUserTemplates.tlsUser(testStorage.getClusterName(), bobUserName).build());
    kafkaClients = new KafkaClientsBuilder(kafkaClients).withConsumerGroup(ClientUtils.generateRandomConsumerGroup()).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(testStorage.getClusterName())).withUserName(bobUserName).build();
    resourceManager.createResource(extensionContext, kafkaClients.consumerTlsStrimzi(testStorage.getClusterName()));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    if (!zkShouldRoll) {
        assertThat("ZK pods should not roll, but did.", PodUtils.podSnapshot(testStorage.getNamespaceName(), testStorage.getZookeeperSelector()), is(zkPods));
    }
    if (!kafkaShouldRoll) {
        assertThat("Kafka pods should not roll, but did.", PodUtils.podSnapshot(testStorage.getNamespaceName(), testStorage.getKafkaSelector()), is(kafkaPods));
    }
    if (!eoShouldRoll) {
        assertThat("EO pod should not roll, but did.", DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), KafkaResources.entityOperatorDeploymentName(testStorage.getClusterName())), is(eoPod));
    }
    if (!keAndCCShouldRoll) {
        assertThat("CC pod should not roll, but did.", DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), CruiseControlResources.deploymentName(testStorage.getClusterName())), is(ccPod));
        assertThat("KE pod should not roll, but did.", DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), KafkaExporterResources.deploymentName(testStorage.getClusterName())), is(kePod));
    }
}