Examples with TestStorage io.strimzi.systemtest.storage.TestStorage used on opensource projects

Example 46 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi-kafka-operator by strimzi.

the class ConnectBuilderIsolatedST method testBuildOtherPluginTypeWithAndWithoutFileName.

@ParallelTest
void testBuildOtherPluginTypeWithAndWithoutFileName(ExtensionContext extensionContext) {
    TestStorage storage = new TestStorage(extensionContext, clusterOperator.getDeploymentNamespace());
    final String imageName = getImageNameForTestCase();
    String topicName = KafkaTopicUtils.generateRandomNameOfTopic();
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(storage.getNamespaceName(), topicName).build());
    resourceManager.createResource(extensionContext, KafkaConnectTemplates.kafkaConnect(storage.getClusterName(), storage.getNamespaceName(), storage.getNamespaceName(), 1).editMetadata().addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().editOrNewSpec().addToConfig("key.converter.schemas.enable", false).addToConfig("value.converter.schemas.enable", false).addToConfig("key.converter", "org.apache.kafka.connect.storage.StringConverter").addToConfig("value.converter", "org.apache.kafka.connect.storage.StringConverter").withNewBuild().withPlugins(PLUGIN_WITH_OTHER_TYPE).withNewDockerOutput().withImage(imageName).endDockerOutput().endBuild().endSpec().build());
    String deploymentName = KafkaConnectResources.deploymentName(storage.getClusterName());
    Map<String, String> connectSnapshot = DeploymentUtils.depSnapshot(deploymentName);
    String connectPodName = kubeClient().listPods(storage.getClusterName(), Labels.STRIMZI_KIND_LABEL, KafkaConnect.RESOURCE_KIND).get(0).getMetadata().getName();
    LOGGER.info("Checking that plugin has correct file name: {}", ECHO_SINK_FILE_NAME);
    assertEquals(getPluginFileNameFromConnectPod(connectPodName), ECHO_SINK_FILE_NAME);
    final Plugin pluginWithoutFileName = new PluginBuilder().withName(PLUGIN_WITH_OTHER_TYPE_NAME).withArtifacts(new OtherArtifactBuilder().withUrl(ECHO_SINK_JAR_URL).withSha512sum(ECHO_SINK_JAR_CHECKSUM).build()).build();
    LOGGER.info("Removing file name from the plugin, hash should be used");
    KafkaConnectResource.replaceKafkaConnectResource(storage.getClusterName(), connect -> {
        connect.getSpec().getBuild().setPlugins(Collections.singletonList(pluginWithoutFileName));
    });
    DeploymentUtils.waitTillDepHasRolled(deploymentName, 1, connectSnapshot);
    LOGGER.info("Checking that plugin has different name than before");
    connectPodName = kubeClient().listPods(storage.getClusterName(), Labels.STRIMZI_KIND_LABEL, KafkaConnect.RESOURCE_KIND).get(0).getMetadata().getName();
    String fileName = getPluginFileNameFromConnectPod(connectPodName);
    assertNotEquals(fileName, ECHO_SINK_FILE_NAME);
    assertEquals(fileName, Util.hashStub(ECHO_SINK_JAR_URL));
}

Example 47 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi-kafka-operator by strimzi.

the class ConnectBuilderIsolatedST method testBuildFailsWithWrongChecksumOfArtifact.

@ParallelTest
void testBuildFailsWithWrongChecksumOfArtifact(ExtensionContext extensionContext) {
    TestStorage storage = new TestStorage(extensionContext, clusterOperator.getDeploymentNamespace());
    final String imageName = getImageNameForTestCase();
    Plugin pluginWithWrongChecksum = new PluginBuilder().withName("connector-with-wrong-checksum").withArtifacts(new JarArtifactBuilder().withUrl(ECHO_SINK_JAR_URL).withSha512sum(ECHO_SINK_JAR_WRONG_CHECKSUM).build()).build();
    resourceManager.createResource(extensionContext, ScraperTemplates.scraperPod(storage.getNamespaceName(), storage.getScraperName()).build());
    resourceManager.createResource(extensionContext, false, KafkaConnectTemplates.kafkaConnect(storage.getClusterName(), storage.getNamespaceName(), storage.getNamespaceName(), 1).editMetadata().addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().editOrNewSpec().withNewBuild().withPlugins(pluginWithWrongChecksum).withNewDockerOutput().withImage(imageName).endDockerOutput().endBuild().endSpec().build());
    KafkaConnectUtils.waitForConnectNotReady(storage.getClusterName());
    KafkaConnectUtils.waitUntilKafkaConnectStatusConditionContainsMessage(storage.getClusterName(), storage.getNamespaceName(), "The Kafka Connect build failed(.*)?");
    LOGGER.info("Checking if KafkaConnect status condition contains message about build failure");
    KafkaConnect kafkaConnect = KafkaConnectResource.kafkaConnectClient().inNamespace(storage.getNamespaceName()).withName(storage.getClusterName()).get();
    LOGGER.info("Deploying network policies for KafkaConnect");
    NetworkPolicyResource.deployNetworkPolicyForResource(extensionContext, kafkaConnect, KafkaConnectResources.deploymentName(storage.getClusterName()));
    Condition connectCondition = kafkaConnect.getStatus().getConditions().stream().findFirst().orElseThrow();
    assertTrue(connectCondition.getMessage().matches("The Kafka Connect build failed(.*)?"));
    assertThat(connectCondition.getType(), is(NotReady.toString()));
    LOGGER.info("Replacing plugin's checksum with right one");
    KafkaConnectResource.replaceKafkaConnectResource(storage.getClusterName(), kC -> {
        Plugin pluginWithRightChecksum = new PluginBuilder().withName("connector-with-right-checksum").withArtifacts(new JarArtifactBuilder().withUrl(ECHO_SINK_JAR_URL).withSha512sum(ECHO_SINK_JAR_CHECKSUM).build()).build();
        kC.getSpec().getBuild().getPlugins().remove(0);
        kC.getSpec().getBuild().getPlugins().add(pluginWithRightChecksum);
    });
    KafkaConnectUtils.waitForConnectReady(storage.getClusterName());
    String scraperPodName = kubeClient(storage.getNamespaceName()).listPodsByPrefixInName(storage.getScraperName()).get(0).getMetadata().getName();
    LOGGER.info("Checking if KafkaConnect API contains EchoSink connector");
    String plugins = cmdKubeClient().execInPod(scraperPodName, "curl", "-X", "GET", "http://" + KafkaConnectResources.serviceName(storage.getClusterName()) + ":8083/connector-plugins").out();
    assertTrue(plugins.contains(ECHO_SINK_CLASS_NAME));
    LOGGER.info("Checking if KafkaConnect resource contains EchoSink connector in status");
    kafkaConnect = KafkaConnectResource.kafkaConnectClient().inNamespace(storage.getNamespaceName()).withName(storage.getClusterName()).get();
    assertTrue(kafkaConnect.getStatus().getConnectorPlugins().stream().anyMatch(connectorPlugin -> connectorPlugin.getConnectorClass().contains(ECHO_SINK_CLASS_NAME)));
}

Example 48 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi-kafka-operator by strimzi.

the class ConnectIsolatedST method testSecretsWithKafkaConnectWithTlsAndScramShaAuthentication.

@KRaftNotSupported("UserOperator is not supported by KRaft mode and is used in this test class")
@ParallelNamespaceTest
@Tag(INTERNAL_CLIENTS_USED)
void testSecretsWithKafkaConnectWithTlsAndScramShaAuthentication(ExtensionContext extensionContext) {
    TestStorage storage = new TestStorage(extensionContext);
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(storage.getClusterName(), 3).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationScramSha512()).build()).endKafka().endSpec().build());
    KafkaUser kafkaUser = KafkaUserTemplates.scramShaUser(storage.getClusterName(), storage.getUserName()).build();
    resourceManager.createResource(extensionContext, kafkaUser);
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(storage.getClusterName(), storage.getTopicName()).build());
    KafkaConnect connect = KafkaConnectTemplates.kafkaConnectWithFilePlugin(storage.getNamespaceName(), storage.getClusterName(), 1).editSpec().addToConfig("key.converter.schemas.enable", false).addToConfig("value.converter.schemas.enable", false).addToConfig("key.converter", "org.apache.kafka.connect.storage.StringConverter").addToConfig("value.converter", "org.apache.kafka.connect.storage.StringConverter").withNewTls().addNewTrustedCertificate().withSecretName(storage.getClusterName() + "-cluster-ca-cert").withCertificate("ca.crt").endTrustedCertificate().endTls().withBootstrapServers(storage.getClusterName() + "-kafka-bootstrap:9093").withNewKafkaClientAuthenticationScramSha512().withUsername(storage.getUserName()).withNewPasswordSecret().withSecretName(storage.getUserName()).withPassword("password").endPasswordSecret().endKafkaClientAuthenticationScramSha512().endSpec().build();
    resourceManager.createResource(extensionContext, connect, ScraperTemplates.scraperPod(storage.getNamespaceName(), storage.getScraperName()).build());
    LOGGER.info("Deploy NetworkPolicies for KafkaConnect");
    NetworkPolicyResource.deployNetworkPolicyForResource(extensionContext, connect, KafkaConnectResources.deploymentName(storage.getClusterName()));
    final String kafkaConnectPodName = kubeClient(storage.getNamespaceName()).listPodsByPrefixInName(KafkaConnectResources.deploymentName(storage.getClusterName())).get(0).getMetadata().getName();
    final String kafkaConnectLogs = kubeClient(storage.getNamespaceName()).logs(kafkaConnectPodName);
    final String scraperPodName = kubeClient(storage.getNamespaceName()).listPodsByPrefixInName(storage.getScraperName()).get(0).getMetadata().getName();
    LOGGER.info("Verifying that KafkaConnect pod logs don't contain ERRORs");
    assertThat(kafkaConnectLogs, not(containsString("ERROR")));
    LOGGER.info("Creating FileStreamSink connector via pod {} with topic {}", scraperPodName, storage.getTopicName());
    KafkaConnectorUtils.createFileSinkConnector(storage.getNamespaceName(), scraperPodName, storage.getTopicName(), Constants.DEFAULT_SINK_FILE_PATH, KafkaConnectResources.url(storage.getClusterName(), storage.getNamespaceName(), 8083));
    KafkaClients kafkaClients = new KafkaClientsBuilder().withTopicName(storage.getTopicName()).withMessageCount(MESSAGE_COUNT).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(storage.getClusterName())).withProducerName(storage.getProducerName()).withConsumerName(storage.getConsumerName()).withNamespaceName(storage.getNamespaceName()).withUserName(storage.getUserName()).build();
    resourceManager.createResource(extensionContext, kafkaClients.producerScramShaTlsStrimzi(storage.getClusterName()), kafkaClients.consumerScramShaTlsStrimzi(storage.getClusterName()));
    ClientUtils.waitForClientsSuccess(storage.getProducerName(), storage.getConsumerName(), storage.getNamespaceName(), MESSAGE_COUNT);
    KafkaConnectUtils.waitForMessagesInKafkaConnectFileSink(storage.getNamespaceName(), kafkaConnectPodName, Constants.DEFAULT_SINK_FILE_PATH, "99");
}

Example 49 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi-kafka-operator by strimzi.

the class ConnectIsolatedST method testKafkaConnectAndPausedConnectorWithFileSinkPlugin.

@ParallelNamespaceTest
@Tag(SANITY)
@Tag(SMOKE)
@Tag(INTERNAL_CLIENTS_USED)
void testKafkaConnectAndPausedConnectorWithFileSinkPlugin(ExtensionContext extensionContext) {
    TestStorage storage = new TestStorage(extensionContext);
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(storage.getClusterName(), 3).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(storage.getClusterName(), storage.getTopicName()).build());
    resourceManager.createResource(extensionContext, KafkaConnectTemplates.kafkaConnectWithFilePlugin(storage.getNamespaceName(), storage.getClusterName(), 1).editMetadata().addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().editSpec().addToConfig("key.converter.schemas.enable", false).addToConfig("value.converter.schemas.enable", false).addToConfig("key.converter", "org.apache.kafka.connect.storage.StringConverter").addToConfig("value.converter", "org.apache.kafka.connect.storage.StringConverter").endSpec().build());
    final String kafkaConnectPodName = kubeClient(storage.getNamespaceName()).listPodsByPrefixInName(KafkaConnectResources.deploymentName(storage.getClusterName())).get(0).getMetadata().getName();
    KafkaConnectUtils.waitUntilKafkaConnectRestApiIsAvailable(storage.getNamespaceName(), kafkaConnectPodName);
    LOGGER.info("Creating KafkaConnector with 'pause: true'");
    resourceManager.createResource(extensionContext, KafkaConnectorTemplates.kafkaConnector(storage.getClusterName()).editSpec().withClassName("org.apache.kafka.connect.file.FileStreamSinkConnector").addToConfig("topics", storage.getTopicName()).addToConfig("file", Constants.DEFAULT_SINK_FILE_PATH).addToConfig("key.converter", "org.apache.kafka.connect.storage.StringConverter").addToConfig("value.converter", "org.apache.kafka.connect.storage.StringConverter").endSpec().build());
    KafkaClients kafkaClients = new KafkaClientsBuilder().withTopicName(storage.getTopicName()).withMessageCount(MESSAGE_COUNT).withBootstrapAddress(KafkaResources.plainBootstrapAddress(storage.getClusterName())).withProducerName(storage.getProducerName()).withConsumerName(storage.getConsumerName()).withNamespaceName(storage.getNamespaceName()).build();
    resourceManager.createResource(extensionContext, kafkaClients.producerStrimzi(), kafkaClients.consumerStrimzi());
    ClientUtils.waitForClientsSuccess(storage.getProducerName(), storage.getConsumerName(), storage.getNamespaceName(), MESSAGE_COUNT);
    KafkaConnectUtils.waitForMessagesInKafkaConnectFileSink(storage.getNamespaceName(), kafkaConnectPodName, Constants.DEFAULT_SINK_FILE_PATH, "99");
    LOGGER.info("Pausing KafkaConnector: {}", storage.getClusterName());
    KafkaConnectorResource.replaceKafkaConnectorResourceInSpecificNamespace(storage.getClusterName(), kafkaConnector -> kafkaConnector.getSpec().setPause(true), storage.getNamespaceName());
    KafkaConnectorUtils.waitForConnectorReady(storage.getNamespaceName(), storage.getClusterName());
    LOGGER.info("Clearing FileSink file to check if KafkaConnector will be really paused");
    KafkaConnectUtils.clearFileSinkFile(storage.getNamespaceName(), kafkaConnectPodName, Constants.DEFAULT_SINK_FILE_PATH);
    resourceManager.createResource(extensionContext, kafkaClients.producerStrimzi(), kafkaClients.consumerStrimzi());
    ClientUtils.waitForClientsSuccess(storage.getProducerName(), storage.getConsumerName(), storage.getNamespaceName(), MESSAGE_COUNT);
    LOGGER.info("Because KafkaConnector is paused, no messages should appear to FileSink file");
    assertThrows(Exception.class, () -> KafkaConnectUtils.waitForMessagesInKafkaConnectFileSink(storage.getNamespaceName(), kafkaConnectPodName, Constants.DEFAULT_SINK_FILE_PATH, "99"));
    LOGGER.info("Unpausing KafkaConnector, messages should again appear to FileSink file");
    KafkaConnectorResource.replaceKafkaConnectorResourceInSpecificNamespace(storage.getClusterName(), kafkaConnector -> kafkaConnector.getSpec().setPause(false), storage.getNamespaceName());
    KafkaConnectorUtils.waitForConnectorReady(storage.getNamespaceName(), storage.getClusterName());
    KafkaConnectUtils.waitForMessagesInKafkaConnectFileSink(storage.getNamespaceName(), kafkaConnectPodName, Constants.DEFAULT_SINK_FILE_PATH, "99");
}

Example 50 with TestStorage

use of io.strimzi.systemtest.storage.TestStorage in project strimzi-kafka-operator by strimzi.

the class LoggingChangeST method testDynamicallySetConnectLoggingLevels.

@ParallelNamespaceTest
@Tag(ROLLING_UPDATE)
@Tag(CONNECT)
@Tag(CONNECT_COMPONENTS)
void testDynamicallySetConnectLoggingLevels(ExtensionContext extensionContext) {
    TestStorage testStorage = new TestStorage(extensionContext);
    InlineLogging ilOff = new InlineLogging();
    Map<String, String> loggers = new HashMap<>();
    loggers.put("connect.root.logger.level", "OFF");
    ilOff.setLoggers(loggers);
    KafkaConnect connect = KafkaConnectTemplates.kafkaConnect(testStorage.getClusterName(), 1).editSpec().withInlineLogging(ilOff).endSpec().editMetadata().addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().build();
    resourceManager.createResource(extensionContext, false, KafkaTemplates.kafkaEphemeral(testStorage.getClusterName(), 3).build(), connect, ScraperTemplates.scraperPod(testStorage.getNamespaceName(), testStorage.getScraperName()).build());
    resourceManager.synchronizeResources(extensionContext);
    LOGGER.info("Deploy NetworkPolicies for KafkaConnect");
    NetworkPolicyResource.deployNetworkPolicyForResource(extensionContext, connect, KafkaConnectResources.deploymentName(testStorage.getClusterName()));
    String scraperPodName = PodUtils.getPodsByPrefixInNameWithDynamicWait(testStorage.getNamespaceName(), testStorage.getScraperName()).get(0).getMetadata().getName();
    Map<String, String> connectSnapshot = DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), KafkaConnectResources.deploymentName(testStorage.getClusterName()));
    final String connectPodName = connectSnapshot.keySet().iterator().next();
    LOGGER.info("Asserting if log is without records");
    assertFalse(DEFAULT_LOG4J_PATTERN.matcher(StUtils.getLogFromPodByTime(testStorage.getNamespaceName(), connectPodName, "", "30s")).find());
    LOGGER.info("Changing rootLogger level to DEBUG with inline logging");
    InlineLogging ilDebug = new InlineLogging();
    loggers.put("connect.root.logger.level", "DEBUG");
    ilDebug.setLoggers(loggers);
    KafkaConnectResource.replaceKafkaConnectResourceInSpecificNamespace(testStorage.getClusterName(), conn -> {
        conn.getSpec().setLogging(ilDebug);
    }, testStorage.getNamespaceName());
    LOGGER.info("Waiting for log4j.properties will contain desired settings");
    TestUtils.waitFor("Logger change", Constants.GLOBAL_POLL_INTERVAL, Constants.GLOBAL_TIMEOUT, () -> cmdKubeClient().namespace(testStorage.getNamespaceName()).execInPod(scraperPodName, "curl", "http://" + KafkaConnectResources.serviceName(testStorage.getClusterName()) + ":8083/admin/loggers/root").out().contains("DEBUG"));
    TestUtils.waitFor("log to not be empty", Duration.ofMillis(100).toMillis(), Constants.SAFETY_RECONCILIATION_INTERVAL, () -> {
        String kcLog = StUtils.getLogFromPodByTime(testStorage.getNamespaceName(), connectPodName, "", "30s");
        return kcLog != null && !kcLog.isEmpty() && DEFAULT_LOG4J_PATTERN.matcher(kcLog).find();
    });
    String log4jConfig = "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender\n" + "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout\n" + "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} %p %X{connector.context}%m (%c) [%t]%n\n" + "log4j.rootLogger=OFF, CONSOLE\n" + "log4j.logger.org.apache.zookeeper=ERROR\n" + "log4j.logger.org.I0Itec.zkclient=ERROR\n" + "log4j.logger.org.reflections=ERROR";
    String externalCmName = "external-cm";
    ConfigMap connectLoggingMap = new ConfigMapBuilder().withNewMetadata().addToLabels("app", "strimzi").withName(externalCmName).withNamespace(testStorage.getNamespaceName()).endMetadata().withData(Collections.singletonMap("log4j.properties", log4jConfig)).build();
    kubeClient().getClient().configMaps().inNamespace(testStorage.getNamespaceName()).createOrReplace(connectLoggingMap);
    ExternalLogging connectXternalLogging = new ExternalLoggingBuilder().withNewValueFrom().withConfigMapKeyRef(new ConfigMapKeySelectorBuilder().withName(externalCmName).withKey("log4j.properties").build()).endValueFrom().build();
    LOGGER.info("Setting log level of Connect to OFF");
    // change to the external logging
    KafkaConnectResource.replaceKafkaConnectResourceInSpecificNamespace(testStorage.getClusterName(), conn -> {
        conn.getSpec().setLogging(connectXternalLogging);
    }, testStorage.getNamespaceName());
    TestUtils.waitFor("Logger change", Constants.GLOBAL_POLL_INTERVAL, Constants.GLOBAL_TIMEOUT, () -> cmdKubeClient().namespace(testStorage.getNamespaceName()).execInPod(scraperPodName, "curl", "http://" + KafkaConnectResources.serviceName(testStorage.getClusterName()) + ":8083/admin/loggers/root").out().contains("OFF"));
    TestUtils.waitFor("log to be empty", Duration.ofMillis(100).toMillis(), Constants.SAFETY_RECONCILIATION_INTERVAL, () -> {
        String kcLog = StUtils.getLogFromPodByTime(testStorage.getNamespaceName(), connectPodName, "", "30s");
        return kcLog != null && kcLog.isEmpty() && !DEFAULT_LOG4J_PATTERN.matcher(kcLog).find();
    });
    assertThat("Connect pod should not roll", DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), KafkaConnectResources.deploymentName(testStorage.getClusterName())), equalTo(connectSnapshot));
}