Examples with CONNECT io.strimzi.systemtest.Constants.CONNECT used on opensource projects

Example 6 with CONNECT

use of io.strimzi.systemtest.Constants.CONNECT in project strimzi by strimzi.

the class MetricsIsolatedST method testKafkaConnectIoNetwork.

@ParallelTest
@Tag(CONNECT)
@Tag(CONNECT_COMPONENTS)
void testKafkaConnectIoNetwork() {
    kafkaConnectMetricsData = collector.toBuilder().withComponentType(ComponentType.KafkaConnect).build().collectMetricsFromPods();
    Pattern connectIoNetwork = Pattern.compile("kafka_connect_network_io_total\\{clientid=\".*\",} ([\\d.][^\\n]+)", Pattern.CASE_INSENSITIVE);
    ArrayList<Double> values = MetricsCollector.collectSpecificMetric(connectIoNetwork, kafkaConnectMetricsData);
    assertThat("KafkaConnect IO network count doesn't match expected value", values.stream().mapToDouble(i -> i).sum() > 0);
}

Example 7 with CONNECT

use of io.strimzi.systemtest.Constants.CONNECT in project strimzi by strimzi.

the class ConnectBuilderIsolatedST method testUpdateConnectWithAnotherPlugin.

@ParallelTest
void testUpdateConnectWithAnotherPlugin(ExtensionContext extensionContext) {
    TestStorage storage = new TestStorage(extensionContext, clusterOperator.getDeploymentNamespace());
    String echoConnector = "echo-sink-connector";
    String camelConnector = "camel-http-connector";
    final String imageName = getImageNameForTestCase();
    Plugin secondPlugin = new PluginBuilder().withName("camel-connector").withArtifacts(new TgzArtifactBuilder().withUrl(CAMEL_CONNECTOR_TGZ_URL).withSha512sum(CAMEL_CONNECTOR_TGZ_CHECKSUM).build()).build();
    String topicName = KafkaTopicUtils.generateRandomNameOfTopic();
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(storage.getNamespaceName(), topicName).build());
    KafkaConnect connect = KafkaConnectTemplates.kafkaConnect(storage.getClusterName(), storage.getNamespaceName(), storage.getNamespaceName(), 1).editMetadata().addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().editOrNewSpec().addToConfig("key.converter.schemas.enable", false).addToConfig("value.converter.schemas.enable", false).addToConfig("key.converter", "org.apache.kafka.connect.storage.StringConverter").addToConfig("value.converter", "org.apache.kafka.connect.storage.StringConverter").withNewBuild().withPlugins(PLUGIN_WITH_TAR_AND_JAR).withNewDockerOutput().withImage(imageName).endDockerOutput().endBuild().withNewInlineLogging().addToLoggers("connect.root.logger.level", "INFO").endInlineLogging().endSpec().build();
    resourceManager.createResource(extensionContext, connect, ScraperTemplates.scraperPod(storage.getNamespaceName(), storage.getScraperName()).build());
    LOGGER.info("Deploy NetworkPolicies for KafkaConnect");
    NetworkPolicyResource.deployNetworkPolicyForResource(extensionContext, connect, KafkaConnectResources.deploymentName(storage.getClusterName()));
    Map<String, Object> echoSinkConfig = new HashMap<>();
    echoSinkConfig.put("topics", topicName);
    echoSinkConfig.put("level", "INFO");
    LOGGER.info("Creating EchoSink connector");
    resourceManager.createResource(extensionContext, KafkaConnectorTemplates.kafkaConnector(echoConnector, storage.getClusterName()).editOrNewSpec().withClassName(ECHO_SINK_CLASS_NAME).withConfig(echoSinkConfig).endSpec().build());
    String deploymentName = KafkaConnectResources.deploymentName(storage.getClusterName());
    Map<String, String> connectSnapshot = DeploymentUtils.depSnapshot(deploymentName);
    String scraperPodName = kubeClient(storage.getNamespaceName()).listPodsByPrefixInName(storage.getScraperName()).get(0).getMetadata().getName();
    LOGGER.info("Checking that KafkaConnect API contains EchoSink connector and not Camel-Telegram Connector class name");
    String plugins = cmdKubeClient().execInPod(scraperPodName, "curl", "-X", "GET", "http://" + KafkaConnectResources.serviceName(storage.getClusterName()) + ":8083/connector-plugins").out();
    assertFalse(plugins.contains(CAMEL_CONNECTOR_HTTP_SINK_CLASS_NAME));
    assertTrue(plugins.contains(ECHO_SINK_CLASS_NAME));
    LOGGER.info("Adding one more connector to the KafkaConnect");
    KafkaConnectResource.replaceKafkaConnectResource(storage.getClusterName(), kafkaConnect -> {
        kafkaConnect.getSpec().getBuild().getPlugins().add(secondPlugin);
    });
    DeploymentUtils.waitTillDepHasRolled(deploymentName, 1, connectSnapshot);
    Map<String, Object> camelHttpConfig = new HashMap<>();
    camelHttpConfig.put("camel.sink.path.httpUri", "http://" + KafkaConnectResources.serviceName(storage.getClusterName()) + ":8083");
    camelHttpConfig.put("topics", topicName);
    LOGGER.info("Creating Camel-HTTP-Sink connector");
    resourceManager.createResource(extensionContext, KafkaConnectorTemplates.kafkaConnector(camelConnector, storage.getClusterName()).editOrNewSpec().withClassName(CAMEL_CONNECTOR_HTTP_SINK_CLASS_NAME).withConfig(camelHttpConfig).endSpec().build());
    KafkaConnect kafkaConnect = KafkaConnectResource.kafkaConnectClient().inNamespace(storage.getNamespaceName()).withName(storage.getClusterName()).get();
    LOGGER.info("Checking if both Connectors were created and Connect contains both plugins");
    assertThat(kafkaConnect.getSpec().getBuild().getPlugins().size(), is(2));
    assertTrue(kafkaConnect.getStatus().getConnectorPlugins().stream().anyMatch(connectorPlugin -> connectorPlugin.getConnectorClass().contains(ECHO_SINK_CLASS_NAME)));
    assertTrue(kafkaConnect.getStatus().getConnectorPlugins().stream().anyMatch(connectorPlugin -> connectorPlugin.getConnectorClass().contains(CAMEL_CONNECTOR_HTTP_SINK_CLASS_NAME)));
}

Example 8 with CONNECT

use of io.strimzi.systemtest.Constants.CONNECT in project strimzi by strimzi.

the class ConnectBuilderIsolatedST method testBuildFailsWithWrongChecksumOfArtifact.

@ParallelTest
void testBuildFailsWithWrongChecksumOfArtifact(ExtensionContext extensionContext) {
    TestStorage storage = new TestStorage(extensionContext, clusterOperator.getDeploymentNamespace());
    final String imageName = getImageNameForTestCase();
    Plugin pluginWithWrongChecksum = new PluginBuilder().withName("connector-with-wrong-checksum").withArtifacts(new JarArtifactBuilder().withUrl(ECHO_SINK_JAR_URL).withSha512sum(ECHO_SINK_JAR_WRONG_CHECKSUM).build()).build();
    resourceManager.createResource(extensionContext, ScraperTemplates.scraperPod(storage.getNamespaceName(), storage.getScraperName()).build());
    resourceManager.createResource(extensionContext, false, KafkaConnectTemplates.kafkaConnect(storage.getClusterName(), storage.getNamespaceName(), storage.getNamespaceName(), 1).editMetadata().addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().editOrNewSpec().withNewBuild().withPlugins(pluginWithWrongChecksum).withNewDockerOutput().withImage(imageName).endDockerOutput().endBuild().endSpec().build());
    KafkaConnectUtils.waitForConnectNotReady(storage.getClusterName());
    KafkaConnectUtils.waitUntilKafkaConnectStatusConditionContainsMessage(storage.getClusterName(), storage.getNamespaceName(), "The Kafka Connect build failed(.*)?");
    LOGGER.info("Checking if KafkaConnect status condition contains message about build failure");
    KafkaConnect kafkaConnect = KafkaConnectResource.kafkaConnectClient().inNamespace(storage.getNamespaceName()).withName(storage.getClusterName()).get();
    LOGGER.info("Deploying network policies for KafkaConnect");
    NetworkPolicyResource.deployNetworkPolicyForResource(extensionContext, kafkaConnect, KafkaConnectResources.deploymentName(storage.getClusterName()));
    Condition connectCondition = kafkaConnect.getStatus().getConditions().stream().findFirst().orElseThrow();
    assertTrue(connectCondition.getMessage().matches("The Kafka Connect build failed(.*)?"));
    assertThat(connectCondition.getType(), is(NotReady.toString()));
    LOGGER.info("Replacing plugin's checksum with right one");
    KafkaConnectResource.replaceKafkaConnectResource(storage.getClusterName(), kC -> {
        Plugin pluginWithRightChecksum = new PluginBuilder().withName("connector-with-right-checksum").withArtifacts(new JarArtifactBuilder().withUrl(ECHO_SINK_JAR_URL).withSha512sum(ECHO_SINK_JAR_CHECKSUM).build()).build();
        kC.getSpec().getBuild().getPlugins().remove(0);
        kC.getSpec().getBuild().getPlugins().add(pluginWithRightChecksum);
    });
    KafkaConnectUtils.waitForConnectReady(storage.getClusterName());
    String scraperPodName = kubeClient(storage.getNamespaceName()).listPodsByPrefixInName(storage.getScraperName()).get(0).getMetadata().getName();
    LOGGER.info("Checking if KafkaConnect API contains EchoSink connector");
    String plugins = cmdKubeClient().execInPod(scraperPodName, "curl", "-X", "GET", "http://" + KafkaConnectResources.serviceName(storage.getClusterName()) + ":8083/connector-plugins").out();
    assertTrue(plugins.contains(ECHO_SINK_CLASS_NAME));
    LOGGER.info("Checking if KafkaConnect resource contains EchoSink connector in status");
    kafkaConnect = KafkaConnectResource.kafkaConnectClient().inNamespace(storage.getNamespaceName()).withName(storage.getClusterName()).get();
    assertTrue(kafkaConnect.getStatus().getConnectorPlugins().stream().anyMatch(connectorPlugin -> connectorPlugin.getConnectorClass().contains(ECHO_SINK_CLASS_NAME)));
}

Example 9 with CONNECT

use of io.strimzi.systemtest.Constants.CONNECT in project strimzi-kafka-operator by strimzi.

the class MetricsIsolatedST method testKafkaConnectRequests.

@ParallelTest
@Tag(CONNECT)
@Tag(CONNECT_COMPONENTS)
void testKafkaConnectRequests() {
    kafkaConnectMetricsData = collector.toBuilder().withComponentType(ComponentType.KafkaConnect).build().collectMetricsFromPods();
    Pattern connectRequests = Pattern.compile("kafka_connect_node_request_total\\{clientid=\".*\",} ([\\d.][^\\n]+)", Pattern.CASE_INSENSITIVE);
    ArrayList<Double> values = MetricsCollector.collectSpecificMetric(connectRequests, kafkaConnectMetricsData);
    assertThat("KafkaConnect requests count doesn't match expected value", values.stream().mapToDouble(i -> i).sum() > 0);
}

Example 10 with CONNECT

use of io.strimzi.systemtest.Constants.CONNECT in project strimzi-kafka-operator by strimzi.

the class ConnectBuilderIsolatedST method testUpdateConnectWithAnotherPlugin.

@ParallelTest
void testUpdateConnectWithAnotherPlugin(ExtensionContext extensionContext) {
    TestStorage storage = new TestStorage(extensionContext, clusterOperator.getDeploymentNamespace());
    String echoConnector = "echo-sink-connector";
    String camelConnector = "camel-http-connector";
    final String imageName = getImageNameForTestCase();
    Plugin secondPlugin = new PluginBuilder().withName("camel-connector").withArtifacts(new TgzArtifactBuilder().withUrl(CAMEL_CONNECTOR_TGZ_URL).withSha512sum(CAMEL_CONNECTOR_TGZ_CHECKSUM).build()).build();
    String topicName = KafkaTopicUtils.generateRandomNameOfTopic();
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(storage.getNamespaceName(), topicName).build());
    KafkaConnect connect = KafkaConnectTemplates.kafkaConnect(storage.getClusterName(), storage.getNamespaceName(), storage.getNamespaceName(), 1).editMetadata().addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().editOrNewSpec().addToConfig("key.converter.schemas.enable", false).addToConfig("value.converter.schemas.enable", false).addToConfig("key.converter", "org.apache.kafka.connect.storage.StringConverter").addToConfig("value.converter", "org.apache.kafka.connect.storage.StringConverter").withNewBuild().withPlugins(PLUGIN_WITH_TAR_AND_JAR).withNewDockerOutput().withImage(imageName).endDockerOutput().endBuild().withNewInlineLogging().addToLoggers("connect.root.logger.level", "INFO").endInlineLogging().endSpec().build();
    resourceManager.createResource(extensionContext, connect, ScraperTemplates.scraperPod(storage.getNamespaceName(), storage.getScraperName()).build());
    LOGGER.info("Deploy NetworkPolicies for KafkaConnect");
    NetworkPolicyResource.deployNetworkPolicyForResource(extensionContext, connect, KafkaConnectResources.deploymentName(storage.getClusterName()));
    Map<String, Object> echoSinkConfig = new HashMap<>();
    echoSinkConfig.put("topics", topicName);
    echoSinkConfig.put("level", "INFO");
    LOGGER.info("Creating EchoSink connector");
    resourceManager.createResource(extensionContext, KafkaConnectorTemplates.kafkaConnector(echoConnector, storage.getClusterName()).editOrNewSpec().withClassName(ECHO_SINK_CLASS_NAME).withConfig(echoSinkConfig).endSpec().build());
    String deploymentName = KafkaConnectResources.deploymentName(storage.getClusterName());
    Map<String, String> connectSnapshot = DeploymentUtils.depSnapshot(deploymentName);
    String scraperPodName = kubeClient(storage.getNamespaceName()).listPodsByPrefixInName(storage.getScraperName()).get(0).getMetadata().getName();
    LOGGER.info("Checking that KafkaConnect API contains EchoSink connector and not Camel-Telegram Connector class name");
    String plugins = cmdKubeClient().execInPod(scraperPodName, "curl", "-X", "GET", "http://" + KafkaConnectResources.serviceName(storage.getClusterName()) + ":8083/connector-plugins").out();
    assertFalse(plugins.contains(CAMEL_CONNECTOR_HTTP_SINK_CLASS_NAME));
    assertTrue(plugins.contains(ECHO_SINK_CLASS_NAME));
    LOGGER.info("Adding one more connector to the KafkaConnect");
    KafkaConnectResource.replaceKafkaConnectResource(storage.getClusterName(), kafkaConnect -> {
        kafkaConnect.getSpec().getBuild().getPlugins().add(secondPlugin);
    });
    DeploymentUtils.waitTillDepHasRolled(deploymentName, 1, connectSnapshot);
    Map<String, Object> camelHttpConfig = new HashMap<>();
    camelHttpConfig.put("camel.sink.path.httpUri", "http://" + KafkaConnectResources.serviceName(storage.getClusterName()) + ":8083");
    camelHttpConfig.put("topics", topicName);
    LOGGER.info("Creating Camel-HTTP-Sink connector");
    resourceManager.createResource(extensionContext, KafkaConnectorTemplates.kafkaConnector(camelConnector, storage.getClusterName()).editOrNewSpec().withClassName(CAMEL_CONNECTOR_HTTP_SINK_CLASS_NAME).withConfig(camelHttpConfig).endSpec().build());
    KafkaConnect kafkaConnect = KafkaConnectResource.kafkaConnectClient().inNamespace(storage.getNamespaceName()).withName(storage.getClusterName()).get();
    LOGGER.info("Checking if both Connectors were created and Connect contains both plugins");
    assertThat(kafkaConnect.getSpec().getBuild().getPlugins().size(), is(2));
    assertTrue(kafkaConnect.getStatus().getConnectorPlugins().stream().anyMatch(connectorPlugin -> connectorPlugin.getConnectorClass().contains(ECHO_SINK_CLASS_NAME)));
    assertTrue(kafkaConnect.getStatus().getConnectorPlugins().stream().anyMatch(connectorPlugin -> connectorPlugin.getConnectorClass().contains(CAMEL_CONNECTOR_HTTP_SINK_CLASS_NAME)));
}