Examples with KafkaConnect io.strimzi.api.kafka.model.KafkaConnect used on opensource projects

Example 66 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi by strimzi.

the class KafkaConnectBuildAssemblyOperatorOpenShiftTest method testUpdateWithoutRebuildOnOpenShift.

@Test
public void testUpdateWithoutRebuildOnOpenShift(VertxTestContext context) {
    Plugin plugin1 = new PluginBuilder().withName("plugin1").withArtifacts(new JarArtifactBuilder().withUrl("https://my-domain.tld/my.jar").build()).build();
    KafkaConnect kc = new KafkaConnectBuilder().withNewMetadata().withName(NAME).withNamespace(NAMESPACE).endMetadata().withNewSpec().withReplicas(1).withBootstrapServers("my-cluster-kafka-bootstrap:9092").withNewBuild().withNewDockerOutput().withImage(OUTPUT_IMAGE).withPushSecret("my-docker-credentials").endDockerOutput().withPlugins(plugin1).endBuild().endSpec().build();
    KafkaConnectCluster connect = KafkaConnectCluster.fromCrd(Reconciliation.DUMMY_RECONCILIATION, kc, VERSIONS);
    KafkaConnectBuild build = KafkaConnectBuild.fromCrd(Reconciliation.DUMMY_RECONCILIATION, kc, VERSIONS);
    // Prepare and get mocks
    ResourceOperatorSupplier supplier = ResourceUtils.supplierWithMocks(true);
    CrdOperator mockConnectOps = supplier.connectOperator;
    DeploymentOperator mockDepOps = supplier.deploymentOperations;
    PodDisruptionBudgetOperator mockPdbOps = supplier.podDisruptionBudgetOperator;
    PodDisruptionBudgetV1Beta1Operator mockPdbOpsV1Beta1 = supplier.podDisruptionBudgetV1Beta1Operator;
    ConfigMapOperator mockCmOps = supplier.configMapOperations;
    ServiceOperator mockServiceOps = supplier.serviceOperations;
    NetworkPolicyOperator mockNetPolOps = supplier.networkPolicyOperator;
    PodOperator mockPodOps = supplier.podOperations;
    BuildConfigOperator mockBcOps = supplier.buildConfigOperations;
    BuildOperator mockBuildOps = supplier.buildOperations;
    SecretOperator mockSecretOps = supplier.secretOperations;
    CrdOperator<KubernetesClient, KafkaConnector, KafkaConnectorList> mockConnectorOps = supplier.kafkaConnectorOperator;
    // Mock KafkaConnector ops
    when(mockConnectorOps.listAsync(anyString(), any(Optional.class))).thenReturn(Future.succeededFuture(emptyList()));
    // Mock KafkaConnect ops
    when(mockConnectOps.get(NAMESPACE, NAME)).thenReturn(kc);
    when(mockConnectOps.getAsync(anyString(), anyString())).thenReturn(Future.succeededFuture(kc));
    // Mock and capture service ops
    ArgumentCaptor<Service> serviceCaptor = ArgumentCaptor.forClass(Service.class);
    when(mockServiceOps.reconcile(any(), anyString(), anyString(), serviceCaptor.capture())).thenReturn(Future.succeededFuture());
    // Mock and capture deployment ops
    ArgumentCaptor<Deployment> depCaptor = ArgumentCaptor.forClass(Deployment.class);
    when(mockDepOps.reconcile(any(), anyString(), anyString(), depCaptor.capture())).thenReturn(Future.succeededFuture());
    when(mockDepOps.getAsync(eq(NAMESPACE), eq(KafkaConnectResources.deploymentName(NAME)))).thenAnswer(inv -> {
        Deployment dep = connect.generateDeployment(emptyMap(), false, null, null);
        dep.getSpec().getTemplate().getMetadata().getAnnotations().put(Annotations.STRIMZI_IO_CONNECT_BUILD_REVISION, build.generateDockerfile().hashStub() + OUTPUT_IMAGE_HASH_STUB);
        dep.getSpec().getTemplate().getSpec().getContainers().get(0).setImage("my-connect-build@sha256:blablabla");
        return Future.succeededFuture(dep);
    });
    when(mockDepOps.scaleUp(any(), anyString(), anyString(), anyInt())).thenReturn(Future.succeededFuture(42));
    when(mockDepOps.scaleDown(any(), anyString(), anyString(), anyInt())).thenReturn(Future.succeededFuture(42));
    when(mockDepOps.readiness(any(), anyString(), anyString(), anyLong(), anyLong())).thenReturn(Future.succeededFuture());
    when(mockDepOps.waitForObserved(any(), anyString(), anyString(), anyLong(), anyLong())).thenReturn(Future.succeededFuture());
    when(mockSecretOps.reconcile(any(), anyString(), anyString(), any())).thenReturn(Future.succeededFuture());
    // Mock and capture CM ops
    when(mockCmOps.reconcile(any(), anyString(), any(), any())).thenReturn(Future.succeededFuture(ReconcileResult.created(new ConfigMap())));
    // Mock and capture Pod ops
    when(mockPodOps.reconcile(any(), eq(NAMESPACE), eq(KafkaConnectResources.buildPodName(NAME)), eq(null))).thenReturn(Future.succeededFuture(ReconcileResult.noop(null)));
    // Mock and capture BuildConfig ops
    ArgumentCaptor<BuildConfig> buildConfigCaptor = ArgumentCaptor.forClass(BuildConfig.class);
    when(mockBcOps.reconcile(any(), eq(NAMESPACE), eq(KafkaConnectResources.buildConfigName(NAME)), buildConfigCaptor.capture())).thenReturn(Future.succeededFuture(ReconcileResult.noop(null)));
    // Mock and capture NP ops
    when(mockNetPolOps.reconcile(any(), eq(NAMESPACE), eq(KafkaConnectResources.deploymentName(NAME)), any())).thenReturn(Future.succeededFuture(ReconcileResult.created(new NetworkPolicy())));
    // Mock and capture PDB ops
    when(mockPdbOps.reconcile(any(), anyString(), any(), any())).thenReturn(Future.succeededFuture());
    when(mockPdbOpsV1Beta1.reconcile(any(), anyString(), any(), any())).thenReturn(Future.succeededFuture());
    // Mock and capture KafkaConnect ops for status update
    ArgumentCaptor<KafkaConnect> connectCaptor = ArgumentCaptor.forClass(KafkaConnect.class);
    when(mockConnectOps.updateStatusAsync(any(), connectCaptor.capture())).thenReturn(Future.succeededFuture());
    // Mock KafkaConnect API client
    KafkaConnectApi mockConnectClient = mock(KafkaConnectApi.class);
    // Prepare and run reconciliation
    KafkaConnectAssemblyOperator ops = new KafkaConnectAssemblyOperator(vertx, new PlatformFeaturesAvailability(true, kubernetesVersion), supplier, ResourceUtils.dummyClusterOperatorConfig(VERSIONS), x -> mockConnectClient);
    Checkpoint async = context.checkpoint();
    ops.reconcile(new Reconciliation("test-trigger", KafkaConnect.RESOURCE_KIND, NAMESPACE, NAME)).onComplete(context.succeeding(v -> context.verify(() -> {
        // Verify Deployment
        List<Deployment> capturedDeps = depCaptor.getAllValues();
        assertThat(capturedDeps, hasSize(1));
        Deployment dep = capturedDeps.get(0);
        assertThat(dep.getMetadata().getName(), is(connect.getName()));
        assertThat(dep.getSpec().getTemplate().getSpec().getContainers().get(0).getImage(), is("my-connect-build@sha256:blablabla"));
        assertThat(Annotations.stringAnnotation(dep.getSpec().getTemplate(), Annotations.STRIMZI_IO_CONNECT_BUILD_REVISION, null), is(build.generateDockerfile().hashStub() + OUTPUT_IMAGE_HASH_STUB));
        // Verify BuildConfig
        List<BuildConfig> capturedBcs = buildConfigCaptor.getAllValues();
        assertThat(capturedBcs, hasSize(0));
        // Verify status
        List<KafkaConnect> capturedConnects = connectCaptor.getAllValues();
        assertThat(capturedConnects, hasSize(1));
        KafkaConnectStatus connectStatus = capturedConnects.get(0).getStatus();
        assertThat(connectStatus.getConditions().get(0).getStatus(), is("True"));
        assertThat(connectStatus.getConditions().get(0).getType(), is("Ready"));
        async.flag();
    })));
}

Example 67 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi by strimzi.

the class KafkaConnectTemplates method kafkaConnectWithMetrics.

public static KafkaConnectBuilder kafkaConnectWithMetrics(String name, String clusterName, int kafkaConnectReplicas) {
    KafkaConnect kafkaConnect = getKafkaConnectFromYaml(Constants.PATH_TO_KAFKA_CONNECT_METRICS_CONFIG);
    ConfigMap metricsCm = TestUtils.configMapFromYaml(Constants.PATH_TO_KAFKA_CONNECT_METRICS_CONFIG, "connect-metrics");
    KubeClusterResource.kubeClient().getClient().configMaps().inNamespace(kubeClient().getNamespace()).createOrReplace(metricsCm);
    return defaultKafkaConnect(kafkaConnect, ResourceManager.kubeClient().getNamespace(), name, clusterName, kafkaConnectReplicas);
}

Example 68 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi by strimzi.

the class ConnectIsolatedST method testConfigureDeploymentStrategy.

@ParallelNamespaceTest
void testConfigureDeploymentStrategy(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(INFRA_NAMESPACE, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(false, kafkaClientsName).build());
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(clusterName, 3).build());
    resourceManager.createResource(extensionContext, KafkaConnectTemplates.kafkaConnect(extensionContext, clusterName, 1).editSpec().editOrNewTemplate().editOrNewDeployment().withDeploymentStrategy(DeploymentStrategy.RECREATE).endDeployment().endTemplate().endSpec().build());
    String connectDepName = KafkaConnectResources.deploymentName(clusterName);
    LOGGER.info("Adding label to Connect resource, the CR should be recreated");
    KafkaConnectResource.replaceKafkaConnectResourceInSpecificNamespace(clusterName, kc -> kc.getMetadata().setLabels(Collections.singletonMap("some", "label")), namespaceName);
    DeploymentUtils.waitForDeploymentAndPodsReady(namespaceName, connectDepName, 1);
    KafkaConnect kafkaConnect = KafkaConnectResource.kafkaConnectClient().inNamespace(namespaceName).withName(clusterName).get();
    LOGGER.info("Checking that observed gen. is still on 1 (recreation) and new label is present");
    assertThat(kafkaConnect.getStatus().getObservedGeneration(), is(1L));
    assertThat(kafkaConnect.getMetadata().getLabels().toString(), containsString("some=label"));
    assertThat(kafkaConnect.getSpec().getTemplate().getDeployment().getDeploymentStrategy(), is(DeploymentStrategy.RECREATE));
    LOGGER.info("Changing deployment strategy to {}", DeploymentStrategy.ROLLING_UPDATE);
    KafkaConnectResource.replaceKafkaConnectResourceInSpecificNamespace(clusterName, kc -> kc.getSpec().getTemplate().getDeployment().setDeploymentStrategy(DeploymentStrategy.ROLLING_UPDATE), namespaceName);
    KafkaConnectUtils.waitForConnectReady(namespaceName, clusterName);
    LOGGER.info("Adding another label to Connect resource, pods should be rolled");
    KafkaConnectResource.replaceKafkaConnectResourceInSpecificNamespace(clusterName, kc -> kc.getMetadata().getLabels().put("another", "label"), namespaceName);
    DeploymentUtils.waitForDeploymentAndPodsReady(namespaceName, connectDepName, 1);
    LOGGER.info("Checking that observed gen. higher (rolling update) and label is changed");
    kafkaConnect = KafkaConnectResource.kafkaConnectClient().inNamespace(namespaceName).withName(clusterName).get();
    assertThat(kafkaConnect.getStatus().getObservedGeneration(), is(2L));
    assertThat(kafkaConnect.getMetadata().getLabels().toString(), containsString("another=label"));
    assertThat(kafkaConnect.getSpec().getTemplate().getDeployment().getDeploymentStrategy(), is(DeploymentStrategy.ROLLING_UPDATE));
}

Example 69 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi by strimzi.

the class MetricsIsolatedST method testKafkaConnectResponse.

@ParallelTest
@Tag(CONNECT)
@Tag(CONNECT_COMPONENTS)
void testKafkaConnectResponse() {
    kafkaConnectMetricsData = collector.toBuilder().withComponentType(ComponentType.KafkaConnect).build().collectMetricsFromPods();
    Pattern connectResponse = Pattern.compile("kafka_connect_node_response_total\\{clientid=\".*\",.*} ([\\d.][^\\n]+)", Pattern.CASE_INSENSITIVE);
    ArrayList<Double> values = MetricsCollector.collectSpecificMetric(connectResponse, kafkaConnectMetricsData);
    assertThat("KafkaConnect response count doesn't match expected value", values.stream().mapToDouble(i -> i).sum() > 0);
}

Example 70 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi by strimzi.

the class MetricsIsolatedST method testKafkaConnectRequests.

@ParallelTest
@Tag(ACCEPTANCE)
@Tag(CONNECT)
@Tag(CONNECT_COMPONENTS)
void testKafkaConnectRequests() {
    kafkaConnectMetricsData = collector.toBuilder().withComponentType(ComponentType.KafkaConnect).build().collectMetricsFromPods();
    Pattern connectRequests = Pattern.compile("kafka_connect_node_request_total\\{clientid=\".*\",} ([\\d.][^\\n]+)", Pattern.CASE_INSENSITIVE);
    ArrayList<Double> values = MetricsCollector.collectSpecificMetric(connectRequests, kafkaConnectMetricsData);
    assertThat("KafkaConnect requests count doesn't match expected value", values.stream().mapToDouble(i -> i).sum() > 0);
}