Examples with KafkaConnect io.strimzi.api.kafka.model.KafkaConnect used on opensource projects

Example 36 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi by strimzi.

the class KafkaConnectBuildTest method testValidKanikoOptions.

@ParallelTest
public void testValidKanikoOptions() {
    List<String> expectedArgs = new ArrayList<>(defaultArgs);
    expectedArgs.add("--reproducible");
    expectedArgs.add("--single-snapshot");
    expectedArgs.add("--log-format=json");
    KafkaConnect kc = new KafkaConnectBuilder().withNewMetadata().withName(cluster).withNamespace(namespace).endMetadata().withNewSpec().withBootstrapServers("my-kafka:9092").withNewBuild().withNewDockerOutput().withImage("my-image:latest").withPushSecret("my-docker-credentials").withAdditionalKanikoOptions("--reproducible", "--single-snapshot", "--log-format=json").endDockerOutput().withPlugins(new PluginBuilder().withName("my-connector").withArtifacts(jarArtifactWithChecksum).build(), new PluginBuilder().withName("my-connector2").withArtifacts(jarArtifactNoChecksum).build()).endBuild().endSpec().build();
    KafkaConnectBuild build = KafkaConnectBuild.fromCrd(new Reconciliation("test", kc.getKind(), kc.getMetadata().getNamespace(), kc.getMetadata().getName()), kc, VERSIONS);
    Pod pod = build.generateBuilderPod(true, ImagePullPolicy.IFNOTPRESENT, null, null);
    assertThat(pod.getSpec().getContainers().get(0).getArgs(), is(expectedArgs));
}

Example 37 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi by strimzi.

the class KafkaConnectBuildTest method testInvalidKanikoOptions.

@ParallelTest
public void testInvalidKanikoOptions() {
    KafkaConnect kc = new KafkaConnectBuilder().withNewMetadata().withName(cluster).withNamespace(namespace).endMetadata().withNewSpec().withBootstrapServers("my-kafka:9092").withNewBuild().withNewDockerOutput().withImage("my-image:latest").withPushSecret("my-docker-credentials").withAdditionalKanikoOptions("--reproducible", "--reproducible-something", "--build-arg", "--single-snapshot", "--digest-file=/dev/null", "--log-format=json").endDockerOutput().withPlugins(new PluginBuilder().withName("my-connector").withArtifacts(jarArtifactWithChecksum).build(), new PluginBuilder().withName("my-connector2").withArtifacts(jarArtifactNoChecksum).build()).endBuild().endSpec().build();
    InvalidResourceException e = assertThrows(InvalidResourceException.class, () -> KafkaConnectBuild.fromCrd(new Reconciliation("test", kc.getKind(), kc.getMetadata().getNamespace(), kc.getMetadata().getName()), kc, VERSIONS));
    assertThat(e.getMessage(), containsString(".spec.build.additionalKanikoOptions contains forbidden options: [--reproducible-something, --build-arg, --digest-file]"));
}

Example 38 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi by strimzi.

the class ConnectorMockTest method testChangeStrimziClusterLabel.

/**
 * Change the cluster label from one cluster to another
 * check the connector is deleted from the old cluster
 * check the connector is added to the new cluster
 */
@Test
public void testChangeStrimziClusterLabel(VertxTestContext context) throws InterruptedException {
    String oldConnectClusterName = "cluster1";
    String newConnectClusterName = "cluster2";
    String connectorName = "connector";
    // Create two connect clusters
    KafkaConnect connect = new KafkaConnectBuilder().withNewMetadata().withNamespace(NAMESPACE).withName(oldConnectClusterName).addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().withNewSpec().withReplicas(1).endSpec().build();
    Crds.kafkaConnectOperation(client).inNamespace(NAMESPACE).create(connect);
    waitForConnectReady(oldConnectClusterName);
    KafkaConnect connect2 = new KafkaConnectBuilder().withNewMetadata().withNamespace(NAMESPACE).withName(newConnectClusterName).addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().withNewSpec().withReplicas(1).endSpec().build();
    Crds.kafkaConnectOperation(client).inNamespace(NAMESPACE).create(connect2);
    waitForConnectReady(newConnectClusterName);
    // Create KafkaConnector associated with the first cluster using the Strimzi Cluster label and wait till it's ready
    KafkaConnector connector = new KafkaConnectorBuilder().withNewMetadata().withName(connectorName).withNamespace(NAMESPACE).addToLabels(Labels.STRIMZI_CLUSTER_LABEL, oldConnectClusterName).endMetadata().withNewSpec().withTasksMax(1).withClassName("Dummy").endSpec().build();
    Crds.kafkaConnectorOperation(client).inNamespace(NAMESPACE).create(connector);
    waitForConnectorReady(connectorName);
    // triggered twice (Connect creation, Connector Status update) for the first cluster
    verify(api, times(1)).createOrUpdatePutRequest(any(), eq(KafkaConnectResources.qualifiedServiceName(oldConnectClusterName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName), any());
    // never triggered for the second cluster as connector's Strimzi cluster label does not match cluster 2
    verify(api, never()).createOrUpdatePutRequest(any(), eq(KafkaConnectResources.qualifiedServiceName(newConnectClusterName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName), any());
    // patch connector with new Strimzi cluster label associated with cluster 2
    Crds.kafkaConnectorOperation(client).inNamespace(NAMESPACE).withName(connectorName).patch(new KafkaConnectorBuilder().withNewMetadata().withName(connectorName).withNamespace(NAMESPACE).addToLabels(Labels.STRIMZI_CLUSTER_LABEL, newConnectClusterName).endMetadata().withNewSpec().withTasksMax(1).withClassName("Dummy").endSpec().build());
    waitForConnectorReady(connectorName);
    // Note: The connector does not get deleted immediately from the first cluster, only on the next timed reconciliation
    verify(api, never()).delete(any(), eq(KafkaConnectResources.qualifiedServiceName(oldConnectClusterName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName));
    verify(api, times(1)).createOrUpdatePutRequest(any(), eq(KafkaConnectResources.qualifiedServiceName(newConnectClusterName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName), any());
    // Force reconciliation to assert connector deletion request occurs for first cluster
    Checkpoint async = context.checkpoint();
    kafkaConnectOperator.reconcile(new Reconciliation("test", "KafkaConnect", NAMESPACE, oldConnectClusterName)).onComplete(context.succeeding(v -> context.verify(() -> {
        verify(api, times(1)).delete(any(), eq(KafkaConnectResources.qualifiedServiceName(oldConnectClusterName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName));
        async.flag();
    })));
}

Example 39 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi by strimzi.

the class ConnectorMockTest method testConnectScaleToZero.

/**
 * Create connect, create connector, Scale to 0
 */
@Test
public void testConnectScaleToZero() {
    String connectName = "cluster";
    String connectorName = "connector";
    // Create KafkaConnect cluster and wait till it's ready
    KafkaConnect connect = new KafkaConnectBuilder().withNewMetadata().withNamespace(NAMESPACE).withName(connectName).addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().withNewSpec().withReplicas(1).endSpec().build();
    Crds.kafkaConnectOperation(client).inNamespace(NAMESPACE).create(connect);
    waitForConnectReady(connectName);
    // could be triggered twice (creation followed by status update) but waitForConnectReady could be satisfied with single
    verify(api, atLeastOnce()).list(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT));
    verify(api, never()).createOrUpdatePutRequest(any(), eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName), any());
    // Create KafkaConnector and wait till it's ready
    KafkaConnector connector = new KafkaConnectorBuilder().withNewMetadata().withName(connectorName).withNamespace(NAMESPACE).addToLabels(Labels.STRIMZI_CLUSTER_LABEL, connectName).endMetadata().withNewSpec().withTasksMax(1).withClassName("Dummy").endSpec().build();
    Crds.kafkaConnectorOperation(client).inNamespace(NAMESPACE).create(connector);
    waitForConnectorReady(connectorName);
    verify(api, times(2)).list(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT));
    verify(api, times(1)).createOrUpdatePutRequest(any(), eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName), any());
    assertThat(runningConnectors.keySet(), is(Collections.singleton(key("cluster-connect-api.ns.svc", connectorName))));
    when(api.list(any(), anyInt())).thenReturn(Future.failedFuture(new ConnectTimeoutException("connection timed out")));
    when(api.listConnectorPlugins(any(), any(), anyInt())).thenReturn(Future.failedFuture(new ConnectTimeoutException("connection timed out")));
    when(api.createOrUpdatePutRequest(any(), any(), anyInt(), anyString(), any())).thenReturn(Future.failedFuture(new ConnectTimeoutException("connection timed out")));
    when(api.getConnectorConfig(any(), any(), anyInt(), any())).thenReturn(Future.failedFuture(new ConnectTimeoutException("connection timed out")));
    when(api.getConnector(any(), any(), anyInt(), any())).thenReturn(Future.failedFuture(new ConnectTimeoutException("connection timed out")));
    Crds.kafkaConnectOperation(client).inNamespace(NAMESPACE).withName(connectName).edit(spec -> new KafkaConnectBuilder(spec).editSpec().withReplicas(0).endSpec().build());
    waitForConnectReady(connectName);
    waitForConnectorNotReady(connectorName, "RuntimeException", "Kafka Connect cluster 'cluster' in namespace ns has 0 replicas.");
}

Example 40 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi by strimzi.

the class ConnectorMockTest method testConnectorNotReadyWithoutSpec.

@Test
public void testConnectorNotReadyWithoutSpec() {
    String connectName = "cluster";
    String connectorName = "connector";
    KafkaConnect connect = new KafkaConnectBuilder().withNewMetadata().withNamespace(NAMESPACE).withName(connectName).addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().withNewSpec().withReplicas(1).endSpec().build();
    Crds.kafkaConnectOperation(client).inNamespace(NAMESPACE).create(connect);
    waitForConnectReady(connectName);
    KafkaConnector connector = new KafkaConnectorBuilder().withNewMetadata().withNamespace(NAMESPACE).withName(connectorName).addToLabels(Labels.STRIMZI_CLUSTER_LABEL, connectName).endMetadata().build();
    Crds.kafkaConnectorOperation(client).inNamespace(NAMESPACE).create(connector);
    waitForConnectorNotReady(connectorName, "InvalidResourceException", "spec property is required");
}