Examples with KafkaConnect io.strimzi.api.kafka.model.KafkaConnect used on opensource projects

Example 76 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi-kafka-operator by strimzi.

the class ConnectorMockTest method testConnectorConnectConnectorConnect.

/**
 * Create connector, create connect, delete connector, delete connect
 */
@Test
public void testConnectorConnectConnectorConnect() {
    String connectName = "cluster";
    String connectorName = "connector";
    // Create KafkaConnector and wait till it's ready
    KafkaConnector connector = new KafkaConnectorBuilder().withNewMetadata().withName(connectorName).withNamespace(NAMESPACE).addToLabels(Labels.STRIMZI_CLUSTER_LABEL, connectName).endMetadata().withNewSpec().endSpec().build();
    Crds.kafkaConnectorOperation(client).inNamespace(NAMESPACE).create(connector);
    waitForConnectorNotReady(connectorName, "NoSuchResourceException", "KafkaConnect resource 'cluster' identified by label '" + Labels.STRIMZI_CLUSTER_LABEL + "' does not exist in namespace ns.");
    verify(api, never()).list(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT));
    verify(api, never()).createOrUpdatePutRequest(any(), eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName), any());
    assertThat(runningConnectors.keySet(), is(empty()));
    // Create KafkaConnect cluster and wait till it's ready
    KafkaConnect connect = new KafkaConnectBuilder().withNewMetadata().withNamespace(NAMESPACE).withName(connectName).addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().withNewSpec().withReplicas(1).endSpec().build();
    Crds.kafkaConnectOperation(client).inNamespace(NAMESPACE).create(connect);
    waitForConnectReady(connectName);
    // could be triggered twice (creation followed by status update) but waitForConnectReady could be satisfied with single
    verify(api, atLeastOnce()).list(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT));
    // triggered three times (Connect creation, Connector Status update, Connect Status update)
    verify(api, times(3)).createOrUpdatePutRequest(any(), eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName), any());
    assertThat(runningConnectors.keySet(), is(Collections.singleton(key("cluster-connect-api.ns.svc", connectorName))));
    boolean connectorDeleted = Crds.kafkaConnectorOperation(client).inNamespace(NAMESPACE).withName(connectorName).delete();
    assertThat(connectorDeleted, is(true));
    waitFor("delete call on connect REST api", 1_000, 30_000, () -> runningConnectors.isEmpty());
    verify(api).delete(any(), eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName));
    boolean connectDeleted = Crds.kafkaConnectOperation(client).inNamespace(NAMESPACE).withName(connectName).delete();
    assertThat(connectDeleted, is(true));
}

Example 77 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi-kafka-operator by strimzi.

the class ConnectorMockTest method testConnectConnectorConnectConnector.

/**
 * Create connect, create connector, delete connect, delete connector
 */
@Test
public void testConnectConnectorConnectConnector() {
    String connectName = "cluster";
    String connectorName = "connector";
    // Create KafkaConnect cluster and wait till it's ready
    KafkaConnect connect = new KafkaConnectBuilder().withNewMetadata().withNamespace(NAMESPACE).withName(connectName).addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().withNewSpec().withReplicas(1).endSpec().build();
    Crds.kafkaConnectOperation(client).inNamespace(NAMESPACE).create(connect);
    waitForConnectReady(connectName);
    // could be triggered twice (creation followed by status update) but waitForConnectReady could be satisfied with single
    verify(api, atLeastOnce()).list(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT));
    verify(api, never()).createOrUpdatePutRequest(any(), eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName), any());
    // Create KafkaConnector and wait till it's ready
    KafkaConnector connector = new KafkaConnectorBuilder().withNewMetadata().withName(connectorName).withNamespace(NAMESPACE).addToLabels(Labels.STRIMZI_CLUSTER_LABEL, connectName).endMetadata().withNewSpec().withTasksMax(1).withClassName("Dummy").endSpec().build();
    Crds.kafkaConnectorOperation(client).inNamespace(NAMESPACE).create(connector);
    waitForConnectorReady(connectorName);
    // could be triggered twice (creation followed by status update) but waitForConnectReady could be satisfied with single
    verify(api, atLeastOnce()).list(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT));
    // triggered twice (Connect creation, Connector Status update)
    verify(api, times(1)).createOrUpdatePutRequest(any(), eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName), any());
    assertThat(runningConnectors.keySet(), is(Collections.singleton(key("cluster-connect-api.ns.svc", connectorName))));
    boolean connectDeleted = Crds.kafkaConnectOperation(client).inNamespace(NAMESPACE).withName(connectName).delete();
    assertThat(connectDeleted, is(true));
    waitForConnectorNotReady(connectorName, "NoSuchResourceException", "KafkaConnect resource 'cluster' identified by label '" + Labels.STRIMZI_CLUSTER_LABEL + "' does not exist in namespace ns.");
    boolean connectorDeleted = Crds.kafkaConnectorOperation(client).inNamespace(NAMESPACE).withName(connectorName).delete();
    assertThat(connectorDeleted, is(true));
    verify(api, never()).delete(any(), eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName));
}

Example 78 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi-kafka-operator by strimzi.

the class ConnectorMockTest method testConnectorPauseResume.

/**
 * Create connect, create connector, pause connector, resume connector
 */
@Test
public void testConnectorPauseResume() {
    String connectName = "cluster";
    String connectorName = "connector";
    // Create KafkaConnect cluster and wait till it's ready
    KafkaConnect connect = new KafkaConnectBuilder().withNewMetadata().withNamespace(NAMESPACE).withName(connectName).addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().withNewSpec().withReplicas(1).endSpec().build();
    Crds.kafkaConnectOperation(client).inNamespace(NAMESPACE).create(connect);
    waitForConnectReady(connectName);
    // could be triggered twice (creation followed by status update) but waitForConnectReady could be satisfied with single
    verify(api, atLeastOnce()).list(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT));
    verify(api, never()).createOrUpdatePutRequest(any(), eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName), any());
    // Create KafkaConnector and wait till it's ready
    KafkaConnector connector = new KafkaConnectorBuilder().withNewMetadata().withName(connectorName).withNamespace(NAMESPACE).addToLabels(Labels.STRIMZI_CLUSTER_LABEL, connectName).endMetadata().withNewSpec().withTasksMax(1).withClassName("Dummy").endSpec().build();
    Crds.kafkaConnectorOperation(client).inNamespace(NAMESPACE).create(connector);
    waitForConnectorReady(connectorName);
    waitForConnectorState(connectorName, "RUNNING");
    verify(api, times(2)).list(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT));
    verify(api, times(1)).createOrUpdatePutRequest(any(), eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName), any());
    assertThat(runningConnectors.keySet(), is(Collections.singleton(key("cluster-connect-api.ns.svc", connectorName))));
    verify(api, never()).pause(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName));
    verify(api, never()).resume(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName));
    Crds.kafkaConnectorOperation(client).inNamespace(NAMESPACE).withName(connectorName).edit(spec -> new KafkaConnectorBuilder(spec).editSpec().withPause(true).endSpec().build());
    waitForConnectorState(connectorName, "PAUSED");
    verify(api, times(1)).pause(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName));
    verify(api, never()).resume(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName));
    Crds.kafkaConnectorOperation(client).inNamespace(NAMESPACE).withName(connectorName).edit(sp -> new KafkaConnectorBuilder(sp).editSpec().withPause(false).endSpec().build());
    waitForConnectorState(connectorName, "RUNNING");
    verify(api, times(1)).pause(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName));
    verify(api, times(1)).resume(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName));
}

Example 79 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi-kafka-operator by strimzi.

the class ConnectorMockTest method testConnectRestAPIIssues.

/**
 * Create connect, create connector, break the REST API
 */
@Test
public void testConnectRestAPIIssues() {
    String connectName = "cluster";
    String connectorName = "connector";
    // Create KafkaConnect cluster and wait till it's ready
    KafkaConnect connect = new KafkaConnectBuilder().withNewMetadata().withNamespace(NAMESPACE).withName(connectName).addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().withNewSpec().withReplicas(1).endSpec().build();
    Crds.kafkaConnectOperation(client).inNamespace(NAMESPACE).create(connect);
    waitForConnectReady(connectName);
    // could be triggered twice (creation followed by status update) but waitForConnectReady could be satisfied with single
    verify(api, atLeastOnce()).list(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT));
    verify(api, never()).createOrUpdatePutRequest(any(), eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName), any());
    // Create KafkaConnector and wait till it's ready
    KafkaConnector connector = new KafkaConnectorBuilder().withNewMetadata().withName(connectorName).withNamespace(NAMESPACE).addToLabels(Labels.STRIMZI_CLUSTER_LABEL, connectName).endMetadata().withNewSpec().withTasksMax(1).withClassName("Dummy").endSpec().build();
    Crds.kafkaConnectorOperation(client).inNamespace(NAMESPACE).create(connector);
    waitForConnectorReady(connectorName);
    verify(api, times(2)).list(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT));
    verify(api, times(1)).createOrUpdatePutRequest(any(), eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName), any());
    assertThat(runningConnectors.keySet(), is(Collections.singleton(key("cluster-connect-api.ns.svc", connectorName))));
    when(api.list(any(), anyInt())).thenReturn(Future.failedFuture(new ConnectTimeoutException("connection timed out")));
    when(api.listConnectorPlugins(any(), any(), anyInt())).thenReturn(Future.failedFuture(new ConnectTimeoutException("connection timed out")));
    when(api.createOrUpdatePutRequest(any(), any(), anyInt(), anyString(), any())).thenReturn(Future.failedFuture(new ConnectTimeoutException("connection timed out")));
    when(api.getConnectorConfig(any(), any(), any(), anyInt(), any())).thenReturn(Future.failedFuture(new ConnectTimeoutException("connection timed out")));
    when(api.getConnector(any(), any(), anyInt(), any())).thenReturn(Future.failedFuture(new ConnectTimeoutException("connection timed out")));
    Crds.kafkaConnectOperation(client).inNamespace(NAMESPACE).withName(connectName).edit(sp -> new KafkaConnectBuilder(sp).editSpec().withNewTemplate().endTemplate().endSpec().build());
    // Wait for Status change due to the broker REST API
    waitForConnectNotReady(connectName, "ConnectTimeoutException", "connection timed out");
    waitForConnectorNotReady(connectorName, "ConnectTimeoutException", "connection timed out");
}

Example 80 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi-kafka-operator by strimzi.

the class ConnectorMockTest method setup.

@SuppressWarnings({ "checkstyle:MethodLength" })
@BeforeEach
public void setup(VertxTestContext testContext) {
    vertx = Vertx.vertx();
    client = new MockKube().withCustomResourceDefinition(Crds.kafkaConnect(), KafkaConnect.class, KafkaConnectList.class, KafkaConnect::getStatus, KafkaConnect::setStatus).end().withCustomResourceDefinition(Crds.kafkaConnector(), KafkaConnector.class, KafkaConnectorList.class, KafkaConnector::getStatus, KafkaConnector::setStatus).end().build();
    PlatformFeaturesAvailability pfa = new PlatformFeaturesAvailability(true, KubernetesVersion.V1_18);
    setupMockConnectAPI();
    ResourceOperatorSupplier ros = new ResourceOperatorSupplier(vertx, client, new ZookeeperLeaderFinder(vertx, // Retry up to 3 times (4 attempts), with overall max delay of 35000ms
    () -> new BackOff(5_000, 2, 4)), new DefaultAdminClientProvider(), new DefaultZookeeperScalerProvider(), ResourceUtils.metricsProvider(), pfa, FeatureGates.NONE, 10_000);
    ClusterOperatorConfig config = ClusterOperatorConfig.fromMap(map(ClusterOperatorConfig.STRIMZI_KAFKA_IMAGES, KafkaVersionTestUtils.getKafkaImagesEnvVarString(), ClusterOperatorConfig.STRIMZI_KAFKA_CONNECT_IMAGES, KafkaVersionTestUtils.getKafkaConnectImagesEnvVarString(), ClusterOperatorConfig.STRIMZI_KAFKA_MIRROR_MAKER_2_IMAGES, KafkaVersionTestUtils.getKafkaMirrorMaker2ImagesEnvVarString(), ClusterOperatorConfig.STRIMZI_FULL_RECONCILIATION_INTERVAL_MS, Long.toString(Long.MAX_VALUE)), KafkaVersionTestUtils.getKafkaVersionLookup());
    kafkaConnectOperator = new KafkaConnectAssemblyOperator(vertx, pfa, ros, config, x -> api);
    Checkpoint async = testContext.checkpoint();
    // Fail test if watcher closes for any reason
    kafkaConnectOperator.createWatch(NAMESPACE, e -> testContext.failNow(e)).onComplete(testContext.succeeding()).compose(watch -> AbstractConnectOperator.createConnectorWatch(kafkaConnectOperator, NAMESPACE, null)).onComplete(testContext.succeeding(v -> async.flag()));
}