Examples with KafkaConnect io.strimzi.api.kafka.model.KafkaConnect used on opensource projects

Example 71 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi by strimzi.

the class SpecificIsolatedST method testRackAwareConnectCorrectDeployment.

@IsolatedTest("Modification of shared Cluster Operator configuration")
@Tag(CONNECT)
@Tag(CONNECT_COMPONENTS)
@Tag(ACCEPTANCE)
@Tag(INTERNAL_CLIENTS_USED)
void testRackAwareConnectCorrectDeployment(ExtensionContext extensionContext) {
    assumeFalse(Environment.isNamespaceRbacScope());
    assumeTrue(!Environment.isHelmInstall() && !Environment.isOlmInstall());
    String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    // We need to update CO configuration to set OPERATION_TIMEOUT to shorter value, because we expect timeout in that test
    Map<String, String> coSnapshot = DeploymentUtils.depSnapshot(clusterOperator.getDeploymentNamespace(), ResourceManager.getCoDeploymentName());
    // We have to install CO in class stack, otherwise it will be deleted at the end of test case and all following tests will fail
    clusterOperator.unInstall();
    clusterOperator = clusterOperator.defaultInstallation().withOperationTimeout(CO_OPERATION_TIMEOUT_SHORT).withReconciliationInterval(Constants.RECONCILIATION_INTERVAL).createInstallation().runBundleInstallation();
    coSnapshot = DeploymentUtils.waitTillDepHasRolled(ResourceManager.getCoDeploymentName(), 1, coSnapshot);
    String rackKey = "rack-key";
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(clusterName, 3).editSpec().editKafka().withNewRack().withTopologyKey(rackKey).endRack().addToConfig("replica.selector.class", "org.apache.kafka.common.replica.RackAwareReplicaSelector").endKafka().endSpec().build());
    // we should create topic before KafkaConnect - topic is recreated if we delete it before KafkaConnect
    String topicName = "rw-" + KafkaTopicUtils.generateRandomNameOfTopic();
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName).build());
    resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(false, kafkaClientsName).build());
    String kafkaClientsPodName = kubeClient().listPodsByPrefixInName(kafkaClientsName).get(0).getMetadata().getName();
    LOGGER.info("Deploy KafkaConnect with correct rack-aware topology key: {}", rackKey);
    KafkaConnect kc = KafkaConnectTemplates.kafkaConnect(extensionContext, clusterName, 1).editSpec().withNewRack().withTopologyKey(rackKey).endRack().addToConfig("key.converter.schemas.enable", false).addToConfig("value.converter.schemas.enable", false).addToConfig("key.converter", "org.apache.kafka.connect.storage.StringConverter").addToConfig("value.converter", "org.apache.kafka.connect.storage.StringConverter").endSpec().build();
    resourceManager.createResource(extensionContext, kc);
    NetworkPolicyResource.deployNetworkPolicyForResource(extensionContext, kc, KafkaConnectResources.deploymentName(clusterName));
    String connectPodName = kubeClient().listPodNames(Labels.STRIMZI_KIND_LABEL, KafkaConnect.RESOURCE_KIND).get(0);
    Affinity connectPodSpecAffinity = kubeClient().getDeployment(KafkaConnectResources.deploymentName(clusterName)).getSpec().getTemplate().getSpec().getAffinity();
    NodeSelectorRequirement connectPodNodeSelectorRequirement = connectPodSpecAffinity.getNodeAffinity().getRequiredDuringSchedulingIgnoredDuringExecution().getNodeSelectorTerms().get(0).getMatchExpressions().get(0);
    Pod connectPod = kubeClient().getPod(connectPodName);
    NodeAffinity nodeAffinity = connectPod.getSpec().getAffinity().getNodeAffinity();
    LOGGER.info("PodName: {}\nNodeAffinity: {}", connectPodName, nodeAffinity);
    assertThat(connectPodNodeSelectorRequirement.getKey(), is(rackKey));
    assertThat(connectPodNodeSelectorRequirement.getOperator(), is("Exists"));
    KafkaConnectUtils.sendReceiveMessagesThroughConnect(connectPodName, topicName, kafkaClientsPodName, clusterOperator.getDeploymentNamespace(), clusterName);
    // Revert changes for CO deployment
    clusterOperator.unInstall();
    clusterOperator = clusterOperator.defaultInstallation().createInstallation().runInstallation();
    DeploymentUtils.waitTillDepHasRolled(ResourceManager.getCoDeploymentName(), 1, coSnapshot);
}

Example 72 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project debezium by debezium.

the class OcpKafkaConnectDeployer method deploy.

/**
 * Deploys Kafka Connect Cluster
 * @return {@link OcpKafkaController} instance for deployed cluster
 */
@Override
public OcpKafkaConnectController deploy() throws InterruptedException {
    LOGGER.info("Deploying KafkaConnect from " + yamlPath);
    KafkaConnect kafkaConnect = YAML.fromResource(yamlPath, KafkaConnect.class);
    Build kcBuild = kafkaConnect.getSpec().getBuild();
    KafkaConnectBuilder kcBuilder = new KafkaConnectBuilder(kafkaConnect);
    if (cfgYamlPath != null) {
        deployConfigMap();
    }
    if (connectorResources) {
        configureConnectorResources(kcBuilder);
    }
    if (pullSecretName != null) {
        configurePullSecret(kcBuilder, kcBuild);
    }
    if (kcBuild != null && "imagestream".equals(kcBuild.getOutput().getType())) {
        deployImageStream(kcBuild);
    }
    kafkaConnect = kcBuilder.build();
    kafkaConnect = kafkaConnectOperation().createOrReplace(kafkaConnect);
    OcpKafkaConnectController controller = new OcpKafkaConnectController(kafkaConnect, operatorController, ocp, http, connectorResources);
    controller.waitForCluster();
    return controller;
}

Example 73 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi by strimzi.

the class AbstractConnectOperator method reconcileConnectors.

/**
 * Reconcile all the connectors selected by the given connect instance, updated each connectors status with the result.
 * @param reconciliation The reconciliation
 * @param connect The connector
 * @param connectStatus Status of the KafkaConnect  resource (will be used to set the available
 *                      connector plugins)
 * @param scaledToZero  Indicated whether the related Connect cluster is currently scaled to 0 replicas
 * @return A future, failed if any of the connectors' statuses could not be updated.
 */
protected Future<Void> reconcileConnectors(Reconciliation reconciliation, T connect, S connectStatus, boolean scaledToZero, String desiredLogging, OrderedProperties defaultLogging) {
    String connectName = connect.getMetadata().getName();
    String namespace = connect.getMetadata().getNamespace();
    String host = KafkaConnectResources.qualifiedServiceName(connectName, namespace);
    if (!isUseResources(connect)) {
        return Future.succeededFuture();
    }
    if (scaledToZero) {
        return connectorOperator.listAsync(namespace, Optional.of(new LabelSelectorBuilder().addToMatchLabels(Labels.STRIMZI_CLUSTER_LABEL, connectName).build())).compose(connectors -> CompositeFuture.join(connectors.stream().map(connector -> maybeUpdateConnectorStatus(reconciliation, connector, null, zeroReplicas(namespace, connectName))).collect(Collectors.toList()))).map((Void) null);
    }
    KafkaConnectApi apiClient = connectClientProvider.apply(vertx);
    return CompositeFuture.join(apiClient.list(host, port), connectorOperator.listAsync(namespace, Optional.of(new LabelSelectorBuilder().addToMatchLabels(Labels.STRIMZI_CLUSTER_LABEL, connectName).build())), apiClient.listConnectorPlugins(reconciliation, host, port), apiClient.updateConnectLoggers(reconciliation, host, port, desiredLogging, defaultLogging)).compose(cf -> {
        List<String> runningConnectorNames = cf.resultAt(0);
        List<KafkaConnector> desiredConnectors = cf.resultAt(1);
        List<ConnectorPlugin> connectorPlugins = cf.resultAt(2);
        LOGGER.debugCr(reconciliation, "Setting list of connector plugins in Kafka Connect status");
        connectStatus.setConnectorPlugins(connectorPlugins);
        connectorsResourceCounter(namespace).set(desiredConnectors.size());
        Set<String> deleteConnectorNames = new HashSet<>(runningConnectorNames);
        deleteConnectorNames.removeAll(desiredConnectors.stream().map(c -> c.getMetadata().getName()).collect(Collectors.toSet()));
        LOGGER.debugCr(reconciliation, "{} cluster: delete connectors: {}", kind(), deleteConnectorNames);
        Stream<Future<Void>> deletionFutures = deleteConnectorNames.stream().map(connectorName -> reconcileConnectorAndHandleResult(reconciliation, host, apiClient, true, connectorName, null));
        LOGGER.debugCr(reconciliation, "{} cluster: required connectors: {}", kind(), desiredConnectors);
        Stream<Future<Void>> createUpdateFutures = desiredConnectors.stream().map(connector -> reconcileConnectorAndHandleResult(reconciliation, host, apiClient, true, connector.getMetadata().getName(), connector));
        return CompositeFuture.join(Stream.concat(deletionFutures, createUpdateFutures).collect(Collectors.toList())).map((Void) null);
    }).recover(error -> {
        if (error instanceof ConnectTimeoutException) {
            Promise<Void> connectorStatuses = Promise.promise();
            LOGGER.warnCr(reconciliation, "Failed to connect to the REST API => trying to update the connector status");
            connectorOperator.listAsync(namespace, Optional.of(new LabelSelectorBuilder().addToMatchLabels(Labels.STRIMZI_CLUSTER_LABEL, connectName).build())).compose(connectors -> CompositeFuture.join(connectors.stream().map(connector -> maybeUpdateConnectorStatus(reconciliation, connector, null, error)).collect(Collectors.toList()))).onComplete(ignore -> connectorStatuses.fail(error));
            return connectorStatuses.future();
        } else {
            return Future.failedFuture(error);
        }
    });
}

Example 74 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi-kafka-operator by strimzi.

the class AbstractConnectOperator method createConnectorWatch.

/**
 * Create a watch on {@code KafkaConnector} in the given {@code namespace}.
 * The watcher will:
 * <ul>
 * <li>{@linkplain #reconcileConnectors(Reconciliation, CustomResource, KafkaConnectStatus, boolean, String, OrderedProperties)} on the KafkaConnect
 * identified by {@code KafkaConnector.metadata.labels[strimzi.io/cluster]}.</li>
 * <li>The {@code KafkaConnector} status is updated with the result.</li>
 * </ul>
 * @param connectOperator The operator for {@code KafkaConnect}.
 * @param watchNamespaceOrWildcard The namespace to watch.
 * @param selectorLabels Selector labels for filtering the custom resources
 *
 * @return A future which completes when the watch has been set up.
 */
public static Future<Void> createConnectorWatch(AbstractConnectOperator<KubernetesClient, KafkaConnect, KafkaConnectList, Resource<KafkaConnect>, KafkaConnectSpec, KafkaConnectStatus> connectOperator, String watchNamespaceOrWildcard, Labels selectorLabels) {
    Optional<LabelSelector> selector = (selectorLabels == null || selectorLabels.toMap().isEmpty()) ? Optional.empty() : Optional.of(new LabelSelector(null, selectorLabels.toMap()));
    return Util.async(connectOperator.vertx, () -> {
        connectOperator.connectorOperator.watch(watchNamespaceOrWildcard, new Watcher<KafkaConnector>() {

            @Override
            public void eventReceived(Action action, KafkaConnector kafkaConnector) {
                String connectorName = kafkaConnector.getMetadata().getName();
                String connectorNamespace = kafkaConnector.getMetadata().getNamespace();
                String connectorKind = kafkaConnector.getKind();
                String connectName = kafkaConnector.getMetadata().getLabels() == null ? null : kafkaConnector.getMetadata().getLabels().get(Labels.STRIMZI_CLUSTER_LABEL);
                String connectNamespace = connectorNamespace;
                switch(action) {
                    case ADDED:
                    case DELETED:
                    case MODIFIED:
                        if (connectName != null) {
                            // Check whether a KafkaConnect exists
                            connectOperator.resourceOperator.getAsync(connectNamespace, connectName).compose(connect -> {
                                KafkaConnectApi apiClient = connectOperator.connectClientProvider.apply(connectOperator.vertx);
                                if (connect == null) {
                                    Reconciliation r = new Reconciliation("connector-watch", connectOperator.kind(), kafkaConnector.getMetadata().getNamespace(), connectName);
                                    updateStatus(r, noConnectCluster(connectNamespace, connectName), kafkaConnector, connectOperator.connectorOperator);
                                    LOGGER.infoCr(r, "{} {} in namespace {} was {}, but Connect cluster {} does not exist", connectorKind, connectorName, connectorNamespace, action, connectName);
                                    return Future.succeededFuture();
                                } else {
                                    // grab the lock and call reconcileConnectors()
                                    // (i.e. short circuit doing a whole KafkaConnect reconciliation).
                                    Reconciliation reconciliation = new Reconciliation("connector-watch", connectOperator.kind(), kafkaConnector.getMetadata().getNamespace(), connectName);
                                    if (!Util.matchesSelector(selector, connect)) {
                                        LOGGER.debugCr(reconciliation, "{} {} in namespace {} was {}, but Connect cluster {} does not match label selector {} and will be ignored", connectorKind, connectorName, connectorNamespace, action, connectName, selectorLabels);
                                        return Future.succeededFuture();
                                    } else if (connect.getSpec() != null && connect.getSpec().getReplicas() == 0) {
                                        LOGGER.infoCr(reconciliation, "{} {} in namespace {} was {}, but Connect cluster {} has 0 replicas", connectorKind, connectorName, connectorNamespace, action, connectName);
                                        updateStatus(reconciliation, zeroReplicas(connectNamespace, connectName), kafkaConnector, connectOperator.connectorOperator);
                                        return Future.succeededFuture();
                                    } else {
                                        LOGGER.infoCr(reconciliation, "{} {} in namespace {} was {}", connectorKind, connectorName, connectorNamespace, action);
                                        return connectOperator.withLock(reconciliation, LOCK_TIMEOUT_MS, () -> connectOperator.reconcileConnectorAndHandleResult(reconciliation, KafkaConnectResources.qualifiedServiceName(connectName, connectNamespace), apiClient, isUseResources(connect), kafkaConnector.getMetadata().getName(), action == Action.DELETED ? null : kafkaConnector).compose(reconcileResult -> {
                                            LOGGER.infoCr(reconciliation, "reconciled");
                                            return Future.succeededFuture(reconcileResult);
                                        }));
                                    }
                                }
                            });
                        } else {
                            updateStatus(new Reconciliation("connector-watch", connectOperator.kind(), kafkaConnector.getMetadata().getNamespace(), null), new InvalidResourceException("Resource lacks label '" + Labels.STRIMZI_CLUSTER_LABEL + "': No connect cluster in which to create this connector."), kafkaConnector, connectOperator.connectorOperator);
                        }
                        break;
                    case ERROR:
                        LOGGER.errorCr(new Reconciliation("connector-watch", connectorKind, connectName, connectorNamespace), "Failed {} {} in namespace {} ", connectorKind, connectorName, connectorNamespace);
                        break;
                    default:
                        LOGGER.errorCr(new Reconciliation("connector-watch", connectorKind, connectName, connectorNamespace), "Unknown action: {} {} in namespace {}", connectorKind, connectorName, connectorNamespace);
                }
            }

            @Override
            public void onClose(WatcherException e) {
                if (e != null) {
                    throw new KubernetesClientException(e.getMessage());
                }
            }
        });
        return null;
    });
}

Example 75 with KafkaConnect

use of io.strimzi.api.kafka.model.KafkaConnect in project strimzi-kafka-operator by strimzi.

the class ConnectorMockTest method testConnectScaleToZero.

/**
 * Create connect, create connector, Scale to 0
 */
@Test
public void testConnectScaleToZero() {
    String connectName = "cluster";
    String connectorName = "connector";
    // Create KafkaConnect cluster and wait till it's ready
    KafkaConnect connect = new KafkaConnectBuilder().withNewMetadata().withNamespace(NAMESPACE).withName(connectName).addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().withNewSpec().withReplicas(1).endSpec().build();
    Crds.kafkaConnectOperation(client).inNamespace(NAMESPACE).create(connect);
    waitForConnectReady(connectName);
    // could be triggered twice (creation followed by status update) but waitForConnectReady could be satisfied with single
    verify(api, atLeastOnce()).list(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT));
    verify(api, never()).createOrUpdatePutRequest(any(), eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName), any());
    // Create KafkaConnector and wait till it's ready
    KafkaConnector connector = new KafkaConnectorBuilder().withNewMetadata().withName(connectorName).withNamespace(NAMESPACE).addToLabels(Labels.STRIMZI_CLUSTER_LABEL, connectName).endMetadata().withNewSpec().withTasksMax(1).withClassName("Dummy").endSpec().build();
    Crds.kafkaConnectorOperation(client).inNamespace(NAMESPACE).create(connector);
    waitForConnectorReady(connectorName);
    verify(api, times(2)).list(eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT));
    verify(api, times(1)).createOrUpdatePutRequest(any(), eq(KafkaConnectResources.qualifiedServiceName(connectName, NAMESPACE)), eq(KafkaConnectCluster.REST_API_PORT), eq(connectorName), any());
    assertThat(runningConnectors.keySet(), is(Collections.singleton(key("cluster-connect-api.ns.svc", connectorName))));
    when(api.list(any(), anyInt())).thenReturn(Future.failedFuture(new ConnectTimeoutException("connection timed out")));
    when(api.listConnectorPlugins(any(), any(), anyInt())).thenReturn(Future.failedFuture(new ConnectTimeoutException("connection timed out")));
    when(api.createOrUpdatePutRequest(any(), any(), anyInt(), anyString(), any())).thenReturn(Future.failedFuture(new ConnectTimeoutException("connection timed out")));
    when(api.getConnectorConfig(any(), any(), anyInt(), any())).thenReturn(Future.failedFuture(new ConnectTimeoutException("connection timed out")));
    when(api.getConnector(any(), any(), anyInt(), any())).thenReturn(Future.failedFuture(new ConnectTimeoutException("connection timed out")));
    Crds.kafkaConnectOperation(client).inNamespace(NAMESPACE).withName(connectName).edit(spec -> new KafkaConnectBuilder(spec).editSpec().withReplicas(0).endSpec().build());
    waitForConnectReady(connectName);
    waitForConnectorNotReady(connectorName, "RuntimeException", "Kafka Connect cluster 'cluster' in namespace ns has 0 replicas.");
}