Search in sources :

Example 1 with KafkaMirrorMaker2ConnectorSpec

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2ConnectorSpec in project strimzi by strimzi.

the class KafkaMirrorMaker2AssemblyOperator method reconcileConnectors.

/**
 * Reconcile all the MirrorMaker 2.0 connectors selected by the given MirrorMaker 2.0 instance.
 * @param reconciliation The reconciliation
 * @param kafkaMirrorMaker2 The MirrorMaker 2.0
 * @return A future, failed if any of the connectors could not be reconciled.
 */
protected Future<Void> reconcileConnectors(Reconciliation reconciliation, KafkaMirrorMaker2 kafkaMirrorMaker2, KafkaMirrorMaker2Cluster mirrorMaker2Cluster, KafkaMirrorMaker2Status mirrorMaker2Status, String desiredLogging) {
    String mirrorMaker2Name = kafkaMirrorMaker2.getMetadata().getName();
    if (kafkaMirrorMaker2.getSpec() == null) {
        return maybeUpdateMirrorMaker2Status(reconciliation, kafkaMirrorMaker2, new InvalidResourceException("spec property is required"));
    }
    List<KafkaMirrorMaker2MirrorSpec> mirrors = ModelUtils.asListOrEmptyList(kafkaMirrorMaker2.getSpec().getMirrors());
    String host = KafkaMirrorMaker2Resources.qualifiedServiceName(mirrorMaker2Name, reconciliation.namespace());
    KafkaConnectApi apiClient = getKafkaConnectApi();
    return apiClient.list(host, KafkaConnectCluster.REST_API_PORT).compose(deleteMirrorMaker2ConnectorNames -> {
        for (Map.Entry<String, Function<KafkaMirrorMaker2MirrorSpec, KafkaMirrorMaker2ConnectorSpec>> connectorEntry : MIRRORMAKER2_CONNECTORS.entrySet()) {
            deleteMirrorMaker2ConnectorNames.removeAll(mirrors.stream().filter(// filter out non-existent connectors
            mirror -> connectorEntry.getValue().apply(mirror) != null).map(mirror -> mirror.getSourceCluster() + "->" + mirror.getTargetCluster() + connectorEntry.getKey()).collect(Collectors.toSet()));
        }
        LOGGER.debugCr(reconciliation, "delete MirrorMaker 2.0 connectors: {}", deleteMirrorMaker2ConnectorNames);
        Stream<Future<Void>> deletionFutures = deleteMirrorMaker2ConnectorNames.stream().map(connectorName -> apiClient.delete(reconciliation, host, KafkaConnectCluster.REST_API_PORT, connectorName));
        Stream<Future<Void>> createUpdateFutures = mirrors.stream().map(mirror -> reconcileMirrorMaker2Connectors(reconciliation, host, apiClient, kafkaMirrorMaker2, mirror, mirrorMaker2Cluster, mirrorMaker2Status, desiredLogging));
        return CompositeFuture.join(Stream.concat(deletionFutures, createUpdateFutures).collect(Collectors.toList())).map((Void) null);
    });
}
Also used : InvalidResourceException(io.strimzi.operator.cluster.model.InvalidResourceException) KafkaMirrorMaker2Builder(io.strimzi.api.kafka.model.KafkaMirrorMaker2Builder) Annotations(io.strimzi.operator.common.Annotations) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_TASK(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_TASK) Resource(io.fabric8.kubernetes.client.dsl.Resource) Matcher(java.util.regex.Matcher) Map(java.util.Map) KafkaConnectorSpec(io.strimzi.api.kafka.model.KafkaConnectorSpec) ReconciliationException(io.strimzi.operator.common.ReconciliationException) ResourceOperatorSupplier(io.strimzi.operator.cluster.operator.resource.ResourceOperatorSupplier) AbstractModel(io.strimzi.operator.cluster.model.AbstractModel) ModelUtils(io.strimzi.operator.cluster.model.ModelUtils) StatusUtils(io.strimzi.operator.common.operator.resource.StatusUtils) ANNO_STRIMZI_IO_RESTART_CONNECTOR(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) KafkaMirrorMaker2ConnectorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ConnectorSpec) KafkaMirrorMaker2Spec(io.strimzi.api.kafka.model.KafkaMirrorMaker2Spec) DeploymentOperator(io.strimzi.operator.common.operator.resource.DeploymentOperator) KafkaVersion(io.strimzi.operator.cluster.model.KafkaVersion) Collectors(java.util.stream.Collectors) Future(io.vertx.core.Future) Serializable(java.io.Serializable) KafkaMirrorMaker2Resources(io.strimzi.api.kafka.model.KafkaMirrorMaker2Resources) KafkaClientAuthenticationScramSha256(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationScramSha256) KafkaMirrorMaker2List(io.strimzi.api.kafka.KafkaMirrorMaker2List) KafkaMirrorMaker2(io.strimzi.api.kafka.model.KafkaMirrorMaker2) KafkaMirrorMaker2Status(io.strimzi.api.kafka.model.status.KafkaMirrorMaker2Status) List(java.util.List) KafkaConnectCluster(io.strimzi.operator.cluster.model.KafkaConnectCluster) Stream(java.util.stream.Stream) Condition(io.strimzi.api.kafka.model.status.Condition) PlatformFeaturesAvailability(io.strimzi.operator.PlatformFeaturesAvailability) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN) ClusterOperatorConfig(io.strimzi.operator.cluster.ClusterOperatorConfig) CustomResource(io.fabric8.kubernetes.client.CustomResource) AuthenticationUtils(io.strimzi.operator.cluster.model.AuthenticationUtils) HashMap(java.util.HashMap) KafkaClientAuthenticationPlain(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationPlain) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK) CompositeFuture(io.vertx.core.CompositeFuture) KafkaClientAuthenticationOAuth(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationOAuth) KafkaClientAuthenticationTls(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationTls) SaslConfigs(org.apache.kafka.common.config.SaslConfigs) SslConfigs(org.apache.kafka.common.config.SslConfigs) ReconciliationLogger(io.strimzi.operator.common.ReconciliationLogger) Collections.emptyMap(java.util.Collections.emptyMap) KafkaMirrorMaker2ClusterSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec) InvalidResourceException(io.strimzi.operator.cluster.model.InvalidResourceException) AdminClientConfig(org.apache.kafka.clients.admin.AdminClientConfig) Promise(io.vertx.core.Promise) Vertx(io.vertx.core.Vertx) KafkaMirrorMaker2MirrorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2MirrorSpec) KafkaMirrorMaker2Cluster(io.strimzi.operator.cluster.model.KafkaMirrorMaker2Cluster) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_CONNECTOR(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_CONNECTOR) KafkaClientAuthenticationScramSha512(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationScramSha512) Reconciliation(io.strimzi.operator.common.Reconciliation) Util(io.strimzi.operator.common.Util) KubernetesClient(io.fabric8.kubernetes.client.KubernetesClient) Comparator(java.util.Comparator) Collections(java.util.Collections) Deployment(io.fabric8.kubernetes.api.model.apps.Deployment) KafkaConnectorSpecBuilder(io.strimzi.api.kafka.model.KafkaConnectorSpecBuilder) Function(java.util.function.Function) Future(io.vertx.core.Future) CompositeFuture(io.vertx.core.CompositeFuture) KafkaMirrorMaker2MirrorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2MirrorSpec) Map(java.util.Map) HashMap(java.util.HashMap) Collections.emptyMap(java.util.Collections.emptyMap)

Example 2 with KafkaMirrorMaker2ConnectorSpec

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2ConnectorSpec in project strimzi-kafka-operator by strimzi.

the class KafkaMirrorMaker2AssemblyOperator method reconcileConnectors.

/**
 * Reconcile all the MirrorMaker 2.0 connectors selected by the given MirrorMaker 2.0 instance.
 * @param reconciliation The reconciliation
 * @param kafkaMirrorMaker2 The MirrorMaker 2.0
 * @return A future, failed if any of the connectors could not be reconciled.
 */
protected Future<Void> reconcileConnectors(Reconciliation reconciliation, KafkaMirrorMaker2 kafkaMirrorMaker2, KafkaMirrorMaker2Cluster mirrorMaker2Cluster, KafkaMirrorMaker2Status mirrorMaker2Status, String desiredLogging) {
    String mirrorMaker2Name = kafkaMirrorMaker2.getMetadata().getName();
    if (kafkaMirrorMaker2.getSpec() == null) {
        return maybeUpdateMirrorMaker2Status(reconciliation, kafkaMirrorMaker2, new InvalidResourceException("spec property is required"));
    }
    List<KafkaMirrorMaker2MirrorSpec> mirrors = ModelUtils.asListOrEmptyList(kafkaMirrorMaker2.getSpec().getMirrors());
    String host = KafkaMirrorMaker2Resources.qualifiedServiceName(mirrorMaker2Name, reconciliation.namespace());
    KafkaConnectApi apiClient = getKafkaConnectApi();
    return apiClient.list(host, KafkaConnectCluster.REST_API_PORT).compose(deleteMirrorMaker2ConnectorNames -> {
        for (Map.Entry<String, Function<KafkaMirrorMaker2MirrorSpec, KafkaMirrorMaker2ConnectorSpec>> connectorEntry : MIRRORMAKER2_CONNECTORS.entrySet()) {
            deleteMirrorMaker2ConnectorNames.removeAll(mirrors.stream().filter(// filter out non-existent connectors
            mirror -> connectorEntry.getValue().apply(mirror) != null).map(mirror -> mirror.getSourceCluster() + "->" + mirror.getTargetCluster() + connectorEntry.getKey()).collect(Collectors.toSet()));
        }
        LOGGER.debugCr(reconciliation, "delete MirrorMaker 2.0 connectors: {}", deleteMirrorMaker2ConnectorNames);
        Stream<Future<Void>> deletionFutures = deleteMirrorMaker2ConnectorNames.stream().map(connectorName -> apiClient.delete(reconciliation, host, KafkaConnectCluster.REST_API_PORT, connectorName));
        Stream<Future<Void>> createUpdateFutures = mirrors.stream().map(mirror -> reconcileMirrorMaker2Connectors(reconciliation, host, apiClient, kafkaMirrorMaker2, mirror, mirrorMaker2Cluster, mirrorMaker2Status, desiredLogging));
        return CompositeFuture.join(Stream.concat(deletionFutures, createUpdateFutures).collect(Collectors.toList())).map((Void) null);
    });
}
Also used : InvalidResourceException(io.strimzi.operator.cluster.model.InvalidResourceException) KafkaMirrorMaker2Builder(io.strimzi.api.kafka.model.KafkaMirrorMaker2Builder) Annotations(io.strimzi.operator.common.Annotations) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_TASK(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_TASK) Resource(io.fabric8.kubernetes.client.dsl.Resource) Matcher(java.util.regex.Matcher) Map(java.util.Map) KafkaConnectorSpec(io.strimzi.api.kafka.model.KafkaConnectorSpec) ReconciliationException(io.strimzi.operator.common.ReconciliationException) ResourceOperatorSupplier(io.strimzi.operator.cluster.operator.resource.ResourceOperatorSupplier) AbstractModel(io.strimzi.operator.cluster.model.AbstractModel) ModelUtils(io.strimzi.operator.cluster.model.ModelUtils) StatusUtils(io.strimzi.operator.common.operator.resource.StatusUtils) ANNO_STRIMZI_IO_RESTART_CONNECTOR(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) KafkaMirrorMaker2ConnectorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ConnectorSpec) KafkaMirrorMaker2Spec(io.strimzi.api.kafka.model.KafkaMirrorMaker2Spec) DeploymentOperator(io.strimzi.operator.common.operator.resource.DeploymentOperator) KafkaVersion(io.strimzi.operator.cluster.model.KafkaVersion) Collectors(java.util.stream.Collectors) Future(io.vertx.core.Future) Serializable(java.io.Serializable) KafkaMirrorMaker2Resources(io.strimzi.api.kafka.model.KafkaMirrorMaker2Resources) KafkaClientAuthenticationScramSha256(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationScramSha256) KafkaMirrorMaker2List(io.strimzi.api.kafka.KafkaMirrorMaker2List) KafkaMirrorMaker2(io.strimzi.api.kafka.model.KafkaMirrorMaker2) KafkaMirrorMaker2Status(io.strimzi.api.kafka.model.status.KafkaMirrorMaker2Status) List(java.util.List) KafkaConnectCluster(io.strimzi.operator.cluster.model.KafkaConnectCluster) Stream(java.util.stream.Stream) Condition(io.strimzi.api.kafka.model.status.Condition) PlatformFeaturesAvailability(io.strimzi.operator.PlatformFeaturesAvailability) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN) ClusterOperatorConfig(io.strimzi.operator.cluster.ClusterOperatorConfig) CustomResource(io.fabric8.kubernetes.client.CustomResource) AuthenticationUtils(io.strimzi.operator.cluster.model.AuthenticationUtils) HashMap(java.util.HashMap) KafkaClientAuthenticationPlain(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationPlain) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK) CompositeFuture(io.vertx.core.CompositeFuture) KafkaClientAuthenticationOAuth(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationOAuth) KafkaClientAuthenticationTls(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationTls) SaslConfigs(org.apache.kafka.common.config.SaslConfigs) SslConfigs(org.apache.kafka.common.config.SslConfigs) ReconciliationLogger(io.strimzi.operator.common.ReconciliationLogger) Collections.emptyMap(java.util.Collections.emptyMap) KafkaMirrorMaker2ClusterSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec) InvalidResourceException(io.strimzi.operator.cluster.model.InvalidResourceException) AdminClientConfig(org.apache.kafka.clients.admin.AdminClientConfig) Promise(io.vertx.core.Promise) Vertx(io.vertx.core.Vertx) KafkaMirrorMaker2MirrorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2MirrorSpec) KafkaMirrorMaker2Cluster(io.strimzi.operator.cluster.model.KafkaMirrorMaker2Cluster) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_CONNECTOR(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_CONNECTOR) KafkaClientAuthenticationScramSha512(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationScramSha512) Reconciliation(io.strimzi.operator.common.Reconciliation) Util(io.strimzi.operator.common.Util) KubernetesClient(io.fabric8.kubernetes.client.KubernetesClient) Comparator(java.util.Comparator) Collections(java.util.Collections) Deployment(io.fabric8.kubernetes.api.model.apps.Deployment) KafkaConnectorSpecBuilder(io.strimzi.api.kafka.model.KafkaConnectorSpecBuilder) Function(java.util.function.Function) Future(io.vertx.core.Future) CompositeFuture(io.vertx.core.CompositeFuture) KafkaMirrorMaker2MirrorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2MirrorSpec) Map(java.util.Map) HashMap(java.util.HashMap) Collections.emptyMap(java.util.Collections.emptyMap)

Example 3 with KafkaMirrorMaker2ConnectorSpec

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2ConnectorSpec in project strimzi-kafka-operator by strimzi.

the class KafkaMirrorMaker2AssemblyOperator method reconcileMirrorMaker2Connectors.

private Future<Void> reconcileMirrorMaker2Connectors(Reconciliation reconciliation, String host, KafkaConnectApi apiClient, KafkaMirrorMaker2 mirrorMaker2, KafkaMirrorMaker2MirrorSpec mirror, KafkaMirrorMaker2Cluster mirrorMaker2Cluster, KafkaMirrorMaker2Status mirrorMaker2Status, String desiredLogging) {
    String targetClusterAlias = mirror.getTargetCluster();
    String sourceClusterAlias = mirror.getSourceCluster();
    if (targetClusterAlias == null) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("targetCluster property is required"));
    } else if (sourceClusterAlias == null) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("sourceCluster property is required"));
    }
    List<KafkaMirrorMaker2ClusterSpec> clusters = ModelUtils.asListOrEmptyList(mirrorMaker2.getSpec().getClusters());
    Map<String, KafkaMirrorMaker2ClusterSpec> clusterMap = clusters.stream().filter(cluster -> targetClusterAlias.equals(cluster.getAlias()) || sourceClusterAlias.equals(cluster.getAlias())).collect(Collectors.toMap(KafkaMirrorMaker2ClusterSpec::getAlias, Function.identity()));
    if (!clusterMap.containsKey(targetClusterAlias)) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("targetCluster with alias " + mirror.getTargetCluster() + " cannot be found in the list of clusters at spec.clusters"));
    } else if (!clusterMap.containsKey(sourceClusterAlias)) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("sourceCluster with alias " + mirror.getSourceCluster() + " cannot be found in the list of clusters at spec.clusters"));
    }
    return CompositeFuture.join(MIRRORMAKER2_CONNECTORS.entrySet().stream().filter(// filter out non-existent connectors
    entry -> entry.getValue().apply(mirror) != null).map(entry -> {
        String connectorName = sourceClusterAlias + "->" + targetClusterAlias + entry.getKey();
        String className = MIRRORMAKER2_CONNECTOR_PACKAGE + entry.getKey();
        KafkaMirrorMaker2ConnectorSpec mm2ConnectorSpec = entry.getValue().apply(mirror);
        KafkaConnectorSpec connectorSpec = new KafkaConnectorSpecBuilder().withClassName(className).withConfig(mm2ConnectorSpec.getConfig()).withPause(mm2ConnectorSpec.getPause()).withTasksMax(mm2ConnectorSpec.getTasksMax()).build();
        prepareMirrorMaker2ConnectorConfig(reconciliation, mirror, clusterMap.get(sourceClusterAlias), clusterMap.get(targetClusterAlias), connectorSpec, mirrorMaker2Cluster);
        LOGGER.debugCr(reconciliation, "creating/updating connector {} config: {}", connectorName, connectorSpec.getConfig());
        return reconcileMirrorMaker2Connector(reconciliation, mirrorMaker2, apiClient, host, connectorName, connectorSpec, mirrorMaker2Status);
    }).collect(Collectors.toList())).map((Void) null).compose(i -> apiClient.updateConnectLoggers(reconciliation, host, KafkaConnectCluster.REST_API_PORT, desiredLogging, mirrorMaker2Cluster.getDefaultLogConfig())).compose(i -> {
        boolean failedConnector = mirrorMaker2Status.getConnectors().stream().anyMatch(connector -> {
            Object state = ((Map) connector.getOrDefault("connector", emptyMap())).get("state");
            return "FAILED".equalsIgnoreCase(state.toString());
        });
        if (failedConnector) {
            return Future.failedFuture("One or more connectors are in FAILED state");
        } else {
            return Future.succeededFuture();
        }
    }).map((Void) null);
}
Also used : InvalidResourceException(io.strimzi.operator.cluster.model.InvalidResourceException) KafkaMirrorMaker2Builder(io.strimzi.api.kafka.model.KafkaMirrorMaker2Builder) Annotations(io.strimzi.operator.common.Annotations) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_TASK(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_TASK) Resource(io.fabric8.kubernetes.client.dsl.Resource) Matcher(java.util.regex.Matcher) Map(java.util.Map) KafkaConnectorSpec(io.strimzi.api.kafka.model.KafkaConnectorSpec) ReconciliationException(io.strimzi.operator.common.ReconciliationException) ResourceOperatorSupplier(io.strimzi.operator.cluster.operator.resource.ResourceOperatorSupplier) AbstractModel(io.strimzi.operator.cluster.model.AbstractModel) ModelUtils(io.strimzi.operator.cluster.model.ModelUtils) StatusUtils(io.strimzi.operator.common.operator.resource.StatusUtils) ANNO_STRIMZI_IO_RESTART_CONNECTOR(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) KafkaMirrorMaker2ConnectorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ConnectorSpec) KafkaMirrorMaker2Spec(io.strimzi.api.kafka.model.KafkaMirrorMaker2Spec) DeploymentOperator(io.strimzi.operator.common.operator.resource.DeploymentOperator) KafkaVersion(io.strimzi.operator.cluster.model.KafkaVersion) Collectors(java.util.stream.Collectors) Future(io.vertx.core.Future) Serializable(java.io.Serializable) KafkaMirrorMaker2Resources(io.strimzi.api.kafka.model.KafkaMirrorMaker2Resources) KafkaClientAuthenticationScramSha256(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationScramSha256) KafkaMirrorMaker2List(io.strimzi.api.kafka.KafkaMirrorMaker2List) KafkaMirrorMaker2(io.strimzi.api.kafka.model.KafkaMirrorMaker2) KafkaMirrorMaker2Status(io.strimzi.api.kafka.model.status.KafkaMirrorMaker2Status) List(java.util.List) KafkaConnectCluster(io.strimzi.operator.cluster.model.KafkaConnectCluster) Stream(java.util.stream.Stream) Condition(io.strimzi.api.kafka.model.status.Condition) PlatformFeaturesAvailability(io.strimzi.operator.PlatformFeaturesAvailability) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN) ClusterOperatorConfig(io.strimzi.operator.cluster.ClusterOperatorConfig) CustomResource(io.fabric8.kubernetes.client.CustomResource) AuthenticationUtils(io.strimzi.operator.cluster.model.AuthenticationUtils) HashMap(java.util.HashMap) KafkaClientAuthenticationPlain(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationPlain) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK) CompositeFuture(io.vertx.core.CompositeFuture) KafkaClientAuthenticationOAuth(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationOAuth) KafkaClientAuthenticationTls(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationTls) SaslConfigs(org.apache.kafka.common.config.SaslConfigs) SslConfigs(org.apache.kafka.common.config.SslConfigs) ReconciliationLogger(io.strimzi.operator.common.ReconciliationLogger) Collections.emptyMap(java.util.Collections.emptyMap) KafkaMirrorMaker2ClusterSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec) InvalidResourceException(io.strimzi.operator.cluster.model.InvalidResourceException) AdminClientConfig(org.apache.kafka.clients.admin.AdminClientConfig) Promise(io.vertx.core.Promise) Vertx(io.vertx.core.Vertx) KafkaMirrorMaker2MirrorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2MirrorSpec) KafkaMirrorMaker2Cluster(io.strimzi.operator.cluster.model.KafkaMirrorMaker2Cluster) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_CONNECTOR(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_CONNECTOR) KafkaClientAuthenticationScramSha512(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationScramSha512) Reconciliation(io.strimzi.operator.common.Reconciliation) Util(io.strimzi.operator.common.Util) KubernetesClient(io.fabric8.kubernetes.client.KubernetesClient) Comparator(java.util.Comparator) Collections(java.util.Collections) Deployment(io.fabric8.kubernetes.api.model.apps.Deployment) KafkaConnectorSpecBuilder(io.strimzi.api.kafka.model.KafkaConnectorSpecBuilder) KafkaConnectorSpecBuilder(io.strimzi.api.kafka.model.KafkaConnectorSpecBuilder) KafkaMirrorMaker2ClusterSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec) KafkaConnectorSpec(io.strimzi.api.kafka.model.KafkaConnectorSpec) KafkaMirrorMaker2ConnectorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ConnectorSpec)

Example 4 with KafkaMirrorMaker2ConnectorSpec

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2ConnectorSpec in project strimzi by strimzi.

the class KafkaMirrorMaker2AssemblyOperator method reconcileMirrorMaker2Connectors.

private Future<Void> reconcileMirrorMaker2Connectors(Reconciliation reconciliation, String host, KafkaConnectApi apiClient, KafkaMirrorMaker2 mirrorMaker2, KafkaMirrorMaker2MirrorSpec mirror, KafkaMirrorMaker2Cluster mirrorMaker2Cluster, KafkaMirrorMaker2Status mirrorMaker2Status, String desiredLogging) {
    String targetClusterAlias = mirror.getTargetCluster();
    String sourceClusterAlias = mirror.getSourceCluster();
    if (targetClusterAlias == null) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("targetCluster property is required"));
    } else if (sourceClusterAlias == null) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("sourceCluster property is required"));
    }
    List<KafkaMirrorMaker2ClusterSpec> clusters = ModelUtils.asListOrEmptyList(mirrorMaker2.getSpec().getClusters());
    Map<String, KafkaMirrorMaker2ClusterSpec> clusterMap = clusters.stream().filter(cluster -> targetClusterAlias.equals(cluster.getAlias()) || sourceClusterAlias.equals(cluster.getAlias())).collect(Collectors.toMap(KafkaMirrorMaker2ClusterSpec::getAlias, Function.identity()));
    if (!clusterMap.containsKey(targetClusterAlias)) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("targetCluster with alias " + mirror.getTargetCluster() + " cannot be found in the list of clusters at spec.clusters"));
    } else if (!clusterMap.containsKey(sourceClusterAlias)) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("sourceCluster with alias " + mirror.getSourceCluster() + " cannot be found in the list of clusters at spec.clusters"));
    }
    return CompositeFuture.join(MIRRORMAKER2_CONNECTORS.entrySet().stream().filter(// filter out non-existent connectors
    entry -> entry.getValue().apply(mirror) != null).map(entry -> {
        String connectorName = sourceClusterAlias + "->" + targetClusterAlias + entry.getKey();
        String className = MIRRORMAKER2_CONNECTOR_PACKAGE + entry.getKey();
        KafkaMirrorMaker2ConnectorSpec mm2ConnectorSpec = entry.getValue().apply(mirror);
        KafkaConnectorSpec connectorSpec = new KafkaConnectorSpecBuilder().withClassName(className).withConfig(mm2ConnectorSpec.getConfig()).withPause(mm2ConnectorSpec.getPause()).withTasksMax(mm2ConnectorSpec.getTasksMax()).build();
        prepareMirrorMaker2ConnectorConfig(reconciliation, mirror, clusterMap.get(sourceClusterAlias), clusterMap.get(targetClusterAlias), connectorSpec, mirrorMaker2Cluster);
        LOGGER.debugCr(reconciliation, "creating/updating connector {} config: {}", connectorName, connectorSpec.getConfig());
        return reconcileMirrorMaker2Connector(reconciliation, mirrorMaker2, apiClient, host, connectorName, connectorSpec, mirrorMaker2Status);
    }).collect(Collectors.toList())).map((Void) null).compose(i -> apiClient.updateConnectLoggers(reconciliation, host, KafkaConnectCluster.REST_API_PORT, desiredLogging, mirrorMaker2Cluster.getDefaultLogConfig())).compose(i -> {
        boolean failedConnector = mirrorMaker2Status.getConnectors().stream().anyMatch(connector -> {
            Object state = ((Map) connector.getOrDefault("connector", emptyMap())).get("state");
            return "FAILED".equalsIgnoreCase(state.toString());
        });
        if (failedConnector) {
            return Future.failedFuture("One or more connectors are in FAILED state");
        } else {
            return Future.succeededFuture();
        }
    }).map((Void) null);
}
Also used : InvalidResourceException(io.strimzi.operator.cluster.model.InvalidResourceException) KafkaMirrorMaker2Builder(io.strimzi.api.kafka.model.KafkaMirrorMaker2Builder) Annotations(io.strimzi.operator.common.Annotations) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_TASK(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_TASK) Resource(io.fabric8.kubernetes.client.dsl.Resource) Matcher(java.util.regex.Matcher) Map(java.util.Map) KafkaConnectorSpec(io.strimzi.api.kafka.model.KafkaConnectorSpec) ReconciliationException(io.strimzi.operator.common.ReconciliationException) ResourceOperatorSupplier(io.strimzi.operator.cluster.operator.resource.ResourceOperatorSupplier) AbstractModel(io.strimzi.operator.cluster.model.AbstractModel) ModelUtils(io.strimzi.operator.cluster.model.ModelUtils) StatusUtils(io.strimzi.operator.common.operator.resource.StatusUtils) ANNO_STRIMZI_IO_RESTART_CONNECTOR(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) KafkaMirrorMaker2ConnectorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ConnectorSpec) KafkaMirrorMaker2Spec(io.strimzi.api.kafka.model.KafkaMirrorMaker2Spec) DeploymentOperator(io.strimzi.operator.common.operator.resource.DeploymentOperator) KafkaVersion(io.strimzi.operator.cluster.model.KafkaVersion) Collectors(java.util.stream.Collectors) Future(io.vertx.core.Future) Serializable(java.io.Serializable) KafkaMirrorMaker2Resources(io.strimzi.api.kafka.model.KafkaMirrorMaker2Resources) KafkaClientAuthenticationScramSha256(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationScramSha256) KafkaMirrorMaker2List(io.strimzi.api.kafka.KafkaMirrorMaker2List) KafkaMirrorMaker2(io.strimzi.api.kafka.model.KafkaMirrorMaker2) KafkaMirrorMaker2Status(io.strimzi.api.kafka.model.status.KafkaMirrorMaker2Status) List(java.util.List) KafkaConnectCluster(io.strimzi.operator.cluster.model.KafkaConnectCluster) Stream(java.util.stream.Stream) Condition(io.strimzi.api.kafka.model.status.Condition) PlatformFeaturesAvailability(io.strimzi.operator.PlatformFeaturesAvailability) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN) ClusterOperatorConfig(io.strimzi.operator.cluster.ClusterOperatorConfig) CustomResource(io.fabric8.kubernetes.client.CustomResource) AuthenticationUtils(io.strimzi.operator.cluster.model.AuthenticationUtils) HashMap(java.util.HashMap) KafkaClientAuthenticationPlain(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationPlain) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK) CompositeFuture(io.vertx.core.CompositeFuture) KafkaClientAuthenticationOAuth(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationOAuth) KafkaClientAuthenticationTls(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationTls) SaslConfigs(org.apache.kafka.common.config.SaslConfigs) SslConfigs(org.apache.kafka.common.config.SslConfigs) ReconciliationLogger(io.strimzi.operator.common.ReconciliationLogger) Collections.emptyMap(java.util.Collections.emptyMap) KafkaMirrorMaker2ClusterSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec) InvalidResourceException(io.strimzi.operator.cluster.model.InvalidResourceException) AdminClientConfig(org.apache.kafka.clients.admin.AdminClientConfig) Promise(io.vertx.core.Promise) Vertx(io.vertx.core.Vertx) KafkaMirrorMaker2MirrorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2MirrorSpec) KafkaMirrorMaker2Cluster(io.strimzi.operator.cluster.model.KafkaMirrorMaker2Cluster) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_CONNECTOR(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_CONNECTOR) KafkaClientAuthenticationScramSha512(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationScramSha512) Reconciliation(io.strimzi.operator.common.Reconciliation) Util(io.strimzi.operator.common.Util) KubernetesClient(io.fabric8.kubernetes.client.KubernetesClient) Comparator(java.util.Comparator) Collections(java.util.Collections) Deployment(io.fabric8.kubernetes.api.model.apps.Deployment) KafkaConnectorSpecBuilder(io.strimzi.api.kafka.model.KafkaConnectorSpecBuilder) KafkaConnectorSpecBuilder(io.strimzi.api.kafka.model.KafkaConnectorSpecBuilder) KafkaMirrorMaker2ClusterSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec) KafkaConnectorSpec(io.strimzi.api.kafka.model.KafkaConnectorSpec) KafkaMirrorMaker2ConnectorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ConnectorSpec)

Aggregations

Deployment (io.fabric8.kubernetes.api.model.apps.Deployment)4 CustomResource (io.fabric8.kubernetes.client.CustomResource)4 KubernetesClient (io.fabric8.kubernetes.client.KubernetesClient)4 Resource (io.fabric8.kubernetes.client.dsl.Resource)4 KafkaMirrorMaker2List (io.strimzi.api.kafka.KafkaMirrorMaker2List)4 CertSecretSource (io.strimzi.api.kafka.model.CertSecretSource)4 KafkaConnectorSpec (io.strimzi.api.kafka.model.KafkaConnectorSpec)4 KafkaConnectorSpecBuilder (io.strimzi.api.kafka.model.KafkaConnectorSpecBuilder)4 KafkaMirrorMaker2 (io.strimzi.api.kafka.model.KafkaMirrorMaker2)4 KafkaMirrorMaker2Builder (io.strimzi.api.kafka.model.KafkaMirrorMaker2Builder)4 KafkaMirrorMaker2ClusterSpec (io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec)4 KafkaMirrorMaker2ConnectorSpec (io.strimzi.api.kafka.model.KafkaMirrorMaker2ConnectorSpec)4 KafkaMirrorMaker2MirrorSpec (io.strimzi.api.kafka.model.KafkaMirrorMaker2MirrorSpec)4 KafkaMirrorMaker2Resources (io.strimzi.api.kafka.model.KafkaMirrorMaker2Resources)4 KafkaMirrorMaker2Spec (io.strimzi.api.kafka.model.KafkaMirrorMaker2Spec)4 KafkaClientAuthenticationOAuth (io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationOAuth)4 KafkaClientAuthenticationPlain (io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationPlain)4 KafkaClientAuthenticationScramSha256 (io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationScramSha256)4 KafkaClientAuthenticationScramSha512 (io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationScramSha512)4 KafkaClientAuthenticationTls (io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationTls)4