Search in sources :

Example 1 with KafkaConnectorSpec

use of io.strimzi.api.kafka.model.KafkaConnectorSpec in project strimzi-kafka-operator by strimzi.

the class KafkaMirrorMaker2AssemblyOperator method reconcileMirrorMaker2Connectors.

private Future<Void> reconcileMirrorMaker2Connectors(Reconciliation reconciliation, String host, KafkaConnectApi apiClient, KafkaMirrorMaker2 mirrorMaker2, KafkaMirrorMaker2MirrorSpec mirror, KafkaMirrorMaker2Cluster mirrorMaker2Cluster, KafkaMirrorMaker2Status mirrorMaker2Status, String desiredLogging) {
    String targetClusterAlias = mirror.getTargetCluster();
    String sourceClusterAlias = mirror.getSourceCluster();
    if (targetClusterAlias == null) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("targetCluster property is required"));
    } else if (sourceClusterAlias == null) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("sourceCluster property is required"));
    }
    List<KafkaMirrorMaker2ClusterSpec> clusters = ModelUtils.asListOrEmptyList(mirrorMaker2.getSpec().getClusters());
    Map<String, KafkaMirrorMaker2ClusterSpec> clusterMap = clusters.stream().filter(cluster -> targetClusterAlias.equals(cluster.getAlias()) || sourceClusterAlias.equals(cluster.getAlias())).collect(Collectors.toMap(KafkaMirrorMaker2ClusterSpec::getAlias, Function.identity()));
    if (!clusterMap.containsKey(targetClusterAlias)) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("targetCluster with alias " + mirror.getTargetCluster() + " cannot be found in the list of clusters at spec.clusters"));
    } else if (!clusterMap.containsKey(sourceClusterAlias)) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("sourceCluster with alias " + mirror.getSourceCluster() + " cannot be found in the list of clusters at spec.clusters"));
    }
    return CompositeFuture.join(MIRRORMAKER2_CONNECTORS.entrySet().stream().filter(// filter out non-existent connectors
    entry -> entry.getValue().apply(mirror) != null).map(entry -> {
        String connectorName = sourceClusterAlias + "->" + targetClusterAlias + entry.getKey();
        String className = MIRRORMAKER2_CONNECTOR_PACKAGE + entry.getKey();
        KafkaMirrorMaker2ConnectorSpec mm2ConnectorSpec = entry.getValue().apply(mirror);
        KafkaConnectorSpec connectorSpec = new KafkaConnectorSpecBuilder().withClassName(className).withConfig(mm2ConnectorSpec.getConfig()).withPause(mm2ConnectorSpec.getPause()).withTasksMax(mm2ConnectorSpec.getTasksMax()).build();
        prepareMirrorMaker2ConnectorConfig(reconciliation, mirror, clusterMap.get(sourceClusterAlias), clusterMap.get(targetClusterAlias), connectorSpec, mirrorMaker2Cluster);
        LOGGER.debugCr(reconciliation, "creating/updating connector {} config: {}", connectorName, connectorSpec.getConfig());
        return reconcileMirrorMaker2Connector(reconciliation, mirrorMaker2, apiClient, host, connectorName, connectorSpec, mirrorMaker2Status);
    }).collect(Collectors.toList())).map((Void) null).compose(i -> apiClient.updateConnectLoggers(reconciliation, host, KafkaConnectCluster.REST_API_PORT, desiredLogging, mirrorMaker2Cluster.getDefaultLogConfig())).compose(i -> {
        boolean failedConnector = mirrorMaker2Status.getConnectors().stream().anyMatch(connector -> {
            Object state = ((Map) connector.getOrDefault("connector", emptyMap())).get("state");
            return "FAILED".equalsIgnoreCase(state.toString());
        });
        if (failedConnector) {
            return Future.failedFuture("One or more connectors are in FAILED state");
        } else {
            return Future.succeededFuture();
        }
    }).map((Void) null);
}
Also used : InvalidResourceException(io.strimzi.operator.cluster.model.InvalidResourceException) KafkaMirrorMaker2Builder(io.strimzi.api.kafka.model.KafkaMirrorMaker2Builder) Annotations(io.strimzi.operator.common.Annotations) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_TASK(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_TASK) Resource(io.fabric8.kubernetes.client.dsl.Resource) Matcher(java.util.regex.Matcher) Map(java.util.Map) KafkaConnectorSpec(io.strimzi.api.kafka.model.KafkaConnectorSpec) ReconciliationException(io.strimzi.operator.common.ReconciliationException) ResourceOperatorSupplier(io.strimzi.operator.cluster.operator.resource.ResourceOperatorSupplier) AbstractModel(io.strimzi.operator.cluster.model.AbstractModel) ModelUtils(io.strimzi.operator.cluster.model.ModelUtils) StatusUtils(io.strimzi.operator.common.operator.resource.StatusUtils) ANNO_STRIMZI_IO_RESTART_CONNECTOR(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) KafkaMirrorMaker2ConnectorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ConnectorSpec) KafkaMirrorMaker2Spec(io.strimzi.api.kafka.model.KafkaMirrorMaker2Spec) DeploymentOperator(io.strimzi.operator.common.operator.resource.DeploymentOperator) KafkaVersion(io.strimzi.operator.cluster.model.KafkaVersion) Collectors(java.util.stream.Collectors) Future(io.vertx.core.Future) Serializable(java.io.Serializable) KafkaMirrorMaker2Resources(io.strimzi.api.kafka.model.KafkaMirrorMaker2Resources) KafkaClientAuthenticationScramSha256(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationScramSha256) KafkaMirrorMaker2List(io.strimzi.api.kafka.KafkaMirrorMaker2List) KafkaMirrorMaker2(io.strimzi.api.kafka.model.KafkaMirrorMaker2) KafkaMirrorMaker2Status(io.strimzi.api.kafka.model.status.KafkaMirrorMaker2Status) List(java.util.List) KafkaConnectCluster(io.strimzi.operator.cluster.model.KafkaConnectCluster) Stream(java.util.stream.Stream) Condition(io.strimzi.api.kafka.model.status.Condition) PlatformFeaturesAvailability(io.strimzi.operator.PlatformFeaturesAvailability) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN) ClusterOperatorConfig(io.strimzi.operator.cluster.ClusterOperatorConfig) CustomResource(io.fabric8.kubernetes.client.CustomResource) AuthenticationUtils(io.strimzi.operator.cluster.model.AuthenticationUtils) HashMap(java.util.HashMap) KafkaClientAuthenticationPlain(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationPlain) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK) CompositeFuture(io.vertx.core.CompositeFuture) KafkaClientAuthenticationOAuth(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationOAuth) KafkaClientAuthenticationTls(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationTls) SaslConfigs(org.apache.kafka.common.config.SaslConfigs) SslConfigs(org.apache.kafka.common.config.SslConfigs) ReconciliationLogger(io.strimzi.operator.common.ReconciliationLogger) Collections.emptyMap(java.util.Collections.emptyMap) KafkaMirrorMaker2ClusterSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec) InvalidResourceException(io.strimzi.operator.cluster.model.InvalidResourceException) AdminClientConfig(org.apache.kafka.clients.admin.AdminClientConfig) Promise(io.vertx.core.Promise) Vertx(io.vertx.core.Vertx) KafkaMirrorMaker2MirrorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2MirrorSpec) KafkaMirrorMaker2Cluster(io.strimzi.operator.cluster.model.KafkaMirrorMaker2Cluster) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_CONNECTOR(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_CONNECTOR) KafkaClientAuthenticationScramSha512(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationScramSha512) Reconciliation(io.strimzi.operator.common.Reconciliation) Util(io.strimzi.operator.common.Util) KubernetesClient(io.fabric8.kubernetes.client.KubernetesClient) Comparator(java.util.Comparator) Collections(java.util.Collections) Deployment(io.fabric8.kubernetes.api.model.apps.Deployment) KafkaConnectorSpecBuilder(io.strimzi.api.kafka.model.KafkaConnectorSpecBuilder) KafkaConnectorSpecBuilder(io.strimzi.api.kafka.model.KafkaConnectorSpecBuilder) KafkaMirrorMaker2ClusterSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec) KafkaConnectorSpec(io.strimzi.api.kafka.model.KafkaConnectorSpec) KafkaMirrorMaker2ConnectorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ConnectorSpec)

Example 2 with KafkaConnectorSpec

use of io.strimzi.api.kafka.model.KafkaConnectorSpec in project strimzi by strimzi.

the class KafkaMirrorMaker2AssemblyOperator method reconcileMirrorMaker2Connectors.

private Future<Void> reconcileMirrorMaker2Connectors(Reconciliation reconciliation, String host, KafkaConnectApi apiClient, KafkaMirrorMaker2 mirrorMaker2, KafkaMirrorMaker2MirrorSpec mirror, KafkaMirrorMaker2Cluster mirrorMaker2Cluster, KafkaMirrorMaker2Status mirrorMaker2Status, String desiredLogging) {
    String targetClusterAlias = mirror.getTargetCluster();
    String sourceClusterAlias = mirror.getSourceCluster();
    if (targetClusterAlias == null) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("targetCluster property is required"));
    } else if (sourceClusterAlias == null) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("sourceCluster property is required"));
    }
    List<KafkaMirrorMaker2ClusterSpec> clusters = ModelUtils.asListOrEmptyList(mirrorMaker2.getSpec().getClusters());
    Map<String, KafkaMirrorMaker2ClusterSpec> clusterMap = clusters.stream().filter(cluster -> targetClusterAlias.equals(cluster.getAlias()) || sourceClusterAlias.equals(cluster.getAlias())).collect(Collectors.toMap(KafkaMirrorMaker2ClusterSpec::getAlias, Function.identity()));
    if (!clusterMap.containsKey(targetClusterAlias)) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("targetCluster with alias " + mirror.getTargetCluster() + " cannot be found in the list of clusters at spec.clusters"));
    } else if (!clusterMap.containsKey(sourceClusterAlias)) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("sourceCluster with alias " + mirror.getSourceCluster() + " cannot be found in the list of clusters at spec.clusters"));
    }
    return CompositeFuture.join(MIRRORMAKER2_CONNECTORS.entrySet().stream().filter(// filter out non-existent connectors
    entry -> entry.getValue().apply(mirror) != null).map(entry -> {
        String connectorName = sourceClusterAlias + "->" + targetClusterAlias + entry.getKey();
        String className = MIRRORMAKER2_CONNECTOR_PACKAGE + entry.getKey();
        KafkaMirrorMaker2ConnectorSpec mm2ConnectorSpec = entry.getValue().apply(mirror);
        KafkaConnectorSpec connectorSpec = new KafkaConnectorSpecBuilder().withClassName(className).withConfig(mm2ConnectorSpec.getConfig()).withPause(mm2ConnectorSpec.getPause()).withTasksMax(mm2ConnectorSpec.getTasksMax()).build();
        prepareMirrorMaker2ConnectorConfig(reconciliation, mirror, clusterMap.get(sourceClusterAlias), clusterMap.get(targetClusterAlias), connectorSpec, mirrorMaker2Cluster);
        LOGGER.debugCr(reconciliation, "creating/updating connector {} config: {}", connectorName, connectorSpec.getConfig());
        return reconcileMirrorMaker2Connector(reconciliation, mirrorMaker2, apiClient, host, connectorName, connectorSpec, mirrorMaker2Status);
    }).collect(Collectors.toList())).map((Void) null).compose(i -> apiClient.updateConnectLoggers(reconciliation, host, KafkaConnectCluster.REST_API_PORT, desiredLogging, mirrorMaker2Cluster.getDefaultLogConfig())).compose(i -> {
        boolean failedConnector = mirrorMaker2Status.getConnectors().stream().anyMatch(connector -> {
            Object state = ((Map) connector.getOrDefault("connector", emptyMap())).get("state");
            return "FAILED".equalsIgnoreCase(state.toString());
        });
        if (failedConnector) {
            return Future.failedFuture("One or more connectors are in FAILED state");
        } else {
            return Future.succeededFuture();
        }
    }).map((Void) null);
}
Also used : InvalidResourceException(io.strimzi.operator.cluster.model.InvalidResourceException) KafkaMirrorMaker2Builder(io.strimzi.api.kafka.model.KafkaMirrorMaker2Builder) Annotations(io.strimzi.operator.common.Annotations) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_TASK(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_TASK) Resource(io.fabric8.kubernetes.client.dsl.Resource) Matcher(java.util.regex.Matcher) Map(java.util.Map) KafkaConnectorSpec(io.strimzi.api.kafka.model.KafkaConnectorSpec) ReconciliationException(io.strimzi.operator.common.ReconciliationException) ResourceOperatorSupplier(io.strimzi.operator.cluster.operator.resource.ResourceOperatorSupplier) AbstractModel(io.strimzi.operator.cluster.model.AbstractModel) ModelUtils(io.strimzi.operator.cluster.model.ModelUtils) StatusUtils(io.strimzi.operator.common.operator.resource.StatusUtils) ANNO_STRIMZI_IO_RESTART_CONNECTOR(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) KafkaMirrorMaker2ConnectorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ConnectorSpec) KafkaMirrorMaker2Spec(io.strimzi.api.kafka.model.KafkaMirrorMaker2Spec) DeploymentOperator(io.strimzi.operator.common.operator.resource.DeploymentOperator) KafkaVersion(io.strimzi.operator.cluster.model.KafkaVersion) Collectors(java.util.stream.Collectors) Future(io.vertx.core.Future) Serializable(java.io.Serializable) KafkaMirrorMaker2Resources(io.strimzi.api.kafka.model.KafkaMirrorMaker2Resources) KafkaClientAuthenticationScramSha256(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationScramSha256) KafkaMirrorMaker2List(io.strimzi.api.kafka.KafkaMirrorMaker2List) KafkaMirrorMaker2(io.strimzi.api.kafka.model.KafkaMirrorMaker2) KafkaMirrorMaker2Status(io.strimzi.api.kafka.model.status.KafkaMirrorMaker2Status) List(java.util.List) KafkaConnectCluster(io.strimzi.operator.cluster.model.KafkaConnectCluster) Stream(java.util.stream.Stream) Condition(io.strimzi.api.kafka.model.status.Condition) PlatformFeaturesAvailability(io.strimzi.operator.PlatformFeaturesAvailability) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN) ClusterOperatorConfig(io.strimzi.operator.cluster.ClusterOperatorConfig) CustomResource(io.fabric8.kubernetes.client.CustomResource) AuthenticationUtils(io.strimzi.operator.cluster.model.AuthenticationUtils) HashMap(java.util.HashMap) KafkaClientAuthenticationPlain(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationPlain) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK) CompositeFuture(io.vertx.core.CompositeFuture) KafkaClientAuthenticationOAuth(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationOAuth) KafkaClientAuthenticationTls(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationTls) SaslConfigs(org.apache.kafka.common.config.SaslConfigs) SslConfigs(org.apache.kafka.common.config.SslConfigs) ReconciliationLogger(io.strimzi.operator.common.ReconciliationLogger) Collections.emptyMap(java.util.Collections.emptyMap) KafkaMirrorMaker2ClusterSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec) InvalidResourceException(io.strimzi.operator.cluster.model.InvalidResourceException) AdminClientConfig(org.apache.kafka.clients.admin.AdminClientConfig) Promise(io.vertx.core.Promise) Vertx(io.vertx.core.Vertx) KafkaMirrorMaker2MirrorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2MirrorSpec) KafkaMirrorMaker2Cluster(io.strimzi.operator.cluster.model.KafkaMirrorMaker2Cluster) ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_CONNECTOR(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_RESTART_CONNECTOR_TASK_PATTERN_CONNECTOR) KafkaClientAuthenticationScramSha512(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationScramSha512) Reconciliation(io.strimzi.operator.common.Reconciliation) Util(io.strimzi.operator.common.Util) KubernetesClient(io.fabric8.kubernetes.client.KubernetesClient) Comparator(java.util.Comparator) Collections(java.util.Collections) Deployment(io.fabric8.kubernetes.api.model.apps.Deployment) KafkaConnectorSpecBuilder(io.strimzi.api.kafka.model.KafkaConnectorSpecBuilder) KafkaConnectorSpecBuilder(io.strimzi.api.kafka.model.KafkaConnectorSpecBuilder) KafkaMirrorMaker2ClusterSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec) KafkaConnectorSpec(io.strimzi.api.kafka.model.KafkaConnectorSpec) KafkaMirrorMaker2ConnectorSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ConnectorSpec)

Aggregations

Deployment (io.fabric8.kubernetes.api.model.apps.Deployment)2 CustomResource (io.fabric8.kubernetes.client.CustomResource)2 KubernetesClient (io.fabric8.kubernetes.client.KubernetesClient)2 Resource (io.fabric8.kubernetes.client.dsl.Resource)2 KafkaMirrorMaker2List (io.strimzi.api.kafka.KafkaMirrorMaker2List)2 CertSecretSource (io.strimzi.api.kafka.model.CertSecretSource)2 KafkaConnectorSpec (io.strimzi.api.kafka.model.KafkaConnectorSpec)2 KafkaConnectorSpecBuilder (io.strimzi.api.kafka.model.KafkaConnectorSpecBuilder)2 KafkaMirrorMaker2 (io.strimzi.api.kafka.model.KafkaMirrorMaker2)2 KafkaMirrorMaker2Builder (io.strimzi.api.kafka.model.KafkaMirrorMaker2Builder)2 KafkaMirrorMaker2ClusterSpec (io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec)2 KafkaMirrorMaker2ConnectorSpec (io.strimzi.api.kafka.model.KafkaMirrorMaker2ConnectorSpec)2 KafkaMirrorMaker2MirrorSpec (io.strimzi.api.kafka.model.KafkaMirrorMaker2MirrorSpec)2 KafkaMirrorMaker2Resources (io.strimzi.api.kafka.model.KafkaMirrorMaker2Resources)2 KafkaMirrorMaker2Spec (io.strimzi.api.kafka.model.KafkaMirrorMaker2Spec)2 KafkaClientAuthenticationOAuth (io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationOAuth)2 KafkaClientAuthenticationPlain (io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationPlain)2 KafkaClientAuthenticationScramSha256 (io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationScramSha256)2 KafkaClientAuthenticationScramSha512 (io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationScramSha512)2 KafkaClientAuthenticationTls (io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationTls)2