Examples with KafkaMirrorMaker2ClusterSpec io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec used on opensource projects

Example 16 with KafkaMirrorMaker2ClusterSpec

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec in project strimzi by strimzi.

the class KafkaMirrorMaker2Cluster method getEnvVars.

@SuppressWarnings({ "checkstyle:CyclomaticComplexity", "checkstyle:NPathComplexity" })
@Override
protected List<EnvVar> getEnvVars() {
    List<EnvVar> varList = super.getEnvVars();
    final StringBuilder clusterAliases = new StringBuilder();
    final StringBuilder clustersTrustedCerts = new StringBuilder();
    boolean hasClusterWithTls = false;
    final StringBuilder clustersTlsAuthCerts = new StringBuilder();
    final StringBuilder clustersTlsAuthKeys = new StringBuilder();
    final StringBuilder clustersSaslPasswordFiles = new StringBuilder();
    boolean hasClusterOauthTrustedCerts = false;
    final StringBuilder clustersOauthClientSecrets = new StringBuilder();
    final StringBuilder clustersOauthAccessTokens = new StringBuilder();
    final StringBuilder clustersOauthRefreshTokens = new StringBuilder();
    for (KafkaMirrorMaker2ClusterSpec mirrorMaker2Cluster : clusters) {
        String clusterAlias = mirrorMaker2Cluster.getAlias();
        if (clusterAliases.length() > 0) {
            clusterAliases.append(";");
        }
        clusterAliases.append(clusterAlias);
        if (mirrorMaker2Cluster.getTls() != null) {
            hasClusterWithTls = true;
        }
        getClusterTrustedCerts(clustersTrustedCerts, mirrorMaker2Cluster, clusterAlias);
        KafkaClientAuthentication authentication = mirrorMaker2Cluster.getAuthentication();
        if (authentication != null) {
            if (authentication instanceof KafkaClientAuthenticationTls) {
                KafkaClientAuthenticationTls tlsAuth = (KafkaClientAuthenticationTls) authentication;
                if (tlsAuth.getCertificateAndKey() != null) {
                    appendCluster(clustersTlsAuthCerts, clusterAlias, () -> tlsAuth.getCertificateAndKey().getSecretName() + "/" + tlsAuth.getCertificateAndKey().getCertificate());
                    appendCluster(clustersTlsAuthKeys, clusterAlias, () -> tlsAuth.getCertificateAndKey().getSecretName() + "/" + tlsAuth.getCertificateAndKey().getKey());
                }
            } else if (authentication instanceof KafkaClientAuthenticationPlain) {
                KafkaClientAuthenticationPlain passwordAuth = (KafkaClientAuthenticationPlain) authentication;
                appendClusterPasswordSecretSource(clustersSaslPasswordFiles, clusterAlias, passwordAuth.getPasswordSecret());
            } else if (authentication instanceof KafkaClientAuthenticationScram) {
                KafkaClientAuthenticationScram passwordAuth = (KafkaClientAuthenticationScram) authentication;
                appendClusterPasswordSecretSource(clustersSaslPasswordFiles, clusterAlias, passwordAuth.getPasswordSecret());
            } else if (authentication instanceof KafkaClientAuthenticationOAuth) {
                KafkaClientAuthenticationOAuth oauth = (KafkaClientAuthenticationOAuth) authentication;
                if (oauth.getTlsTrustedCertificates() != null && !oauth.getTlsTrustedCertificates().isEmpty()) {
                    hasClusterOauthTrustedCerts = true;
                }
                appendClusterOAuthSecretSource(clustersOauthClientSecrets, clusterAlias, oauth.getClientSecret());
                appendClusterOAuthSecretSource(clustersOauthAccessTokens, clusterAlias, oauth.getAccessToken());
                appendClusterOAuthSecretSource(clustersOauthRefreshTokens, clusterAlias, oauth.getRefreshToken());
            }
        }
    }
    varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_CLUSTERS, clusterAliases.toString()));
    if (hasClusterWithTls) {
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_TLS_CLUSTERS, "true"));
    }
    if (clustersTrustedCerts.length() > 0) {
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_TRUSTED_CERTS_CLUSTERS, clustersTrustedCerts.toString()));
    }
    if (clustersTlsAuthCerts.length() > 0 || clustersTlsAuthKeys.length() > 0) {
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_TLS_AUTH_CLUSTERS, "true"));
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_TLS_AUTH_CERTS_CLUSTERS, clustersTlsAuthCerts.toString()));
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_TLS_AUTH_KEYS_CLUSTERS, clustersTlsAuthKeys.toString()));
    }
    if (clustersSaslPasswordFiles.length() > 0) {
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_SASL_PASSWORD_FILES_CLUSTERS, clustersSaslPasswordFiles.toString()));
    }
    if (hasClusterOauthTrustedCerts) {
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_OAUTH_TRUSTED_CERTS, "true"));
    }
    if (clustersOauthClientSecrets.length() > 0) {
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_OAUTH_CLIENT_SECRETS_CLUSTERS, clustersOauthClientSecrets.toString()));
    }
    if (clustersOauthAccessTokens.length() > 0) {
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_OAUTH_ACCESS_TOKENS_CLUSTERS, clustersOauthAccessTokens.toString()));
    }
    if (clustersOauthRefreshTokens.length() > 0) {
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_OAUTH_REFRESH_TOKENS_CLUSTERS, clustersOauthRefreshTokens.toString()));
    }
    if (javaSystemProperties != null) {
        varList.add(buildEnvVar(ENV_VAR_STRIMZI_JAVA_SYSTEM_PROPERTIES, ModelUtils.getJavaSystemPropertiesToString(javaSystemProperties)));
    }
    return varList;
}

Example 17 with KafkaMirrorMaker2ClusterSpec

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec in project strimzi by strimzi.

the class KafkaMirrorMaker2Cluster method getVolumes.

@Override
protected List<Volume> getVolumes(boolean isOpenShift) {
    List<Volume> volumeList = super.getVolumes(isOpenShift);
    for (KafkaMirrorMaker2ClusterSpec mirrorMaker2Cluster : clusters) {
        String alias = mirrorMaker2Cluster.getAlias();
        ClientTls tls = mirrorMaker2Cluster.getTls();
        if (tls != null) {
            VolumeUtils.createSecretVolume(volumeList, tls.getTrustedCertificates(), isOpenShift, alias);
        }
        AuthenticationUtils.configureClientAuthenticationVolumes(mirrorMaker2Cluster.getAuthentication(), volumeList, mirrorMaker2Cluster.getAlias() + "-oauth-certs", isOpenShift, mirrorMaker2Cluster.getAlias() + '-', true);
    }
    return volumeList;
}

Example 18 with KafkaMirrorMaker2ClusterSpec

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec in project strimzi-kafka-operator by strimzi.

the class KafkaMirrorMaker2Cluster method getEnvVars.

@SuppressWarnings({ "checkstyle:CyclomaticComplexity", "checkstyle:NPathComplexity" })
@Override
protected List<EnvVar> getEnvVars() {
    List<EnvVar> varList = super.getEnvVars();
    final StringBuilder clusterAliases = new StringBuilder();
    final StringBuilder clustersTrustedCerts = new StringBuilder();
    boolean hasClusterWithTls = false;
    final StringBuilder clustersTlsAuthCerts = new StringBuilder();
    final StringBuilder clustersTlsAuthKeys = new StringBuilder();
    final StringBuilder clustersSaslPasswordFiles = new StringBuilder();
    boolean hasClusterOauthTrustedCerts = false;
    final StringBuilder clustersOauthClientSecrets = new StringBuilder();
    final StringBuilder clustersOauthAccessTokens = new StringBuilder();
    final StringBuilder clustersOauthRefreshTokens = new StringBuilder();
    for (KafkaMirrorMaker2ClusterSpec mirrorMaker2Cluster : clusters) {
        String clusterAlias = mirrorMaker2Cluster.getAlias();
        if (clusterAliases.length() > 0) {
            clusterAliases.append(";");
        }
        clusterAliases.append(clusterAlias);
        if (mirrorMaker2Cluster.getTls() != null) {
            hasClusterWithTls = true;
        }
        getClusterTrustedCerts(clustersTrustedCerts, mirrorMaker2Cluster, clusterAlias);
        KafkaClientAuthentication authentication = mirrorMaker2Cluster.getAuthentication();
        if (authentication != null) {
            if (authentication instanceof KafkaClientAuthenticationTls) {
                KafkaClientAuthenticationTls tlsAuth = (KafkaClientAuthenticationTls) authentication;
                if (tlsAuth.getCertificateAndKey() != null) {
                    appendCluster(clustersTlsAuthCerts, clusterAlias, () -> tlsAuth.getCertificateAndKey().getSecretName() + "/" + tlsAuth.getCertificateAndKey().getCertificate());
                    appendCluster(clustersTlsAuthKeys, clusterAlias, () -> tlsAuth.getCertificateAndKey().getSecretName() + "/" + tlsAuth.getCertificateAndKey().getKey());
                }
            } else if (authentication instanceof KafkaClientAuthenticationPlain) {
                KafkaClientAuthenticationPlain passwordAuth = (KafkaClientAuthenticationPlain) authentication;
                appendClusterPasswordSecretSource(clustersSaslPasswordFiles, clusterAlias, passwordAuth.getPasswordSecret());
            } else if (authentication instanceof KafkaClientAuthenticationScram) {
                KafkaClientAuthenticationScram passwordAuth = (KafkaClientAuthenticationScram) authentication;
                appendClusterPasswordSecretSource(clustersSaslPasswordFiles, clusterAlias, passwordAuth.getPasswordSecret());
            } else if (authentication instanceof KafkaClientAuthenticationOAuth) {
                KafkaClientAuthenticationOAuth oauth = (KafkaClientAuthenticationOAuth) authentication;
                if (oauth.getTlsTrustedCertificates() != null && !oauth.getTlsTrustedCertificates().isEmpty()) {
                    hasClusterOauthTrustedCerts = true;
                }
                appendClusterOAuthSecretSource(clustersOauthClientSecrets, clusterAlias, oauth.getClientSecret());
                appendClusterOAuthSecretSource(clustersOauthAccessTokens, clusterAlias, oauth.getAccessToken());
                appendClusterOAuthSecretSource(clustersOauthRefreshTokens, clusterAlias, oauth.getRefreshToken());
            }
        }
    }
    varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_CLUSTERS, clusterAliases.toString()));
    if (hasClusterWithTls) {
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_TLS_CLUSTERS, "true"));
    }
    if (clustersTrustedCerts.length() > 0) {
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_TRUSTED_CERTS_CLUSTERS, clustersTrustedCerts.toString()));
    }
    if (clustersTlsAuthCerts.length() > 0 || clustersTlsAuthKeys.length() > 0) {
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_TLS_AUTH_CLUSTERS, "true"));
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_TLS_AUTH_CERTS_CLUSTERS, clustersTlsAuthCerts.toString()));
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_TLS_AUTH_KEYS_CLUSTERS, clustersTlsAuthKeys.toString()));
    }
    if (clustersSaslPasswordFiles.length() > 0) {
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_SASL_PASSWORD_FILES_CLUSTERS, clustersSaslPasswordFiles.toString()));
    }
    if (hasClusterOauthTrustedCerts) {
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_OAUTH_TRUSTED_CERTS, "true"));
    }
    if (clustersOauthClientSecrets.length() > 0) {
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_OAUTH_CLIENT_SECRETS_CLUSTERS, clustersOauthClientSecrets.toString()));
    }
    if (clustersOauthAccessTokens.length() > 0) {
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_OAUTH_ACCESS_TOKENS_CLUSTERS, clustersOauthAccessTokens.toString()));
    }
    if (clustersOauthRefreshTokens.length() > 0) {
        varList.add(buildEnvVar(ENV_VAR_KAFKA_MIRRORMAKER_2_OAUTH_REFRESH_TOKENS_CLUSTERS, clustersOauthRefreshTokens.toString()));
    }
    if (javaSystemProperties != null) {
        varList.add(buildEnvVar(ENV_VAR_STRIMZI_JAVA_SYSTEM_PROPERTIES, ModelUtils.getJavaSystemPropertiesToString(javaSystemProperties)));
    }
    return varList;
}

Example 19 with KafkaMirrorMaker2ClusterSpec

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec in project strimzi-kafka-operator by strimzi.

the class KafkaMirrorMaker2Cluster method fromCrd.

/**
 * Creates instance of KafkaMirrorMaker2Cluster from CRD definition.
 *
 * @param reconciliation    The reconciliation
 * @param kafkaMirrorMaker2 The Custom Resource based on which the cluster model should be created.
 * @param versions The image versions for MirrorMaker 2.0 clusters.
 * @return The MirrorMaker 2.0 cluster model.
 */
public static KafkaMirrorMaker2Cluster fromCrd(Reconciliation reconciliation, KafkaMirrorMaker2 kafkaMirrorMaker2, KafkaVersion.Lookup versions) {
    KafkaMirrorMaker2Cluster cluster = new KafkaMirrorMaker2Cluster(reconciliation, kafkaMirrorMaker2);
    KafkaMirrorMaker2Spec spec = kafkaMirrorMaker2.getSpec();
    cluster.setOwnerReference(kafkaMirrorMaker2);
    cluster.setImage(versions.kafkaMirrorMaker2Version(spec.getImage(), spec.getVersion()));
    List<KafkaMirrorMaker2ClusterSpec> clustersList = ModelUtils.asListOrEmptyList(spec.getClusters());
    cluster.setClusters(clustersList);
    KafkaMirrorMaker2ClusterSpec connectCluster = new KafkaMirrorMaker2ClusterSpecBuilder().build();
    String connectClusterAlias = spec.getConnectCluster();
    if (connectClusterAlias != null) {
        connectCluster = clustersList.stream().filter(clustersListItem -> spec.getConnectCluster().equals(clustersListItem.getAlias())).findFirst().orElseThrow(() -> new InvalidResourceException("connectCluster with alias " + connectClusterAlias + " cannot be found in the list of clusters at spec.clusters"));
    }
    cluster.setConfiguration(new KafkaMirrorMaker2Configuration(reconciliation, connectCluster.getConfig().entrySet()));
    KafkaMirrorMaker2Cluster mm2 = fromSpec(reconciliation, buildKafkaConnectSpec(spec, connectCluster), versions, cluster);
    mm2.templatePodLabels = Util.mergeLabelsOrAnnotations(mm2.templatePodLabels, DEFAULT_POD_LABELS);
    return mm2;
}

Example 20 with KafkaMirrorMaker2ClusterSpec

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec in project strimzi-kafka-operator by strimzi.

the class KafkaMirrorMaker2AssemblyOperator method reconcileMirrorMaker2Connectors.

private Future<Void> reconcileMirrorMaker2Connectors(Reconciliation reconciliation, String host, KafkaConnectApi apiClient, KafkaMirrorMaker2 mirrorMaker2, KafkaMirrorMaker2MirrorSpec mirror, KafkaMirrorMaker2Cluster mirrorMaker2Cluster, KafkaMirrorMaker2Status mirrorMaker2Status, String desiredLogging) {
    String targetClusterAlias = mirror.getTargetCluster();
    String sourceClusterAlias = mirror.getSourceCluster();
    if (targetClusterAlias == null) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("targetCluster property is required"));
    } else if (sourceClusterAlias == null) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("sourceCluster property is required"));
    }
    List<KafkaMirrorMaker2ClusterSpec> clusters = ModelUtils.asListOrEmptyList(mirrorMaker2.getSpec().getClusters());
    Map<String, KafkaMirrorMaker2ClusterSpec> clusterMap = clusters.stream().filter(cluster -> targetClusterAlias.equals(cluster.getAlias()) || sourceClusterAlias.equals(cluster.getAlias())).collect(Collectors.toMap(KafkaMirrorMaker2ClusterSpec::getAlias, Function.identity()));
    if (!clusterMap.containsKey(targetClusterAlias)) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("targetCluster with alias " + mirror.getTargetCluster() + " cannot be found in the list of clusters at spec.clusters"));
    } else if (!clusterMap.containsKey(sourceClusterAlias)) {
        return maybeUpdateMirrorMaker2Status(reconciliation, mirrorMaker2, new InvalidResourceException("sourceCluster with alias " + mirror.getSourceCluster() + " cannot be found in the list of clusters at spec.clusters"));
    }
    return CompositeFuture.join(MIRRORMAKER2_CONNECTORS.entrySet().stream().filter(// filter out non-existent connectors
    entry -> entry.getValue().apply(mirror) != null).map(entry -> {
        String connectorName = sourceClusterAlias + "->" + targetClusterAlias + entry.getKey();
        String className = MIRRORMAKER2_CONNECTOR_PACKAGE + entry.getKey();
        KafkaMirrorMaker2ConnectorSpec mm2ConnectorSpec = entry.getValue().apply(mirror);
        KafkaConnectorSpec connectorSpec = new KafkaConnectorSpecBuilder().withClassName(className).withConfig(mm2ConnectorSpec.getConfig()).withPause(mm2ConnectorSpec.getPause()).withTasksMax(mm2ConnectorSpec.getTasksMax()).build();
        prepareMirrorMaker2ConnectorConfig(reconciliation, mirror, clusterMap.get(sourceClusterAlias), clusterMap.get(targetClusterAlias), connectorSpec, mirrorMaker2Cluster);
        LOGGER.debugCr(reconciliation, "creating/updating connector {} config: {}", connectorName, connectorSpec.getConfig());
        return reconcileMirrorMaker2Connector(reconciliation, mirrorMaker2, apiClient, host, connectorName, connectorSpec, mirrorMaker2Status);
    }).collect(Collectors.toList())).map((Void) null).compose(i -> apiClient.updateConnectLoggers(reconciliation, host, KafkaConnectCluster.REST_API_PORT, desiredLogging, mirrorMaker2Cluster.getDefaultLogConfig())).compose(i -> {
        boolean failedConnector = mirrorMaker2Status.getConnectors().stream().anyMatch(connector -> {
            Object state = ((Map) connector.getOrDefault("connector", emptyMap())).get("state");
            return "FAILED".equalsIgnoreCase(state.toString());
        });
        if (failedConnector) {
            return Future.failedFuture("One or more connectors are in FAILED state");
        } else {
            return Future.succeededFuture();
        }
    }).map((Void) null);
}