Examples with KafkaMirrorMaker2 io.strimzi.api.kafka.model.KafkaMirrorMaker2 used on opensource projects

Example 41 with KafkaMirrorMaker2

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2 in project strimzi by strimzi.

the class KafkaMirrorMaker2AssemblyOperator method createOrUpdate.

@Override
protected Future<KafkaMirrorMaker2Status> createOrUpdate(Reconciliation reconciliation, KafkaMirrorMaker2 kafkaMirrorMaker2) {
    KafkaMirrorMaker2Cluster mirrorMaker2Cluster;
    KafkaMirrorMaker2Status kafkaMirrorMaker2Status = new KafkaMirrorMaker2Status();
    try {
        mirrorMaker2Cluster = KafkaMirrorMaker2Cluster.fromCrd(reconciliation, kafkaMirrorMaker2, versions);
    } catch (Exception e) {
        LOGGER.warnCr(reconciliation, e);
        StatusUtils.setStatusConditionAndObservedGeneration(kafkaMirrorMaker2, kafkaMirrorMaker2Status, Future.failedFuture(e));
        return Future.failedFuture(new ReconciliationException(kafkaMirrorMaker2Status, e));
    }
    Promise<KafkaMirrorMaker2Status> createOrUpdatePromise = Promise.promise();
    String namespace = reconciliation.namespace();
    Map<String, String> annotations = new HashMap<>(1);
    final AtomicReference<String> desiredLogging = new AtomicReference<>();
    boolean mirrorMaker2HasZeroReplicas = mirrorMaker2Cluster.getReplicas() == 0;
    LOGGER.debugCr(reconciliation, "Updating Kafka MirrorMaker 2.0 cluster");
    connectServiceAccount(reconciliation, namespace, KafkaMirrorMaker2Resources.serviceAccountName(mirrorMaker2Cluster.getCluster()), mirrorMaker2Cluster).compose(i -> connectNetworkPolicy(reconciliation, namespace, mirrorMaker2Cluster, true)).compose(i -> deploymentOperations.scaleDown(reconciliation, namespace, mirrorMaker2Cluster.getName(), mirrorMaker2Cluster.getReplicas())).compose(i -> serviceOperations.reconcile(reconciliation, namespace, mirrorMaker2Cluster.getServiceName(), mirrorMaker2Cluster.generateService())).compose(i -> generateMetricsAndLoggingConfigMap(reconciliation, namespace, mirrorMaker2Cluster)).compose(logAndMetricsConfigMap -> {
        String logging = logAndMetricsConfigMap.getData().get(AbstractModel.ANCILLARY_CM_KEY_LOG_CONFIG);
        annotations.put(Annotations.ANNO_STRIMZI_LOGGING_DYNAMICALLY_UNCHANGEABLE_HASH, Util.stringHash(Util.getLoggingDynamicallyUnmodifiableEntries(logging)));
        desiredLogging.set(logging);
        return configMapOperations.reconcile(reconciliation, namespace, mirrorMaker2Cluster.getAncillaryConfigMapName(), logAndMetricsConfigMap);
    }).compose(i -> kafkaConnectJmxSecret(reconciliation, namespace, mirrorMaker2Cluster.getName(), mirrorMaker2Cluster)).compose(i -> pfa.hasPodDisruptionBudgetV1() ? podDisruptionBudgetOperator.reconcile(reconciliation, namespace, mirrorMaker2Cluster.getName(), mirrorMaker2Cluster.generatePodDisruptionBudget()) : Future.succeededFuture()).compose(i -> !pfa.hasPodDisruptionBudgetV1() ? podDisruptionBudgetV1Beta1Operator.reconcile(reconciliation, namespace, mirrorMaker2Cluster.getName(), mirrorMaker2Cluster.generatePodDisruptionBudgetV1Beta1()) : Future.succeededFuture()).compose(i -> generateAuthHash(namespace, kafkaMirrorMaker2.getSpec())).compose(hash -> {
        if (hash != null) {
            annotations.put(Annotations.ANNO_STRIMZI_AUTH_HASH, Integer.toString(hash));
        }
        Deployment deployment = mirrorMaker2Cluster.generateDeployment(annotations, pfa.isOpenshift(), imagePullPolicy, imagePullSecrets);
        return deploymentOperations.reconcile(reconciliation, namespace, mirrorMaker2Cluster.getName(), deployment);
    }).compose(i -> deploymentOperations.scaleUp(reconciliation, namespace, mirrorMaker2Cluster.getName(), mirrorMaker2Cluster.getReplicas())).compose(i -> deploymentOperations.waitForObserved(reconciliation, namespace, mirrorMaker2Cluster.getName(), 1_000, operationTimeoutMs)).compose(i -> mirrorMaker2HasZeroReplicas ? Future.succeededFuture() : deploymentOperations.readiness(reconciliation, namespace, mirrorMaker2Cluster.getName(), 1_000, operationTimeoutMs)).compose(i -> mirrorMaker2HasZeroReplicas ? Future.succeededFuture() : reconcileConnectors(reconciliation, kafkaMirrorMaker2, mirrorMaker2Cluster, kafkaMirrorMaker2Status, desiredLogging.get())).map((Void) null).onComplete(reconciliationResult -> {
        List<Condition> conditions = kafkaMirrorMaker2Status.getConditions();
        StatusUtils.setStatusConditionAndObservedGeneration(kafkaMirrorMaker2, kafkaMirrorMaker2Status, reconciliationResult);
        if (!mirrorMaker2HasZeroReplicas) {
            kafkaMirrorMaker2Status.setUrl(KafkaMirrorMaker2Resources.url(mirrorMaker2Cluster.getCluster(), namespace, KafkaMirrorMaker2Cluster.REST_API_PORT));
        }
        if (conditions != null && !conditions.isEmpty()) {
            kafkaMirrorMaker2Status.addConditions(conditions);
        }
        kafkaMirrorMaker2Status.setReplicas(mirrorMaker2Cluster.getReplicas());
        kafkaMirrorMaker2Status.setLabelSelector(mirrorMaker2Cluster.getSelectorLabels().toSelectorString());
        if (reconciliationResult.succeeded()) {
            createOrUpdatePromise.complete(kafkaMirrorMaker2Status);
        } else {
            createOrUpdatePromise.fail(new ReconciliationException(kafkaMirrorMaker2Status, reconciliationResult.cause()));
        }
    });
    return createOrUpdatePromise.future();
}

Example 42 with KafkaMirrorMaker2

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2 in project strimzi-kafka-operator by strimzi.

the class Crds method crd.

@SuppressWarnings({ "checkstyle:JavaNCSS" })
private static CustomResourceDefinition crd(Class<? extends CustomResource> cls) {
    String scope, plural, singular, group, kind, listKind;
    List<String> versions;
    CustomResourceSubresourceStatus status = null;
    if (cls.equals(Kafka.class)) {
        scope = Kafka.SCOPE;
        plural = Kafka.RESOURCE_PLURAL;
        singular = Kafka.RESOURCE_SINGULAR;
        group = Kafka.RESOURCE_GROUP;
        kind = Kafka.RESOURCE_KIND;
        listKind = Kafka.RESOURCE_LIST_KIND;
        versions = Kafka.VERSIONS;
        status = new CustomResourceSubresourceStatus();
    } else if (cls.equals(KafkaConnect.class)) {
        scope = KafkaConnect.SCOPE;
        plural = KafkaConnect.RESOURCE_PLURAL;
        singular = KafkaConnect.RESOURCE_SINGULAR;
        group = KafkaConnect.RESOURCE_GROUP;
        kind = KafkaConnect.RESOURCE_KIND;
        listKind = KafkaConnect.RESOURCE_LIST_KIND;
        versions = KafkaConnect.VERSIONS;
        status = new CustomResourceSubresourceStatus();
    } else if (cls.equals(KafkaTopic.class)) {
        scope = KafkaTopic.SCOPE;
        plural = KafkaTopic.RESOURCE_PLURAL;
        singular = KafkaTopic.RESOURCE_SINGULAR;
        group = KafkaTopic.RESOURCE_GROUP;
        kind = KafkaTopic.RESOURCE_KIND;
        listKind = KafkaTopic.RESOURCE_LIST_KIND;
        versions = KafkaTopic.VERSIONS;
    } else if (cls.equals(KafkaUser.class)) {
        scope = KafkaUser.SCOPE;
        plural = KafkaUser.RESOURCE_PLURAL;
        singular = KafkaUser.RESOURCE_SINGULAR;
        group = KafkaUser.RESOURCE_GROUP;
        kind = KafkaUser.RESOURCE_KIND;
        listKind = KafkaUser.RESOURCE_LIST_KIND;
        versions = KafkaUser.VERSIONS;
        status = new CustomResourceSubresourceStatus();
    } else if (cls.equals(KafkaMirrorMaker.class)) {
        scope = KafkaMirrorMaker.SCOPE;
        plural = KafkaMirrorMaker.RESOURCE_PLURAL;
        singular = KafkaMirrorMaker.RESOURCE_SINGULAR;
        group = KafkaMirrorMaker.RESOURCE_GROUP;
        kind = KafkaMirrorMaker.RESOURCE_KIND;
        listKind = KafkaMirrorMaker.RESOURCE_LIST_KIND;
        versions = KafkaMirrorMaker.VERSIONS;
        status = new CustomResourceSubresourceStatus();
    } else if (cls.equals(KafkaBridge.class)) {
        scope = KafkaBridge.SCOPE;
        plural = KafkaBridge.RESOURCE_PLURAL;
        singular = KafkaBridge.RESOURCE_SINGULAR;
        group = KafkaBridge.RESOURCE_GROUP;
        kind = KafkaBridge.RESOURCE_KIND;
        listKind = KafkaBridge.RESOURCE_LIST_KIND;
        versions = KafkaBridge.VERSIONS;
        status = new CustomResourceSubresourceStatus();
    } else if (cls.equals(KafkaConnector.class)) {
        scope = KafkaConnector.SCOPE;
        plural = KafkaConnector.RESOURCE_PLURAL;
        singular = KafkaConnector.RESOURCE_SINGULAR;
        group = KafkaConnector.RESOURCE_GROUP;
        kind = KafkaConnector.RESOURCE_KIND;
        listKind = KafkaConnector.RESOURCE_LIST_KIND;
        versions = KafkaConnector.VERSIONS;
        status = new CustomResourceSubresourceStatus();
    } else if (cls.equals(KafkaMirrorMaker2.class)) {
        scope = KafkaMirrorMaker2.SCOPE;
        plural = KafkaMirrorMaker2.RESOURCE_PLURAL;
        singular = KafkaMirrorMaker2.RESOURCE_SINGULAR;
        group = KafkaMirrorMaker2.RESOURCE_GROUP;
        kind = KafkaMirrorMaker2.RESOURCE_KIND;
        listKind = KafkaMirrorMaker2.RESOURCE_LIST_KIND;
        versions = KafkaMirrorMaker2.VERSIONS;
        status = new CustomResourceSubresourceStatus();
    } else if (cls.equals(KafkaRebalance.class)) {
        scope = KafkaRebalance.SCOPE;
        plural = KafkaRebalance.RESOURCE_PLURAL;
        singular = KafkaRebalance.RESOURCE_SINGULAR;
        group = KafkaRebalance.RESOURCE_GROUP;
        kind = KafkaRebalance.RESOURCE_KIND;
        listKind = KafkaRebalance.RESOURCE_LIST_KIND;
        versions = KafkaRebalance.VERSIONS;
        status = new CustomResourceSubresourceStatus();
    } else if (cls.equals(StrimziPodSet.class)) {
        scope = StrimziPodSet.SCOPE;
        plural = StrimziPodSet.RESOURCE_PLURAL;
        singular = StrimziPodSet.RESOURCE_SINGULAR;
        group = StrimziPodSet.RESOURCE_GROUP;
        kind = StrimziPodSet.RESOURCE_KIND;
        listKind = StrimziPodSet.RESOURCE_LIST_KIND;
        versions = StrimziPodSet.VERSIONS;
        status = new CustomResourceSubresourceStatus();
    } else {
        throw new RuntimeException();
    }
    List<CustomResourceDefinitionVersion> crVersions = new ArrayList<>(versions.size());
    for (String apiVersion : versions) {
        crVersions.add(new CustomResourceDefinitionVersionBuilder().withName(apiVersion).withNewSubresources().withStatus(status).endSubresources().withNewSchema().withNewOpenAPIV3Schema().withType("object").withXKubernetesPreserveUnknownFields(true).endOpenAPIV3Schema().endSchema().withStorage("v1beta2".equals(apiVersion)).withServed(true).build());
    }
    return new CustomResourceDefinitionBuilder().withNewMetadata().withName(plural + "." + group).endMetadata().withNewSpec().withScope(scope).withGroup(group).withVersions(crVersions).withNewNames().withSingular(singular).withPlural(plural).withKind(kind).withListKind(listKind).endNames().endSpec().build();
}

Example 43 with KafkaMirrorMaker2

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2 in project strimzi-kafka-operator by strimzi.

the class KafkaMirrorMaker2Cluster method fromCrd.

/**
 * Creates instance of KafkaMirrorMaker2Cluster from CRD definition.
 *
 * @param reconciliation    The reconciliation
 * @param kafkaMirrorMaker2 The Custom Resource based on which the cluster model should be created.
 * @param versions The image versions for MirrorMaker 2.0 clusters.
 * @return The MirrorMaker 2.0 cluster model.
 */
public static KafkaMirrorMaker2Cluster fromCrd(Reconciliation reconciliation, KafkaMirrorMaker2 kafkaMirrorMaker2, KafkaVersion.Lookup versions) {
    KafkaMirrorMaker2Cluster cluster = new KafkaMirrorMaker2Cluster(reconciliation, kafkaMirrorMaker2);
    KafkaMirrorMaker2Spec spec = kafkaMirrorMaker2.getSpec();
    cluster.setOwnerReference(kafkaMirrorMaker2);
    cluster.setImage(versions.kafkaMirrorMaker2Version(spec.getImage(), spec.getVersion()));
    List<KafkaMirrorMaker2ClusterSpec> clustersList = ModelUtils.asListOrEmptyList(spec.getClusters());
    cluster.setClusters(clustersList);
    KafkaMirrorMaker2ClusterSpec connectCluster = new KafkaMirrorMaker2ClusterSpecBuilder().build();
    String connectClusterAlias = spec.getConnectCluster();
    if (connectClusterAlias != null) {
        connectCluster = clustersList.stream().filter(clustersListItem -> spec.getConnectCluster().equals(clustersListItem.getAlias())).findFirst().orElseThrow(() -> new InvalidResourceException("connectCluster with alias " + connectClusterAlias + " cannot be found in the list of clusters at spec.clusters"));
    }
    cluster.setConfiguration(new KafkaMirrorMaker2Configuration(reconciliation, connectCluster.getConfig().entrySet()));
    KafkaMirrorMaker2Cluster mm2 = fromSpec(reconciliation, buildKafkaConnectSpec(spec, connectCluster), versions, cluster);
    mm2.templatePodLabels = Util.mergeLabelsOrAnnotations(mm2.templatePodLabels, DEFAULT_POD_LABELS);
    return mm2;
}

Example 44 with KafkaMirrorMaker2

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2 in project strimzi-kafka-operator by strimzi.

the class AbstractConnectOperator method maybeUpdateStatusCommon.

/**
 * Updates the Status field of the KafkaConnect or KafkaConnector CR. It diffs the desired status against the current status and calls
 * the update only when there is any difference in non-timestamp fields.
 *
 * @param resource The CR of KafkaConnect or KafkaConnector
 * @param reconciliation Reconciliation information
 * @param desiredStatus The KafkaConnectStatus or KafkaConnectorStatus which should be set
 *
 * @return
 */
protected <T extends CustomResource<?, S>, S extends Status, L extends CustomResourceList<T>> Future<Void> maybeUpdateStatusCommon(CrdOperator<KubernetesClient, T, L> resourceOperator, T resource, Reconciliation reconciliation, S desiredStatus, BiFunction<T, S, T> copyWithStatus) {
    Promise<Void> updateStatusPromise = Promise.promise();
    resourceOperator.getAsync(resource.getMetadata().getNamespace(), resource.getMetadata().getName()).onComplete(getRes -> {
        if (getRes.succeeded()) {
            T fetchedResource = getRes.result();
            if (fetchedResource != null) {
                if ((!(fetchedResource instanceof KafkaConnector)) && (!(fetchedResource instanceof KafkaMirrorMaker2))) {
                    LOGGER.warnCr(reconciliation, "{} {} needs to be upgraded from version {} to 'v1beta1' to use the status field", fetchedResource.getKind(), fetchedResource.getMetadata().getName(), fetchedResource.getApiVersion());
                    updateStatusPromise.complete();
                } else {
                    S currentStatus = fetchedResource.getStatus();
                    StatusDiff ksDiff = new StatusDiff(currentStatus, desiredStatus);
                    if (!ksDiff.isEmpty()) {
                        T resourceWithNewStatus = copyWithStatus.apply(fetchedResource, desiredStatus);
                        resourceOperator.updateStatusAsync(reconciliation, resourceWithNewStatus).onComplete(updateRes -> {
                            if (updateRes.succeeded()) {
                                LOGGER.debugCr(reconciliation, "Completed status update");
                                updateStatusPromise.complete();
                            } else {
                                LOGGER.errorCr(reconciliation, "Failed to update status", updateRes.cause());
                                updateStatusPromise.fail(updateRes.cause());
                            }
                        });
                    } else {
                        LOGGER.debugCr(reconciliation, "Status did not change");
                        updateStatusPromise.complete();
                    }
                }
            } else {
                LOGGER.errorCr(reconciliation, "Current {} resource not found", resource.getKind());
                updateStatusPromise.fail("Current " + resource.getKind() + " resource not found");
            }
        } else {
            LOGGER.errorCr(reconciliation, "Failed to get the current {} resource and its status", resource.getKind(), getRes.cause());
            updateStatusPromise.fail(getRes.cause());
        }
    });
    return updateStatusPromise.future();
}

Example 45 with KafkaMirrorMaker2

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2 in project strimzi-kafka-operator by strimzi.

the class KafkaMirrorMaker2AssemblyOperator method reconcileConnectors.

/**
 * Reconcile all the MirrorMaker 2.0 connectors selected by the given MirrorMaker 2.0 instance.
 * @param reconciliation The reconciliation
 * @param kafkaMirrorMaker2 The MirrorMaker 2.0
 * @return A future, failed if any of the connectors could not be reconciled.
 */
protected Future<Void> reconcileConnectors(Reconciliation reconciliation, KafkaMirrorMaker2 kafkaMirrorMaker2, KafkaMirrorMaker2Cluster mirrorMaker2Cluster, KafkaMirrorMaker2Status mirrorMaker2Status, String desiredLogging) {
    String mirrorMaker2Name = kafkaMirrorMaker2.getMetadata().getName();
    if (kafkaMirrorMaker2.getSpec() == null) {
        return maybeUpdateMirrorMaker2Status(reconciliation, kafkaMirrorMaker2, new InvalidResourceException("spec property is required"));
    }
    List<KafkaMirrorMaker2MirrorSpec> mirrors = ModelUtils.asListOrEmptyList(kafkaMirrorMaker2.getSpec().getMirrors());
    String host = KafkaMirrorMaker2Resources.qualifiedServiceName(mirrorMaker2Name, reconciliation.namespace());
    KafkaConnectApi apiClient = getKafkaConnectApi();
    return apiClient.list(host, KafkaConnectCluster.REST_API_PORT).compose(deleteMirrorMaker2ConnectorNames -> {
        for (Map.Entry<String, Function<KafkaMirrorMaker2MirrorSpec, KafkaMirrorMaker2ConnectorSpec>> connectorEntry : MIRRORMAKER2_CONNECTORS.entrySet()) {
            deleteMirrorMaker2ConnectorNames.removeAll(mirrors.stream().filter(// filter out non-existent connectors
            mirror -> connectorEntry.getValue().apply(mirror) != null).map(mirror -> mirror.getSourceCluster() + "->" + mirror.getTargetCluster() + connectorEntry.getKey()).collect(Collectors.toSet()));
        }
        LOGGER.debugCr(reconciliation, "delete MirrorMaker 2.0 connectors: {}", deleteMirrorMaker2ConnectorNames);
        Stream<Future<Void>> deletionFutures = deleteMirrorMaker2ConnectorNames.stream().map(connectorName -> apiClient.delete(reconciliation, host, KafkaConnectCluster.REST_API_PORT, connectorName));
        Stream<Future<Void>> createUpdateFutures = mirrors.stream().map(mirror -> reconcileMirrorMaker2Connectors(reconciliation, host, apiClient, kafkaMirrorMaker2, mirror, mirrorMaker2Cluster, mirrorMaker2Status, desiredLogging));
        return CompositeFuture.join(Stream.concat(deletionFutures, createUpdateFutures).collect(Collectors.toList())).map((Void) null);
    });
}