Examples with KafkaTopic io.strimzi.api.kafka.model.KafkaTopic used on opensource projects

Example 86 with KafkaTopic

use of io.strimzi.api.kafka.model.KafkaTopic in project strimzi-kafka-operator by strimzi.

the class TopicOperator method reconcileOnTopicChange.

/**
 * Called when one of the ZK watches notifies of a change to the topic
 */
private Future<Void> reconcileOnTopicChange(LogContext logContext, TopicName topicName, Topic kafkaTopic, Reconciliation reconciliation) {
    // Look up the private topic to discover the name of kube KafkaTopic
    return topicStore.read(topicName).compose(storeTopic -> {
        ResourceName resourceName = storeTopic != null ? storeTopic.getResourceName() : topicName.asKubeName();
        return k8s.getFromName(resourceName).compose(topic -> {
            reconciliation.observedTopicFuture(kafkaTopic != null ? topic : null);
            Topic k8sTopic = TopicSerialization.fromTopicResource(topic);
            return reconcile(reconciliation, logContext.withKubeTopic(topic), topic, k8sTopic, kafkaTopic, storeTopic);
        });
    });
}

Example 87 with KafkaTopic

use of io.strimzi.api.kafka.model.KafkaTopic in project strimzi-kafka-operator by strimzi.

the class TopicOperator method reconcileAllTopics.

Future<?> reconcileAllTopics(String reconciliationType) {
    LOGGER.infoOp("Starting {} reconciliation", reconciliationType);
    return kafka.listTopics().recover(ex -> Future.failedFuture(new OperatorException("Error listing existing topics during " + reconciliationType + " reconciliation", ex))).compose(topicNamesFromKafka -> reconcileFromKafka(reconciliationType, topicNamesFromKafka.stream().map(TopicName::new).collect(Collectors.toList()))).compose(reconcileState -> {
        Future<List<KafkaTopic>> ktFut = k8s.listResources();
        return ktFut.recover(ex -> Future.failedFuture(new OperatorException("Error listing existing KafkaTopics during " + reconciliationType + " reconciliation", ex))).map(ktList -> {
            reconcileState.setKafkaTopics(ktList);
            return reconcileState;
        });
    }).compose(reconcileState -> {
        List<Future> futs = new ArrayList<>();
        pausedTopicCounter.set(0);
        topicCounter.set(reconcileState.ktList.size());
        for (KafkaTopic kt : reconcileState.ktList) {
            if (Annotations.isReconciliationPausedWithAnnotation(kt)) {
                pausedTopicCounter.getAndIncrement();
            }
            LogContext logContext = LogContext.periodic(reconciliationType + "kube " + kt.getMetadata().getName(), kt.getMetadata().getNamespace(), kt.getMetadata().getName()).withKubeTopic(kt);
            Topic topic = TopicSerialization.fromTopicResource(kt);
            TopicName topicName = topic.getTopicName();
            if (reconcileState.failed.containsKey(topicName)) {
                // we already failed to reconcile this topic in reconcileFromKafka(), /
                // don't bother trying again
                LOGGER.traceCr(logContext.toReconciliation(), "Already failed to reconcile {}", topicName);
                reconciliationsCounter.increment();
                failedReconciliationsCounter.increment();
            } else if (reconcileState.succeeded.contains(topicName)) {
                // we already succeeded in reconciling this topic in reconcileFromKafka()
                LOGGER.traceCr(logContext.toReconciliation(), "Already successfully reconciled {}", topicName);
                reconciliationsCounter.increment();
                successfulReconciliationsCounter.increment();
            } else if (reconcileState.undetermined.contains(topicName)) {
                // The topic didn't exist in topicStore, but now we know which KT it corresponds to
                futs.add(reconcileWithKubeTopic(logContext, kt, reconciliationType, new ResourceName(kt), topic.getTopicName()).compose(r -> {
                    // if success then remove from undetermined add to success
                    reconcileState.undetermined.remove(topicName);
                    reconcileState.succeeded.add(topicName);
                    return Future.succeededFuture(Boolean.TRUE);
                }));
            } else {
                // Topic exists in kube, but not in Kafka
                LOGGER.debugCr(logContext.toReconciliation(), "Topic {} exists in Kubernetes, but not Kafka", topicName, logTopic(kt));
                futs.add(reconcileWithKubeTopic(logContext, kt, reconciliationType, new ResourceName(kt), topic.getTopicName()).compose(r -> {
                    // if success then add to success
                    reconcileState.succeeded.add(topicName);
                    return Future.succeededFuture(Boolean.TRUE);
                }));
            }
        }
        return CompositeFuture.join(futs).compose(joined -> {
            List<Future> futs2 = new ArrayList<>();
            for (Throwable exception : reconcileState.failed.values()) {
                futs2.add(Future.failedFuture(exception));
            }
            // anything left in undetermined doesn't exist in topic store nor kube
            for (TopicName tn : reconcileState.undetermined) {
                LogContext logContext = LogContext.periodic(reconciliationType + "-" + tn, namespace, tn.asKubeName().toString());
                futs2.add(executeWithTopicLockHeld(logContext, tn, new Reconciliation(logContext, "delete-remaining", true) {

                    @Override
                    public Future<Void> execute() {
                        observedTopicFuture(null);
                        return getKafkaAndReconcile(this, logContext, tn, null, null);
                    }
                }));
            }
            return CompositeFuture.join(futs2);
        });
    });
}

Example 88 with KafkaTopic

use of io.strimzi.api.kafka.model.KafkaTopic in project strimzi-kafka-operator by strimzi.

the class TopicOperator method onTopicCreated.

/**
 * Called when a topic znode is created in ZK
 */
Future<Void> onTopicCreated(LogContext logContext, TopicName topicName) {
    // XXX currently runs on the ZK thread, requiring a synchronized inFlight
    // is it better to put this check in the topic deleted event?
    Reconciliation action = new Reconciliation(logContext, "onTopicCreated", true) {

        @Override
        public Future<Void> execute() {
            Reconciliation self = this;
            Promise<Void> promise = Promise.promise();
            TopicMetadataHandler handler = new TopicMetadataHandler(vertx, kafka, topicName, topicMetadataBackOff()) {

                @Override
                public void handle(AsyncResult<TopicMetadata> metadataResult) {
                    if (metadataResult.succeeded()) {
                        if (metadataResult.result() == null) {
                            // In this case it is most likely that we've been notified by ZK
                            // before Kafka has finished creating the topic, so we retry
                            // with exponential backoff.
                            retry(logContext.toReconciliation());
                        } else {
                            // We now have the metadata we need to create the
                            // resource...
                            Topic kafkaTopic = TopicSerialization.fromTopicMetadata(metadataResult.result());
                            reconcileOnTopicChange(logContext, topicName, kafkaTopic, self).onComplete(promise);
                        }
                    } else {
                        promise.fail(metadataResult.cause());
                    }
                }

                @Override
                public void onMaxAttemptsExceeded(MaxAttemptsExceededException e) {
                    promise.fail(e);
                }
            };
            return awaitExistential(logContext, topicName, true).compose(exists -> {
                kafka.topicMetadata(logContext.toReconciliation(), topicName).onComplete(handler);
                return promise.future();
            });
        }
    };
    return executeWithTopicLockHeld(logContext, topicName, action);
}

Example 89 with KafkaTopic

use of io.strimzi.api.kafka.model.KafkaTopic in project strimzi-kafka-operator by strimzi.

the class TopicSerialization method toTopicResource.

/**
 * Create a resource to reflect the given Topic.
 */
public static KafkaTopic toTopicResource(Topic topic, Labels labels) {
    ResourceName resourceName = topic.getOrAsKubeName();
    ObjectMeta om = topic.getMetadata();
    Map<String, String> lbls = new HashMap<>(labels.labels());
    if (om != null) {
        om.setName(resourceName.toString());
        if (topic.getMetadata().getLabels() != null)
            lbls.putAll(topic.getMetadata().getLabels());
        om.setLabels(lbls);
        om.setOwnerReferences(topic.getMetadata().getOwnerReferences());
        om.setAnnotations(topic.getMetadata().getAnnotations());
    } else {
        om = new ObjectMetaBuilder().withName(resourceName.toString()).withLabels(lbls).build();
    }
    KafkaTopic kt = new KafkaTopicBuilder().withMetadata(om).withNewSpec().withTopicName(topic.getTopicName().toString()).withPartitions(topic.getNumPartitions()).withReplicas((int) topic.getNumReplicas()).withConfig(new LinkedHashMap<>(topic.getConfig())).endSpec().build();
    // topic is created with annotations={} (empty map but should be null as well)
    if (topic.getMetadata() != null)
        kt.getMetadata().setAnnotations(topic.getMetadata().getAnnotations());
    return kt;
}

Example 90 with KafkaTopic

use of io.strimzi.api.kafka.model.KafkaTopic in project strimzi-kafka-operator by strimzi.

the class K8sImpl method createResource.

@Override
public Future<KafkaTopic> createResource(KafkaTopic topicResource) {
    Promise<KafkaTopic> handler = Promise.promise();
    vertx.executeBlocking(future -> {
        try {
            KafkaTopic kafkaTopic = operation().inNamespace(namespace).create(topicResource);
            LOGGER.debug("KafkaTopic {} created with version {}->{}", kafkaTopic.getMetadata().getName(), topicResource.getMetadata() != null ? topicResource.getMetadata().getResourceVersion() : null, kafkaTopic.getMetadata().getResourceVersion());
            future.complete(kafkaTopic);
        } catch (Exception e) {
            future.fail(e);
        }
    }, handler);
    return handler.future();
}