Search in sources :

Example 1 with UnknownTopologyException

use of org.apache.kafka.streams.errors.UnknownTopologyException in project kafka by apache.

the class KafkaStreamsNamedTopologyWrapper method removeNamedTopology.

/**
 * Remove an existing NamedTopology from a running Kafka Streams app. If multiple instances of the application are
 * running, you should inform all of them by calling {@code #removeNamedTopology(String)} on each client to ensure
 * it stops processing the old topology.
 * This method is not purely Async.
 *
 * @param topologyToRemove          name of the topology to be removed
 * @param resetOffsets              whether to reset the committed offsets for any source topics
 *
 * May complete exceptionally (does not actually directly throw) with:
 *
 * @throws IllegalArgumentException if this topology name cannot be found
 * @throws IllegalStateException    if streams has not been started or has already shut down
 */
public RemoveNamedTopologyResult removeNamedTopology(final String topologyToRemove, final boolean resetOffsets) {
    log.info("Informed to remove topology {} with resetOffsets={} ", topologyToRemove, resetOffsets);
    final KafkaFutureImpl<Void> removeTopologyFuture = new KafkaFutureImpl<>();
    if (hasStartedOrFinishedShuttingDown()) {
        log.error("Attempted to remove topology {} from while the Kafka Streams was in state {}, " + "application must be started first.", topologyToRemove, state);
        removeTopologyFuture.completeExceptionally(new IllegalStateException("Cannot remove a NamedTopology while the state is " + super.state));
    } else if (!getTopologyByName(topologyToRemove).isPresent()) {
        log.error("Attempted to remove unknown topology {}. This application currently contains the" + "following topologies: {}.", topologyToRemove, topologyMetadata.namedTopologiesView());
        removeTopologyFuture.completeExceptionally(new UnknownTopologyException("Unable to remove topology", topologyToRemove));
    }
    final Set<TopicPartition> partitionsToReset = metadataForLocalThreads().stream().flatMap(t -> {
        final Set<TaskMetadata> tasks = new HashSet<>(t.activeTasks());
        return tasks.stream();
    }).flatMap(t -> t.topicPartitions().stream()).filter(t -> topologyMetadata.sourceTopicsForTopology(topologyToRemove).contains(t.topic())).collect(Collectors.toSet());
    topologyMetadata.unregisterTopology(removeTopologyFuture, topologyToRemove);
    if (resetOffsets) {
        log.info("Resetting offsets for the following partitions of {} removed NamedTopology {}: {}", removeTopologyFuture.isCompletedExceptionally() ? "unsuccessfully" : "successfully", topologyToRemove, partitionsToReset);
        if (!partitionsToReset.isEmpty()) {
            removeTopologyFuture.whenComplete((v, throwable) -> {
                if (throwable != null) {
                    removeTopologyFuture.completeExceptionally(throwable);
                }
                DeleteConsumerGroupOffsetsResult deleteOffsetsResult = null;
                while (deleteOffsetsResult == null) {
                    try {
                        deleteOffsetsResult = adminClient.deleteConsumerGroupOffsets(applicationConfigs.getString(StreamsConfig.APPLICATION_ID_CONFIG), partitionsToReset);
                        deleteOffsetsResult.all().get();
                    } catch (final InterruptedException ex) {
                        ex.printStackTrace();
                        break;
                    } catch (final ExecutionException ex) {
                        if (ex.getCause() != null && ex.getCause() instanceof GroupSubscribedToTopicException && ex.getCause().getMessage().equals("Deleting offsets of a topic is forbidden while the consumer group is actively subscribed to it.")) {
                            ex.printStackTrace();
                        } else if (ex.getCause() != null && ex.getCause() instanceof GroupIdNotFoundException) {
                            log.debug("The offsets have been reset by another client or the group has been deleted, no need to retry further.");
                            break;
                        } else {
                            removeTopologyFuture.completeExceptionally(ex);
                        }
                        deleteOffsetsResult = null;
                    }
                    try {
                        Thread.sleep(100);
                    } catch (final InterruptedException ex) {
                        ex.printStackTrace();
                    }
                }
                removeTopologyFuture.complete(null);
            });
            return new RemoveNamedTopologyResult(removeTopologyFuture, removeTopologyFuture);
        }
    }
    return new RemoveNamedTopologyResult(removeTopologyFuture);
}
Also used : StreamsConfig(org.apache.kafka.streams.StreamsConfig) DefaultKafkaClientSupplier(org.apache.kafka.streams.processor.internals.DefaultKafkaClientSupplier) TaskId(org.apache.kafka.streams.processor.TaskId) TopologyMetadata(org.apache.kafka.streams.processor.internals.TopologyMetadata) UnknownTopologyException(org.apache.kafka.streams.errors.UnknownTopologyException) StreamsException(org.apache.kafka.streams.errors.StreamsException) TopologyException(org.apache.kafka.streams.errors.TopologyException) TaskMetadata(org.apache.kafka.streams.TaskMetadata) UnknownStateStoreException(org.apache.kafka.streams.errors.UnknownStateStoreException) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) LogContext(org.apache.kafka.common.utils.LogContext) Map(java.util.Map) KafkaFutureImpl(org.apache.kafka.common.internals.KafkaFutureImpl) Unstable(org.apache.kafka.common.annotation.InterfaceStability.Unstable) TopicPartition(org.apache.kafka.common.TopicPartition) Logger(org.slf4j.Logger) Properties(java.util.Properties) GroupSubscribedToTopicException(org.apache.kafka.common.errors.GroupSubscribedToTopicException) Collection(java.util.Collection) StreamsMetadata(org.apache.kafka.streams.StreamsMetadata) Set(java.util.Set) KafkaClientSupplier(org.apache.kafka.streams.KafkaClientSupplier) Collectors(java.util.stream.Collectors) Task(org.apache.kafka.streams.processor.internals.Task) ExecutionException(java.util.concurrent.ExecutionException) GroupIdNotFoundException(org.apache.kafka.common.errors.GroupIdNotFoundException) List(java.util.List) ConcurrentSkipListMap(java.util.concurrent.ConcurrentSkipListMap) KeyQueryMetadata(org.apache.kafka.streams.KeyQueryMetadata) LagInfo(org.apache.kafka.streams.LagInfo) StateStore(org.apache.kafka.streams.processor.StateStore) DeleteConsumerGroupOffsetsResult(org.apache.kafka.clients.admin.DeleteConsumerGroupOffsetsResult) Serializer(org.apache.kafka.common.serialization.Serializer) Optional(java.util.Optional) StoreQueryParameters(org.apache.kafka.streams.StoreQueryParameters) InternalTopologyBuilder(org.apache.kafka.streams.processor.internals.InternalTopologyBuilder) KafkaStreams(org.apache.kafka.streams.KafkaStreams) Collections(java.util.Collections) HashSet(java.util.HashSet) Set(java.util.Set) DeleteConsumerGroupOffsetsResult(org.apache.kafka.clients.admin.DeleteConsumerGroupOffsetsResult) UnknownTopologyException(org.apache.kafka.streams.errors.UnknownTopologyException) GroupSubscribedToTopicException(org.apache.kafka.common.errors.GroupSubscribedToTopicException) KafkaFutureImpl(org.apache.kafka.common.internals.KafkaFutureImpl) GroupIdNotFoundException(org.apache.kafka.common.errors.GroupIdNotFoundException) TopicPartition(org.apache.kafka.common.TopicPartition) ExecutionException(java.util.concurrent.ExecutionException)

Aggregations

ArrayList (java.util.ArrayList)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Map (java.util.Map)1 Optional (java.util.Optional)1 Properties (java.util.Properties)1 Set (java.util.Set)1 ConcurrentSkipListMap (java.util.concurrent.ConcurrentSkipListMap)1 ExecutionException (java.util.concurrent.ExecutionException)1 Collectors (java.util.stream.Collectors)1 DeleteConsumerGroupOffsetsResult (org.apache.kafka.clients.admin.DeleteConsumerGroupOffsetsResult)1 TopicPartition (org.apache.kafka.common.TopicPartition)1 Unstable (org.apache.kafka.common.annotation.InterfaceStability.Unstable)1 GroupIdNotFoundException (org.apache.kafka.common.errors.GroupIdNotFoundException)1 GroupSubscribedToTopicException (org.apache.kafka.common.errors.GroupSubscribedToTopicException)1 KafkaFutureImpl (org.apache.kafka.common.internals.KafkaFutureImpl)1 Serializer (org.apache.kafka.common.serialization.Serializer)1 LogContext (org.apache.kafka.common.utils.LogContext)1