Examples with TopicExistsException org.apache.kafka.common.errors.TopicExistsException used on opensource projects

Example 1 with TopicExistsException

use of org.apache.kafka.common.errors.TopicExistsException in project ksql by confluentinc.

the class KafkaTopicClientImpl method createTopic.

@Override
public void createTopic(final String topic, final int numPartitions, final short replicationFactor, final Map<String, ?> configs) {
    if (isTopicExists(topic)) {
        validateTopicProperties(topic, numPartitions, replicationFactor);
        return;
    }
    final NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor);
    newTopic.configs(toStringConfigs(configs));
    try {
        log.info("Creating topic '{}'", topic);
        executeWithRetries(() -> adminClient.createTopics(Collections.singleton(newTopic)).all());
    } catch (final InterruptedException e) {
        throw new KafkaResponseGetFailedException("Failed to guarantee existence of topic " + topic, e);
    } catch (final TopicExistsException e) {
        // if the topic already exists, it is most likely because another node just created it.
        // ensure that it matches the partition count and replication factor before returning
        // success
        validateTopicProperties(topic, numPartitions, replicationFactor);
    } catch (final Exception e) {
        throw new KafkaResponseGetFailedException("Failed to guarantee existence of topic " + topic, e);
    }
}

Example 2 with TopicExistsException

use of org.apache.kafka.common.errors.TopicExistsException in project apache-kafka-on-k8s by banzaicloud.

the class TopicAdmin method createTopics.

/**
 * Attempt to create the topics described by the given definitions, returning all of the names of those topics that
 * were created by this request. Any existing topics with the same name are unchanged, and the names of such topics
 * are excluded from the result.
 * <p>
 * If multiple topic definitions have the same topic name, the last one with that name will be used.
 * <p>
 * Apache Kafka added support for creating topics in 0.10.1.0, so this method works as expected with that and later versions.
 * With brokers older than 0.10.1.0, this method is unable to create topics and always returns an empty set.
 *
 * @param topics the specifications of the topics
 * @return the names of the topics that were created by this operation; never null but possibly empty
 * @throws ConnectException            if an error occurs, the operation takes too long, or the thread is interrupted while
 *                                     attempting to perform this operation
 */
public Set<String> createTopics(NewTopic... topics) {
    Map<String, NewTopic> topicsByName = new HashMap<>();
    if (topics != null) {
        for (NewTopic topic : topics) {
            if (topic != null)
                topicsByName.put(topic.name(), topic);
        }
    }
    if (topicsByName.isEmpty())
        return Collections.emptySet();
    String bootstrapServers = bootstrapServers();
    String topicNameList = Utils.join(topicsByName.keySet(), "', '");
    // Attempt to create any missing topics
    CreateTopicsOptions args = new CreateTopicsOptions().validateOnly(false);
    Map<String, KafkaFuture<Void>> newResults = admin.createTopics(topicsByName.values(), args).values();
    // Iterate over each future so that we can handle individual failures like when some topics already exist
    Set<String> newlyCreatedTopicNames = new HashSet<>();
    for (Map.Entry<String, KafkaFuture<Void>> entry : newResults.entrySet()) {
        String topic = entry.getKey();
        try {
            entry.getValue().get();
            log.info("Created topic {} on brokers at {}", topicsByName.get(topic), bootstrapServers);
            newlyCreatedTopicNames.add(topic);
        } catch (ExecutionException e) {
            Throwable cause = e.getCause();
            if (cause instanceof TopicExistsException) {
                log.debug("Found existing topic '{}' on the brokers at {}", topic, bootstrapServers);
                continue;
            }
            if (cause instanceof UnsupportedVersionException) {
                log.debug("Unable to create topic(s) '{}' since the brokers at {} do not support the CreateTopics API.", " Falling back to assume topic(s) exist or will be auto-created by the broker.", topicNameList, bootstrapServers);
                return Collections.emptySet();
            }
            if (cause instanceof ClusterAuthorizationException) {
                log.debug("Not authorized to create topic(s) '{}'." + " Falling back to assume topic(s) exist or will be auto-created by the broker.", topicNameList, bootstrapServers);
                return Collections.emptySet();
            }
            if (cause instanceof TimeoutException) {
                // Timed out waiting for the operation to complete
                throw new ConnectException("Timed out while checking for or creating topic(s) '" + topicNameList + "'." + " This could indicate a connectivity issue, unavailable topic partitions, or if" + " this is your first use of the topic it may have taken too long to create.", cause);
            }
            throw new ConnectException("Error while attempting to create/find topic(s) '" + topicNameList + "'", e);
        } catch (InterruptedException e) {
            Thread.interrupted();
            throw new ConnectException("Interrupted while attempting to create/find topic(s) '" + topicNameList + "'", e);
        }
    }
    return newlyCreatedTopicNames;
}

Example 3 with TopicExistsException

use of org.apache.kafka.common.errors.TopicExistsException in project apache-kafka-on-k8s by banzaicloud.

the class InternalTopicManager method makeReady.

/**
 * Prepares a set of given internal topics.
 *
 * If a topic does not exist creates a new topic.
 * If a topic with the correct number of partitions exists ignores it.
 * If a topic exists already but has different number of partitions we fail and throw exception requesting user to reset the app before restarting again.
 */
public void makeReady(final Map<String, InternalTopicConfig> topics) {
    final Map<String, Integer> existingTopicPartitions = getNumPartitions(topics.keySet());
    final Set<InternalTopicConfig> topicsToBeCreated = validateTopicPartitions(topics.values(), existingTopicPartitions);
    if (topicsToBeCreated.size() > 0) {
        final Set<NewTopic> newTopics = new HashSet<>();
        for (final InternalTopicConfig internalTopicConfig : topicsToBeCreated) {
            final Map<String, String> topicConfig = internalTopicConfig.getProperties(defaultTopicConfigs, windowChangeLogAdditionalRetention);
            log.debug("Going to create topic {} with {} partitions and config {}.", internalTopicConfig.name(), internalTopicConfig.numberOfPartitions(), topicConfig);
            newTopics.add(new NewTopic(internalTopicConfig.name(), internalTopicConfig.numberOfPartitions(), replicationFactor).configs(topicConfig));
        }
        int remainingRetries = retries;
        boolean retry;
        do {
            retry = false;
            final CreateTopicsResult createTopicsResult = adminClient.createTopics(newTopics);
            final Set<String> createTopicNames = new HashSet<>();
            for (final Map.Entry<String, KafkaFuture<Void>> createTopicResult : createTopicsResult.values().entrySet()) {
                try {
                    createTopicResult.getValue().get();
                    createTopicNames.add(createTopicResult.getKey());
                } catch (final ExecutionException couldNotCreateTopic) {
                    final Throwable cause = couldNotCreateTopic.getCause();
                    final String topicName = createTopicResult.getKey();
                    if (cause instanceof TimeoutException) {
                        retry = true;
                        log.debug("Could not get number of partitions for topic {} due to timeout. " + "Will try again (remaining retries {}).", topicName, remainingRetries - 1);
                    } else if (cause instanceof TopicExistsException) {
                        createTopicNames.add(createTopicResult.getKey());
                        log.info(String.format("Topic %s exist already: %s", topicName, couldNotCreateTopic.getMessage()));
                    } else {
                        throw new StreamsException(String.format("Could not create topic %s.", topicName), couldNotCreateTopic);
                    }
                } catch (final InterruptedException fatalException) {
                    Thread.currentThread().interrupt();
                    log.error(INTERRUPTED_ERROR_MESSAGE, fatalException);
                    throw new IllegalStateException(INTERRUPTED_ERROR_MESSAGE, fatalException);
                }
            }
            if (retry) {
                final Iterator<NewTopic> it = newTopics.iterator();
                while (it.hasNext()) {
                    if (createTopicNames.contains(it.next().name())) {
                        it.remove();
                    }
                }
                continue;
            }
            return;
        } while (remainingRetries-- > 0);
        final String timeoutAndRetryError = "Could not create topics. " + "This can happen if the Kafka cluster is temporary not available. " + "You can increase admin client config `retries` to be resilient against this error.";
        log.error(timeoutAndRetryError);
        throw new StreamsException(timeoutAndRetryError);
    }
}

Example 4 with TopicExistsException

use of org.apache.kafka.common.errors.TopicExistsException in project strimzi by strimzi.

the class ControllerTest method testOnConfigMapAdded_TopicExistsException.

/**
 * 1. controller is notified that a ConfigMap is created
 * 2. error when creating topic in kafka
 */
@Test
public void testOnConfigMapAdded_TopicExistsException(TestContext context) {
    Exception createException = new TopicExistsException("");
    configMapAdded(context, createException, null);
// TODO check a k8s event got created
// TODO what happens when we subsequently reconcile?
}

Example 5 with TopicExistsException

use of org.apache.kafka.common.errors.TopicExistsException in project kafka by apache.

the class MockAdminClient method createTopics.

@Override
public synchronized CreateTopicsResult createTopics(Collection<NewTopic> newTopics, CreateTopicsOptions options) {
    Map<String, KafkaFuture<CreateTopicsResult.TopicMetadataAndConfig>> createTopicResult = new HashMap<>();
    if (timeoutNextRequests > 0) {
        for (final NewTopic newTopic : newTopics) {
            String topicName = newTopic.name();
            KafkaFutureImpl<CreateTopicsResult.TopicMetadataAndConfig> future = new KafkaFutureImpl<>();
            future.completeExceptionally(new TimeoutException());
            createTopicResult.put(topicName, future);
        }
        --timeoutNextRequests;
        return new CreateTopicsResult(createTopicResult);
    }
    for (final NewTopic newTopic : newTopics) {
        KafkaFutureImpl<CreateTopicsResult.TopicMetadataAndConfig> future = new KafkaFutureImpl<>();
        String topicName = newTopic.name();
        if (allTopics.containsKey(topicName)) {
            future.completeExceptionally(new TopicExistsException(String.format("Topic %s exists already.", topicName)));
            createTopicResult.put(topicName, future);
            continue;
        }
        int replicationFactor = newTopic.replicationFactor();
        if (replicationFactor == -1) {
            replicationFactor = defaultReplicationFactor;
        }
        if (replicationFactor > brokers.size()) {
            future.completeExceptionally(new InvalidReplicationFactorException(String.format("Replication factor: %d is larger than brokers: %d", newTopic.replicationFactor(), brokers.size())));
            createTopicResult.put(topicName, future);
            continue;
        }
        List<Node> replicas = new ArrayList<>(replicationFactor);
        for (int i = 0; i < replicationFactor; ++i) {
            replicas.add(brokers.get(i));
        }
        int numberOfPartitions = newTopic.numPartitions();
        if (numberOfPartitions == -1) {
            numberOfPartitions = defaultPartitions;
        }
        List<TopicPartitionInfo> partitions = new ArrayList<>(numberOfPartitions);
        // Partitions start off on the first log directory of each broker, for now.
        List<String> logDirs = new ArrayList<>(numberOfPartitions);
        for (int i = 0; i < numberOfPartitions; i++) {
            partitions.add(new TopicPartitionInfo(i, brokers.get(0), replicas, Collections.emptyList()));
            logDirs.add(brokerLogDirs.get(partitions.get(i).leader().id()).get(0));
        }
        Uuid topicId = Uuid.randomUuid();
        topicIds.put(topicName, topicId);
        topicNames.put(topicId, topicName);
        allTopics.put(topicName, new TopicMetadata(topicId, false, partitions, logDirs, newTopic.configs()));
        future.complete(null);
        createTopicResult.put(topicName, future);
    }
    return new CreateTopicsResult(createTopicResult);
}