use of org.apache.kafka.common.errors.TopicExistsException in project kafka by apache.
the class TopicAdmin method createOrFindTopics.
/**
* Attempt to create the topics described by the given definitions, returning all of the names of those topics that
* were created by this request. Any existing topics with the same name are unchanged, and the names of such topics
* are excluded from the result.
* <p>
* If multiple topic definitions have the same topic name, the last one with that name will be used.
* <p>
* Apache Kafka added support for creating topics in 0.10.1.0, so this method works as expected with that and later versions.
* With brokers older than 0.10.1.0, this method is unable to create topics and always returns an empty set.
*
* @param topics the specifications of the topics
* @return the {@link TopicCreationResponse} with the names of the newly created and existing topics;
* never null but possibly empty
* @throws ConnectException if an error occurs, the operation takes too long, or the thread is interrupted while
* attempting to perform this operation
*/
public TopicCreationResponse createOrFindTopics(NewTopic... topics) {
Map<String, NewTopic> topicsByName = new HashMap<>();
if (topics != null) {
for (NewTopic topic : topics) {
if (topic != null)
topicsByName.put(topic.name(), topic);
}
}
if (topicsByName.isEmpty())
return EMPTY_CREATION;
String bootstrapServers = bootstrapServers();
String topicNameList = Utils.join(topicsByName.keySet(), "', '");
// Attempt to create any missing topics
CreateTopicsOptions args = new CreateTopicsOptions().validateOnly(false);
Map<String, KafkaFuture<Void>> newResults = admin.createTopics(topicsByName.values(), args).values();
// Iterate over each future so that we can handle individual failures like when some topics already exist
Set<String> newlyCreatedTopicNames = new HashSet<>();
Set<String> existingTopicNames = new HashSet<>();
for (Map.Entry<String, KafkaFuture<Void>> entry : newResults.entrySet()) {
String topic = entry.getKey();
try {
entry.getValue().get();
if (logCreation) {
log.info("Created topic {} on brokers at {}", topicsByName.get(topic), bootstrapServers);
}
newlyCreatedTopicNames.add(topic);
} catch (ExecutionException e) {
Throwable cause = e.getCause();
if (cause instanceof TopicExistsException) {
log.debug("Found existing topic '{}' on the brokers at {}", topic, bootstrapServers);
existingTopicNames.add(topic);
continue;
}
if (cause instanceof UnsupportedVersionException) {
log.debug("Unable to create topic(s) '{}' since the brokers at {} do not support the CreateTopics API." + " Falling back to assume topic(s) exist or will be auto-created by the broker.", topicNameList, bootstrapServers);
return EMPTY_CREATION;
}
if (cause instanceof ClusterAuthorizationException) {
log.debug("Not authorized to create topic(s) '{}' upon the brokers {}." + " Falling back to assume topic(s) exist or will be auto-created by the broker.", topicNameList, bootstrapServers);
return EMPTY_CREATION;
}
if (cause instanceof TopicAuthorizationException) {
log.debug("Not authorized to create topic(s) '{}' upon the brokers {}." + " Falling back to assume topic(s) exist or will be auto-created by the broker.", topicNameList, bootstrapServers);
return EMPTY_CREATION;
}
if (cause instanceof InvalidConfigurationException) {
throw new ConnectException("Unable to create topic(s) '" + topicNameList + "': " + cause.getMessage(), cause);
}
if (cause instanceof TimeoutException) {
// Timed out waiting for the operation to complete
throw new ConnectException("Timed out while checking for or creating topic(s) '" + topicNameList + "'." + " This could indicate a connectivity issue, unavailable topic partitions, or if" + " this is your first use of the topic it may have taken too long to create.", cause);
}
throw new ConnectException("Error while attempting to create/find topic(s) '" + topicNameList + "'", e);
} catch (InterruptedException e) {
Thread.interrupted();
throw new ConnectException("Interrupted while attempting to create/find topic(s) '" + topicNameList + "'", e);
}
}
return new TopicCreationResponse(newlyCreatedTopicNames, existingTopicNames);
}
use of org.apache.kafka.common.errors.TopicExistsException in project kafka by apache.
the class InternalTopicManagerTest method shouldOnlyRetryNotSuccessfulFuturesDuringSetup.
@Test
public void shouldOnlyRetryNotSuccessfulFuturesDuringSetup() {
final AdminClient admin = EasyMock.createMock(AdminClient.class);
final StreamsConfig streamsConfig = new StreamsConfig(config);
final InternalTopicManager topicManager = new InternalTopicManager(time, admin, streamsConfig);
final KafkaFutureImpl<TopicMetadataAndConfig> createTopicFailFuture = new KafkaFutureImpl<>();
createTopicFailFuture.completeExceptionally(new TopicExistsException("exists"));
final KafkaFutureImpl<TopicMetadataAndConfig> createTopicSuccessfulFuture = new KafkaFutureImpl<>();
createTopicSuccessfulFuture.complete(new TopicMetadataAndConfig(Uuid.randomUuid(), 1, 1, new Config(Collections.emptyList())));
final InternalTopicConfig internalTopicConfig1 = setupRepartitionTopicConfig(topic1, 1);
final InternalTopicConfig internalTopicConfig2 = setupRepartitionTopicConfig(topic2, 1);
final NewTopic newTopic1 = newTopic(topic1, internalTopicConfig1, streamsConfig);
final NewTopic newTopic2 = newTopic(topic2, internalTopicConfig2, streamsConfig);
EasyMock.expect(admin.createTopics(mkSet(newTopic1, newTopic2))).andAnswer(() -> new MockCreateTopicsResult(mkMap(mkEntry(topic1, createTopicSuccessfulFuture), mkEntry(topic2, createTopicFailFuture))));
EasyMock.expect(admin.createTopics(mkSet(newTopic2))).andAnswer(() -> new MockCreateTopicsResult(mkMap(mkEntry(topic2, createTopicSuccessfulFuture))));
EasyMock.replay(admin);
topicManager.setup(mkMap(mkEntry(topic1, internalTopicConfig1), mkEntry(topic2, internalTopicConfig2)));
EasyMock.verify(admin);
}
use of org.apache.kafka.common.errors.TopicExistsException in project kafka by apache.
the class InternalTopicManagerTest method shouldCleanUpWhenCreateTopicsResultsDoNotContainTopic.
@Test
public void shouldCleanUpWhenCreateTopicsResultsDoNotContainTopic() {
final AdminClient admin = EasyMock.createNiceMock(AdminClient.class);
final StreamsConfig streamsConfig = new StreamsConfig(config);
final InternalTopicManager topicManager = new InternalTopicManager(time, admin, streamsConfig);
final InternalTopicConfig internalTopicConfig1 = setupRepartitionTopicConfig(topic1, 1);
final InternalTopicConfig internalTopicConfig2 = setupRepartitionTopicConfig(topic2, 1);
final KafkaFutureImpl<TopicMetadataAndConfig> createTopicFailFuture1 = new KafkaFutureImpl<>();
createTopicFailFuture1.completeExceptionally(new TopicExistsException("exists"));
final KafkaFutureImpl<TopicMetadataAndConfig> createTopicSuccessfulFuture = new KafkaFutureImpl<>();
createTopicSuccessfulFuture.complete(new TopicMetadataAndConfig(Uuid.randomUuid(), 1, 1, new Config(Collections.emptyList())));
final NewTopic newTopic1 = newTopic(topic1, internalTopicConfig1, streamsConfig);
final NewTopic newTopic2 = newTopic(topic2, internalTopicConfig2, streamsConfig);
EasyMock.expect(admin.createTopics(mkSet(newTopic1, newTopic2))).andAnswer(() -> new MockCreateTopicsResult(mkMap(mkEntry(topic1, createTopicSuccessfulFuture), mkEntry(topic2, createTopicFailFuture1))));
EasyMock.expect(admin.createTopics(mkSet(newTopic2))).andAnswer(() -> new MockCreateTopicsResult(mkMap(mkEntry(topic3, createTopicSuccessfulFuture))));
final KafkaFutureImpl<Void> deleteTopicSuccessfulFuture = new KafkaFutureImpl<>();
deleteTopicSuccessfulFuture.complete(null);
EasyMock.expect(admin.deleteTopics(mkSet(topic1))).andAnswer(() -> new MockDeleteTopicsResult(mkMap(mkEntry(topic1, deleteTopicSuccessfulFuture))));
EasyMock.replay(admin);
assertThrows(IllegalStateException.class, () -> topicManager.setup(mkMap(mkEntry(topic1, internalTopicConfig1), mkEntry(topic2, internalTopicConfig2))));
EasyMock.verify(admin);
}
use of org.apache.kafka.common.errors.TopicExistsException in project kafka by apache.
the class InternalTopicManagerTest method setupCleanUpScenario.
private void setupCleanUpScenario(final AdminClient admin, final StreamsConfig streamsConfig, final InternalTopicConfig internalTopicConfig1, final InternalTopicConfig internalTopicConfig2) {
final KafkaFutureImpl<TopicMetadataAndConfig> createTopicFailFuture1 = new KafkaFutureImpl<>();
createTopicFailFuture1.completeExceptionally(new TopicExistsException("exists"));
final KafkaFutureImpl<TopicMetadataAndConfig> createTopicFailFuture2 = new KafkaFutureImpl<>();
createTopicFailFuture2.completeExceptionally(new IllegalStateException("Nobody expects the Spanish inquisition"));
final KafkaFutureImpl<TopicMetadataAndConfig> createTopicSuccessfulFuture = new KafkaFutureImpl<>();
createTopicSuccessfulFuture.complete(new TopicMetadataAndConfig(Uuid.randomUuid(), 1, 1, new Config(Collections.emptyList())));
final NewTopic newTopic1 = newTopic(topic1, internalTopicConfig1, streamsConfig);
final NewTopic newTopic2 = newTopic(topic2, internalTopicConfig2, streamsConfig);
EasyMock.expect(admin.createTopics(mkSet(newTopic1, newTopic2))).andAnswer(() -> new MockCreateTopicsResult(mkMap(mkEntry(topic1, createTopicSuccessfulFuture), mkEntry(topic2, createTopicFailFuture1))));
EasyMock.expect(admin.createTopics(mkSet(newTopic2))).andAnswer(() -> new MockCreateTopicsResult(mkMap(mkEntry(topic2, createTopicFailFuture2))));
}
use of org.apache.kafka.common.errors.TopicExistsException in project kafka by apache.
the class InternalTopicManagerTest method shouldCleanUpWhenCreateTopicsTimesOut.
@Test
public void shouldCleanUpWhenCreateTopicsTimesOut() {
final AdminClient admin = EasyMock.createNiceMock(AdminClient.class);
final StreamsConfig streamsConfig = new StreamsConfig(config);
final MockTime time = new MockTime((Integer) config.get(StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG)) / 3);
final InternalTopicManager topicManager = new InternalTopicManager(time, admin, streamsConfig);
final InternalTopicConfig internalTopicConfig1 = setupRepartitionTopicConfig(topic1, 1);
final InternalTopicConfig internalTopicConfig2 = setupRepartitionTopicConfig(topic2, 1);
final KafkaFutureImpl<TopicMetadataAndConfig> createTopicFailFuture1 = new KafkaFutureImpl<>();
createTopicFailFuture1.completeExceptionally(new TopicExistsException("exists"));
final KafkaFutureImpl<TopicMetadataAndConfig> createTopicSuccessfulFuture = new KafkaFutureImpl<>();
createTopicSuccessfulFuture.complete(new TopicMetadataAndConfig(Uuid.randomUuid(), 1, 1, new Config(Collections.emptyList())));
final NewTopic newTopic1 = newTopic(topic1, internalTopicConfig1, streamsConfig);
final NewTopic newTopic2 = newTopic(topic2, internalTopicConfig2, streamsConfig);
EasyMock.expect(admin.createTopics(mkSet(newTopic1, newTopic2))).andAnswer(() -> new MockCreateTopicsResult(mkMap(mkEntry(topic1, createTopicSuccessfulFuture), mkEntry(topic2, createTopicFailFuture1))));
final KafkaFutureImpl<TopicMetadataAndConfig> createTopicFutureThatNeverCompletes = new KafkaFutureImpl<>();
EasyMock.expect(admin.createTopics(mkSet(newTopic2))).andStubAnswer(() -> new MockCreateTopicsResult(mkMap(mkEntry(topic2, createTopicFutureThatNeverCompletes))));
final KafkaFutureImpl<Void> deleteTopicSuccessfulFuture = new KafkaFutureImpl<>();
deleteTopicSuccessfulFuture.complete(null);
EasyMock.expect(admin.deleteTopics(mkSet(topic1))).andAnswer(() -> new MockDeleteTopicsResult(mkMap(mkEntry(topic1, deleteTopicSuccessfulFuture))));
EasyMock.replay(admin);
assertThrows(TimeoutException.class, () -> topicManager.setup(mkMap(mkEntry(topic1, internalTopicConfig1), mkEntry(topic2, internalTopicConfig2))));
EasyMock.verify(admin);
}
Aggregations