Examples with TopicExistsException org.apache.kafka.common.errors.TopicExistsException used on opensource projects

Example 16 with TopicExistsException

use of org.apache.kafka.common.errors.TopicExistsException in project apache-kafka-on-k8s by banzaicloud.

the class MockAdminClient method createTopics.

@Override
public CreateTopicsResult createTopics(Collection<NewTopic> newTopics, CreateTopicsOptions options) {
    Map<String, KafkaFuture<Void>> createTopicResult = new HashMap<>();
    if (timeoutNextRequests > 0) {
        for (final NewTopic newTopic : newTopics) {
            String topicName = newTopic.name();
            KafkaFutureImpl<Void> future = new KafkaFutureImpl<>();
            future.completeExceptionally(new TimeoutException());
            createTopicResult.put(topicName, future);
        }
        --timeoutNextRequests;
        return new CreateTopicsResult(createTopicResult);
    }
    for (final NewTopic newTopic : newTopics) {
        KafkaFutureImpl<Void> future = new KafkaFutureImpl<>();
        String topicName = newTopic.name();
        if (allTopics.containsKey(topicName)) {
            future.completeExceptionally(new TopicExistsException(String.format("Topic %s exists already.", topicName)));
            createTopicResult.put(topicName, future);
            continue;
        }
        int replicationFactor = newTopic.replicationFactor();
        List<Node> replicas = new ArrayList<>(replicationFactor);
        for (int i = 0; i < replicationFactor; ++i) {
            replicas.add(brokers.get(i));
        }
        int numberOfPartitions = newTopic.numPartitions();
        List<TopicPartitionInfo> partitions = new ArrayList<>(numberOfPartitions);
        for (int p = 0; p < numberOfPartitions; ++p) {
            partitions.add(new TopicPartitionInfo(p, brokers.get(0), replicas, Collections.<Node>emptyList()));
        }
        allTopics.put(topicName, new TopicMetadata(false, partitions, newTopic.configs()));
        future.complete(null);
        createTopicResult.put(topicName, future);
    }
    return new CreateTopicsResult(createTopicResult);
}

Example 17 with TopicExistsException

use of org.apache.kafka.common.errors.TopicExistsException in project apache-kafka-on-k8s by banzaicloud.

the class WorkerUtils method createTopics.

/**
 * Creates Kafka topics and returns a list of topics that already exist
 * @param log             The logger to use
 * @param adminClient     AdminClient
 * @param topics          List of topics to create
 * @return                Collection of topics names that already exist.
 * @throws Throwable if creation of one or more topics fails (except for topic exists case).
 */
private static Collection<String> createTopics(Logger log, AdminClient adminClient, Collection<NewTopic> topics) throws Throwable {
    long startMs = Time.SYSTEM.milliseconds();
    int tries = 0;
    List<String> existingTopics = new ArrayList<>();
    Map<String, NewTopic> newTopics = new HashMap<>();
    for (NewTopic newTopic : topics) {
        newTopics.put(newTopic.name(), newTopic);
    }
    List<String> topicsToCreate = new ArrayList<>(newTopics.keySet());
    while (true) {
        log.info("Attempting to create {} topics (try {})...", topicsToCreate.size(), ++tries);
        Map<String, Future<Void>> creations = new HashMap<>();
        while (!topicsToCreate.isEmpty()) {
            List<NewTopic> newTopicsBatch = new ArrayList<>();
            for (int i = 0; (i < MAX_CREATE_TOPICS_BATCH_SIZE) && !topicsToCreate.isEmpty(); i++) {
                String topicName = topicsToCreate.remove(0);
                newTopicsBatch.add(newTopics.get(topicName));
            }
            creations.putAll(adminClient.createTopics(newTopicsBatch).values());
        }
        // timeout.  This is a workaround for KAFKA-6368.
        for (Map.Entry<String, Future<Void>> entry : creations.entrySet()) {
            String topicName = entry.getKey();
            Future<Void> future = entry.getValue();
            try {
                future.get();
                log.debug("Successfully created {}.", topicName);
            } catch (Exception e) {
                if ((e.getCause() instanceof TimeoutException) || (e.getCause() instanceof NotEnoughReplicasException)) {
                    log.warn("Attempt to create topic `{}` failed: {}", topicName, e.getCause().getMessage());
                    topicsToCreate.add(topicName);
                } else if (e.getCause() instanceof TopicExistsException) {
                    log.info("Topic {} already exists.", topicName);
                    existingTopics.add(topicName);
                } else {
                    log.warn("Failed to create {}", topicName, e.getCause());
                    throw e.getCause();
                }
            }
        }
        if (topicsToCreate.isEmpty()) {
            break;
        }
        if (Time.SYSTEM.milliseconds() > startMs + CREATE_TOPICS_CALL_TIMEOUT) {
            String str = "Unable to create topic(s): " + Utils.join(topicsToCreate, ", ") + "after " + tries + " attempt(s)";
            log.warn(str);
            throw new TimeoutException(str);
        }
    }
    return existingTopics;
}

Example 18 with TopicExistsException

use of org.apache.kafka.common.errors.TopicExistsException in project samza by apache.

the class KafkaSystemAdmin method createStream.

@Override
public boolean createStream(StreamSpec streamSpec) {
    LOG.info("Creating Kafka topic: {} on system: {}", streamSpec.getPhysicalName(), streamSpec.getSystemName());
    final String replFactor = "replication.factor";
    KafkaStreamSpec kafkaStreamSpec = toKafkaSpec(streamSpec);
    String topicName = kafkaStreamSpec.getPhysicalName();
    // create topic.
    NewTopic newTopic = new NewTopic(topicName, kafkaStreamSpec.getPartitionCount(), (short) kafkaStreamSpec.getReplicationFactor());
    // specify the configs
    Map<String, String> streamConfig = new HashMap<>(kafkaStreamSpec.getConfig());
    // HACK - replication.factor is invalid config for AdminClient.createTopics
    if (streamConfig.containsKey(replFactor)) {
        String repl = streamConfig.get(replFactor);
        LOG.warn("Configuration {}={} for topic={} is invalid. Using kSpec repl factor {}", replFactor, repl, kafkaStreamSpec.getPhysicalName(), kafkaStreamSpec.getReplicationFactor());
        streamConfig.remove(replFactor);
    }
    newTopic.configs(new MapConfig(streamConfig));
    CreateTopicsResult result = adminClient.createTopics(ImmutableSet.of(newTopic));
    try {
        result.all().get(KAFKA_ADMIN_OPS_TIMEOUT_MS, TimeUnit.MILLISECONDS);
    } catch (Exception e) {
        if (e instanceof TopicExistsException || e.getCause() instanceof TopicExistsException) {
            LOG.info("Topic {} already exists.", topicName);
            return false;
        }
        throw new SamzaException(String.format("Creation of topic %s failed.", topicName), e);
    }
    LOG.info("Successfully created topic {}", topicName);
    DescribeTopicsResult desc = adminClient.describeTopics(ImmutableSet.of(topicName));
    try {
        TopicDescription td = desc.all().get(KAFKA_ADMIN_OPS_TIMEOUT_MS, TimeUnit.MILLISECONDS).get(topicName);
        LOG.info("Topic {} created with {}", topicName, td);
        return true;
    } catch (Exception e) {
        LOG.error("'Describe after create' failed for topic " + topicName, e);
        return false;
    }
}

Example 19 with TopicExistsException

use of org.apache.kafka.common.errors.TopicExistsException in project kafka by apache.

the class KafkaExactlyOnceDemo method recreateTopics.

private static void recreateTopics(final int numPartitions) throws ExecutionException, InterruptedException {
    Properties props = new Properties();
    props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, KafkaProperties.KAFKA_SERVER_URL + ":" + KafkaProperties.KAFKA_SERVER_PORT);
    Admin adminClient = Admin.create(props);
    List<String> topicsToDelete = Arrays.asList(INPUT_TOPIC, OUTPUT_TOPIC);
    deleteTopic(adminClient, topicsToDelete);
    // Check topic existence in a retry loop
    while (true) {
        System.out.println("Making sure the topics are deleted successfully: " + topicsToDelete);
        Set<String> listedTopics = adminClient.listTopics().names().get();
        System.out.println("Current list of topics: " + listedTopics);
        boolean hasTopicInfo = false;
        for (String listedTopic : listedTopics) {
            if (topicsToDelete.contains(listedTopic)) {
                hasTopicInfo = true;
                break;
            }
        }
        if (!hasTopicInfo) {
            break;
        }
        Thread.sleep(1000);
    }
    // Create topics in a retry loop
    while (true) {
        final short replicationFactor = 1;
        final List<NewTopic> newTopics = Arrays.asList(new NewTopic(INPUT_TOPIC, numPartitions, replicationFactor), new NewTopic(OUTPUT_TOPIC, numPartitions, replicationFactor));
        try {
            adminClient.createTopics(newTopics).all().get();
            System.out.println("Created new topics: " + newTopics);
            break;
        } catch (ExecutionException e) {
            if (!(e.getCause() instanceof TopicExistsException)) {
                throw e;
            }
            System.out.println("Metadata of the old topics are not cleared yet...");
            deleteTopic(adminClient, topicsToDelete);
            Thread.sleep(1000);
        }
    }
}

Example 20 with TopicExistsException

use of org.apache.kafka.common.errors.TopicExistsException in project kafka by apache.

the class DeadLetterQueueReporter method createAndSetup.

public static DeadLetterQueueReporter createAndSetup(Map<String, Object> adminProps, ConnectorTaskId id, SinkConnectorConfig sinkConfig, Map<String, Object> producerProps, ErrorHandlingMetrics errorHandlingMetrics) {
    String topic = sinkConfig.dlqTopicName();
    try (Admin admin = Admin.create(adminProps)) {
        if (!admin.listTopics().names().get().contains(topic)) {
            log.error("Topic {} doesn't exist. Will attempt to create topic.", topic);
            NewTopic schemaTopicRequest = new NewTopic(topic, DLQ_NUM_DESIRED_PARTITIONS, sinkConfig.dlqTopicReplicationFactor());
            admin.createTopics(singleton(schemaTopicRequest)).all().get();
        }
    } catch (InterruptedException e) {
        throw new ConnectException("Could not initialize dead letter queue with topic=" + topic, e);
    } catch (ExecutionException e) {
        if (!(e.getCause() instanceof TopicExistsException)) {
            throw new ConnectException("Could not initialize dead letter queue with topic=" + topic, e);
        }
    }
    KafkaProducer<byte[], byte[]> dlqProducer = new KafkaProducer<>(producerProps);
    return new DeadLetterQueueReporter(dlqProducer, sinkConfig, id, errorHandlingMetrics);
}