Examples with NewTopic org.apache.kafka.clients.admin.NewTopic used on opensource projects

Example 16 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project flink by apache.

the class KafkaContainerClient method createTopic.

public void createTopic(int replicationFactor, int numPartitions, String topic) {
    Map<String, Object> properties = new HashMap<>();
    properties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, container.getBootstrapServers());
    try (AdminClient admin = AdminClient.create(properties)) {
        admin.createTopics(Collections.singletonList(new NewTopic(topic, numPartitions, (short) replicationFactor)));
    }
}

Example 17 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class WorkerSourceTask method maybeCreateTopic.

// Due to transformations that may change the destination topic of a record (such as
// RegexRouter) topic creation can not be batched for multiple topics
private void maybeCreateTopic(String topic) {
    if (!topicCreation.isTopicCreationRequired(topic)) {
        log.trace("Topic creation by the connector is disabled or the topic {} was previously created." + "If auto.create.topics.enable is enabled on the broker, " + "the topic will be created with default settings", topic);
        return;
    }
    log.info("The task will send records to topic '{}' for the first time. Checking " + "whether topic exists", topic);
    Map<String, TopicDescription> existing = admin.describeTopics(topic);
    if (!existing.isEmpty()) {
        log.info("Topic '{}' already exists.", topic);
        topicCreation.addTopic(topic);
        return;
    }
    log.info("Creating topic '{}'", topic);
    TopicCreationGroup topicGroup = topicCreation.findFirstGroup(topic);
    log.debug("Topic '{}' matched topic creation group: {}", topic, topicGroup);
    NewTopic newTopic = topicGroup.newTopic(topic);
    TopicAdmin.TopicCreationResponse response = admin.createOrFindTopics(newTopic);
    if (response.isCreated(newTopic.name())) {
        topicCreation.addTopic(topic);
        log.info("Created topic '{}' using creation group {}", newTopic, topicGroup);
    } else if (response.isExisting(newTopic.name())) {
        topicCreation.addTopic(topic);
        log.info("Found existing topic '{}'", newTopic);
    } else {
        // The topic still does not exist and could not be created, so treat it as a task failure
        log.warn("Request to create new topic '{}' failed", topic);
        throw new ConnectException("Task failed to create new topic " + newTopic + ". Ensure " + "that the task is authorized to create topics or that the topic exists and " + "restart the task");
    }
}

Example 18 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class KafkaStatusBackingStore method configure.

@Override
public void configure(final WorkerConfig config) {
    this.statusTopic = config.getString(DistributedConfig.STATUS_STORAGE_TOPIC_CONFIG);
    if (this.statusTopic == null || this.statusTopic.trim().length() == 0)
        throw new ConfigException("Must specify topic for connector status.");
    String clusterId = ConnectUtils.lookupKafkaClusterId(config);
    Map<String, Object> originals = config.originals();
    Map<String, Object> producerProps = new HashMap<>(originals);
    producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
    producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
    // we handle retries in this class
    producerProps.put(ProducerConfig.RETRIES_CONFIG, 0);
    // disable idempotence since retries is force to 0
    producerProps.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false);
    ConnectUtils.addMetricsContextProperties(producerProps, config, clusterId);
    Map<String, Object> consumerProps = new HashMap<>(originals);
    consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
    ConnectUtils.addMetricsContextProperties(consumerProps, config, clusterId);
    Map<String, Object> adminProps = new HashMap<>(originals);
    ConnectUtils.addMetricsContextProperties(adminProps, config, clusterId);
    Supplier<TopicAdmin> adminSupplier;
    if (topicAdminSupplier != null) {
        adminSupplier = topicAdminSupplier;
    } else {
        // Create our own topic admin supplier that we'll close when we're stopped
        ownTopicAdmin = new SharedTopicAdmin(adminProps);
        adminSupplier = ownTopicAdmin;
    }
    Map<String, Object> topicSettings = config instanceof DistributedConfig ? ((DistributedConfig) config).statusStorageTopicSettings() : Collections.emptyMap();
    NewTopic topicDescription = TopicAdmin.defineTopic(statusTopic).config(// first so that we override user-supplied settings as needed
    topicSettings).compacted().partitions(config.getInt(DistributedConfig.STATUS_STORAGE_PARTITIONS_CONFIG)).replicationFactor(config.getShort(DistributedConfig.STATUS_STORAGE_REPLICATION_FACTOR_CONFIG)).build();
    Callback<ConsumerRecord<String, byte[]>> readCallback = (error, record) -> read(record);
    this.kafkaLog = createKafkaBasedLog(statusTopic, producerProps, consumerProps, readCallback, topicDescription, adminSupplier);
}

Example 19 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class TopicAdmin method createOrFindTopics.

/**
 * Attempt to create the topics described by the given definitions, returning all of the names of those topics that
 * were created by this request. Any existing topics with the same name are unchanged, and the names of such topics
 * are excluded from the result.
 * <p>
 * If multiple topic definitions have the same topic name, the last one with that name will be used.
 * <p>
 * Apache Kafka added support for creating topics in 0.10.1.0, so this method works as expected with that and later versions.
 * With brokers older than 0.10.1.0, this method is unable to create topics and always returns an empty set.
 *
 * @param topics the specifications of the topics
 * @return the {@link TopicCreationResponse} with the names of the newly created and existing topics;
 *         never null but possibly empty
 * @throws ConnectException if an error occurs, the operation takes too long, or the thread is interrupted while
 *                          attempting to perform this operation
 */
public TopicCreationResponse createOrFindTopics(NewTopic... topics) {
    Map<String, NewTopic> topicsByName = new HashMap<>();
    if (topics != null) {
        for (NewTopic topic : topics) {
            if (topic != null)
                topicsByName.put(topic.name(), topic);
        }
    }
    if (topicsByName.isEmpty())
        return EMPTY_CREATION;
    String bootstrapServers = bootstrapServers();
    String topicNameList = Utils.join(topicsByName.keySet(), "', '");
    // Attempt to create any missing topics
    CreateTopicsOptions args = new CreateTopicsOptions().validateOnly(false);
    Map<String, KafkaFuture<Void>> newResults = admin.createTopics(topicsByName.values(), args).values();
    // Iterate over each future so that we can handle individual failures like when some topics already exist
    Set<String> newlyCreatedTopicNames = new HashSet<>();
    Set<String> existingTopicNames = new HashSet<>();
    for (Map.Entry<String, KafkaFuture<Void>> entry : newResults.entrySet()) {
        String topic = entry.getKey();
        try {
            entry.getValue().get();
            if (logCreation) {
                log.info("Created topic {} on brokers at {}", topicsByName.get(topic), bootstrapServers);
            }
            newlyCreatedTopicNames.add(topic);
        } catch (ExecutionException e) {
            Throwable cause = e.getCause();
            if (cause instanceof TopicExistsException) {
                log.debug("Found existing topic '{}' on the brokers at {}", topic, bootstrapServers);
                existingTopicNames.add(topic);
                continue;
            }
            if (cause instanceof UnsupportedVersionException) {
                log.debug("Unable to create topic(s) '{}' since the brokers at {} do not support the CreateTopics API." + " Falling back to assume topic(s) exist or will be auto-created by the broker.", topicNameList, bootstrapServers);
                return EMPTY_CREATION;
            }
            if (cause instanceof ClusterAuthorizationException) {
                log.debug("Not authorized to create topic(s) '{}' upon the brokers {}." + " Falling back to assume topic(s) exist or will be auto-created by the broker.", topicNameList, bootstrapServers);
                return EMPTY_CREATION;
            }
            if (cause instanceof TopicAuthorizationException) {
                log.debug("Not authorized to create topic(s) '{}' upon the brokers {}." + " Falling back to assume topic(s) exist or will be auto-created by the broker.", topicNameList, bootstrapServers);
                return EMPTY_CREATION;
            }
            if (cause instanceof InvalidConfigurationException) {
                throw new ConnectException("Unable to create topic(s) '" + topicNameList + "': " + cause.getMessage(), cause);
            }
            if (cause instanceof TimeoutException) {
                // Timed out waiting for the operation to complete
                throw new ConnectException("Timed out while checking for or creating topic(s) '" + topicNameList + "'." + " This could indicate a connectivity issue, unavailable topic partitions, or if" + " this is your first use of the topic it may have taken too long to create.", cause);
            }
            throw new ConnectException("Error while attempting to create/find topic(s) '" + topicNameList + "'", e);
        } catch (InterruptedException e) {
            Thread.interrupted();
            throw new ConnectException("Interrupted while attempting to create/find topic(s) '" + topicNameList + "'", e);
        }
    }
    return new TopicCreationResponse(newlyCreatedTopicNames, existingTopicNames);
}

Example 20 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class EmbeddedKafkaCluster method createTopic.

/**
 * Create a Kafka topic with the given parameters.
 *
 * @param topic             The name of the topic.
 * @param partitions        The number of partitions for this topic.
 * @param replication       The replication factor for (partitions of) this topic.
 * @param topicConfig       Additional topic-level configuration settings.
 * @param adminClientConfig Additional admin client configuration settings.
 */
public void createTopic(String topic, int partitions, int replication, Map<String, String> topicConfig, Properties adminClientConfig) {
    if (replication > brokers.length) {
        throw new InvalidReplicationFactorException("Insufficient brokers (" + brokers.length + ") for desired replication (" + replication + ")");
    }
    log.info("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }", topic, partitions, replication, topicConfig);
    final NewTopic newTopic = new NewTopic(topic, partitions, (short) replication);
    newTopic.configs(topicConfig);
    try (final Admin adminClient = createAdminClient(adminClientConfig)) {
        adminClient.createTopics(Collections.singletonList(newTopic)).all().get();
    } catch (final InterruptedException | ExecutionException e) {
        throw new RuntimeException(e);
    }
}