Examples with NewTopic org.apache.kafka.clients.admin.NewTopic used on opensource projects

Example 66 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project apache-kafka-on-k8s by banzaicloud.

the class WorkerUtilsTest method testCreateTopicsFailsIfAtLeastOneTopicExists.

@Test(expected = TopicExistsException.class)
public void testCreateTopicsFailsIfAtLeastOneTopicExists() throws Throwable {
    adminClient.addTopic(false, TEST_TOPIC, Collections.singletonList(new TopicPartitionInfo(0, broker1, singleReplica, Collections.<Node>emptyList())), null);
    Map<String, NewTopic> newTopics = new HashMap<>();
    newTopics.put(TEST_TOPIC, NEW_TEST_TOPIC);
    newTopics.put("another-topic", new NewTopic("another-topic", TEST_PARTITIONS, TEST_REPLICATION_FACTOR));
    newTopics.put("one-more-topic", new NewTopic("one-more-topic", TEST_PARTITIONS, TEST_REPLICATION_FACTOR));
    WorkerUtils.createTopics(log, adminClient, newTopics, true);
}

Example 67 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project apache-kafka-on-k8s by banzaicloud.

the class WorkerUtilsTest method testCreateOneTopic.

@Test
public void testCreateOneTopic() throws Throwable {
    Map<String, NewTopic> newTopics = Collections.singletonMap(TEST_TOPIC, NEW_TEST_TOPIC);
    WorkerUtils.createTopics(log, adminClient, newTopics, true);
    assertEquals(Collections.singleton(TEST_TOPIC), adminClient.listTopics().names().get());
    assertEquals(new TopicDescription(TEST_TOPIC, false, Collections.singletonList(new TopicPartitionInfo(0, broker1, singleReplica, Collections.<Node>emptyList()))), adminClient.describeTopics(Collections.singleton(TEST_TOPIC)).values().get(TEST_TOPIC).get());
}

Example 68 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project apache-kafka-on-k8s by banzaicloud.

the class WorkerUtils method createTopics.

/**
 * Creates Kafka topics and returns a list of topics that already exist
 * @param log             The logger to use
 * @param adminClient     AdminClient
 * @param topics          List of topics to create
 * @return                Collection of topics names that already exist.
 * @throws Throwable if creation of one or more topics fails (except for topic exists case).
 */
private static Collection<String> createTopics(Logger log, AdminClient adminClient, Collection<NewTopic> topics) throws Throwable {
    long startMs = Time.SYSTEM.milliseconds();
    int tries = 0;
    List<String> existingTopics = new ArrayList<>();
    Map<String, NewTopic> newTopics = new HashMap<>();
    for (NewTopic newTopic : topics) {
        newTopics.put(newTopic.name(), newTopic);
    }
    List<String> topicsToCreate = new ArrayList<>(newTopics.keySet());
    while (true) {
        log.info("Attempting to create {} topics (try {})...", topicsToCreate.size(), ++tries);
        Map<String, Future<Void>> creations = new HashMap<>();
        while (!topicsToCreate.isEmpty()) {
            List<NewTopic> newTopicsBatch = new ArrayList<>();
            for (int i = 0; (i < MAX_CREATE_TOPICS_BATCH_SIZE) && !topicsToCreate.isEmpty(); i++) {
                String topicName = topicsToCreate.remove(0);
                newTopicsBatch.add(newTopics.get(topicName));
            }
            creations.putAll(adminClient.createTopics(newTopicsBatch).values());
        }
        // timeout.  This is a workaround for KAFKA-6368.
        for (Map.Entry<String, Future<Void>> entry : creations.entrySet()) {
            String topicName = entry.getKey();
            Future<Void> future = entry.getValue();
            try {
                future.get();
                log.debug("Successfully created {}.", topicName);
            } catch (Exception e) {
                if ((e.getCause() instanceof TimeoutException) || (e.getCause() instanceof NotEnoughReplicasException)) {
                    log.warn("Attempt to create topic `{}` failed: {}", topicName, e.getCause().getMessage());
                    topicsToCreate.add(topicName);
                } else if (e.getCause() instanceof TopicExistsException) {
                    log.info("Topic {} already exists.", topicName);
                    existingTopics.add(topicName);
                } else {
                    log.warn("Failed to create {}", topicName, e.getCause());
                    throw e.getCause();
                }
            }
        }
        if (topicsToCreate.isEmpty()) {
            break;
        }
        if (Time.SYSTEM.milliseconds() > startMs + CREATE_TOPICS_CALL_TIMEOUT) {
            String str = "Unable to create topic(s): " + Utils.join(topicsToCreate, ", ") + "after " + tries + " attempt(s)";
            log.warn(str);
            throw new TimeoutException(str);
        }
    }
    return existingTopics;
}

Example 69 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project apache-kafka-on-k8s by banzaicloud.

the class KafkaOffsetBackingStore method configure.

@Override
public void configure(final WorkerConfig config) {
    String topic = config.getString(DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG);
    if (topic == null || topic.trim().length() == 0)
        throw new ConfigException("Offset storage topic must be specified");
    data = new HashMap<>();
    Map<String, Object> originals = config.originals();
    Map<String, Object> producerProps = new HashMap<>(originals);
    producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
    producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
    producerProps.put(ProducerConfig.RETRIES_CONFIG, Integer.MAX_VALUE);
    Map<String, Object> consumerProps = new HashMap<>(originals);
    consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
    consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
    Map<String, Object> adminProps = new HashMap<>(originals);
    NewTopic topicDescription = TopicAdmin.defineTopic(topic).compacted().partitions(config.getInt(DistributedConfig.OFFSET_STORAGE_PARTITIONS_CONFIG)).replicationFactor(config.getShort(DistributedConfig.OFFSET_STORAGE_REPLICATION_FACTOR_CONFIG)).build();
    offsetLog = createKafkaBasedLog(topic, producerProps, consumerProps, consumedCallback, topicDescription, adminProps);
}

Example 70 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project apache-kafka-on-k8s by banzaicloud.

the class KafkaStatusBackingStore method configure.

@Override
public void configure(final WorkerConfig config) {
    this.topic = config.getString(DistributedConfig.STATUS_STORAGE_TOPIC_CONFIG);
    if (this.topic == null || this.topic.trim().length() == 0)
        throw new ConfigException("Must specify topic for connector status.");
    Map<String, Object> originals = config.originals();
    Map<String, Object> producerProps = new HashMap<>(originals);
    producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
    producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
    // we handle retries in this class
    producerProps.put(ProducerConfig.RETRIES_CONFIG, 0);
    Map<String, Object> consumerProps = new HashMap<>(originals);
    consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
    Map<String, Object> adminProps = new HashMap<>(originals);
    NewTopic topicDescription = TopicAdmin.defineTopic(topic).compacted().partitions(config.getInt(DistributedConfig.STATUS_STORAGE_PARTITIONS_CONFIG)).replicationFactor(config.getShort(DistributedConfig.STATUS_STORAGE_REPLICATION_FACTOR_CONFIG)).build();
    Callback<ConsumerRecord<String, byte[]>> readCallback = new Callback<ConsumerRecord<String, byte[]>>() {

        @Override
        public void onCompletion(Throwable error, ConsumerRecord<String, byte[]> record) {
            read(record);
        }
    };
    this.kafkaLog = createKafkaBasedLog(topic, producerProps, consumerProps, readCallback, topicDescription, adminProps);
}