Examples with NewTopic org.apache.kafka.clients.admin.NewTopic used on opensource projects

Example 1 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project ksql by confluentinc.

the class KafkaTopicClientImpl method createTopic.

@Override
public void createTopic(final String topic, final int numPartitions, final short replicationFactor, final Map<String, ?> configs) {
    if (isTopicExists(topic)) {
        validateTopicProperties(topic, numPartitions, replicationFactor);
        return;
    }
    final NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor);
    newTopic.configs(toStringConfigs(configs));
    try {
        log.info("Creating topic '{}'", topic);
        executeWithRetries(() -> adminClient.createTopics(Collections.singleton(newTopic)).all());
    } catch (final InterruptedException e) {
        throw new KafkaResponseGetFailedException("Failed to guarantee existence of topic " + topic, e);
    } catch (final TopicExistsException e) {
        // if the topic already exists, it is most likely because another node just created it.
        // ensure that it matches the partition count and replication factor before returning
        // success
        validateTopicProperties(topic, numPartitions, replicationFactor);
    } catch (final Exception e) {
        throw new KafkaResponseGetFailedException("Failed to guarantee existence of topic " + topic, e);
    }
}

Example 2 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project ksql by confluentinc.

the class KafkaTopicClientImplTest method shouldSetTopicCleanupPolicyToCompact.

@Test
public void shouldSetTopicCleanupPolicyToCompact() throws InterruptedException, ExecutionException {
    expect(adminClient.listTopics()).andReturn(getEmptyListTopicResult());
    // Verify that the new topic configuration being passed to the admin client is what we expect.
    NewTopic newTopic = new NewTopic(topicName1, 1, (short) 1);
    newTopic.configs(Collections.singletonMap("cleanup.policy", "compact"));
    expect(adminClient.createTopics(singleNewTopic(newTopic))).andReturn(getCreateTopicsResult());
    replay(adminClient);
    KafkaTopicClient kafkaTopicClient = new KafkaTopicClientImpl(adminClient);
    kafkaTopicClient.createTopic(topicName1, 1, (short) 1, Collections.singletonMap("cleanup.policy", "compact"));
    verify(adminClient);
}

Example 3 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project apache-kafka-on-k8s by banzaicloud.

the class WorkerUtilsTest method testExistingTopicsNotCreated.

@Test
public void testExistingTopicsNotCreated() throws Throwable {
    final String existingTopic = "existing-topic";
    List<TopicPartitionInfo> tpInfo = new ArrayList<>();
    tpInfo.add(new TopicPartitionInfo(0, broker1, singleReplica, Collections.<Node>emptyList()));
    tpInfo.add(new TopicPartitionInfo(1, broker2, singleReplica, Collections.<Node>emptyList()));
    tpInfo.add(new TopicPartitionInfo(2, broker3, singleReplica, Collections.<Node>emptyList()));
    adminClient.addTopic(false, existingTopic, tpInfo, null);
    WorkerUtils.createTopics(log, adminClient, Collections.singletonMap(existingTopic, new NewTopic(existingTopic, tpInfo.size(), TEST_REPLICATION_FACTOR)), false);
    assertEquals(Collections.singleton(existingTopic), adminClient.listTopics().names().get());
}

Example 4 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project apache-kafka-on-k8s by banzaicloud.

the class KafkaConfigBackingStore method setupAndCreateKafkaBasedLog.

// package private for testing
KafkaBasedLog<String, byte[]> setupAndCreateKafkaBasedLog(String topic, final WorkerConfig config) {
    Map<String, Object> originals = config.originals();
    Map<String, Object> producerProps = new HashMap<>(originals);
    producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
    producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
    producerProps.put(ProducerConfig.RETRIES_CONFIG, Integer.MAX_VALUE);
    Map<String, Object> consumerProps = new HashMap<>(originals);
    consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
    Map<String, Object> adminProps = new HashMap<>(originals);
    NewTopic topicDescription = TopicAdmin.defineTopic(topic).compacted().partitions(1).replicationFactor(config.getShort(DistributedConfig.CONFIG_STORAGE_REPLICATION_FACTOR_CONFIG)).build();
    return createKafkaBasedLog(topic, producerProps, consumerProps, new ConsumeCallback(), topicDescription, adminProps);
}

Example 5 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project apache-kafka-on-k8s by banzaicloud.

the class TopicAdmin method createTopics.

/**
 * Attempt to create the topics described by the given definitions, returning all of the names of those topics that
 * were created by this request. Any existing topics with the same name are unchanged, and the names of such topics
 * are excluded from the result.
 * <p>
 * If multiple topic definitions have the same topic name, the last one with that name will be used.
 * <p>
 * Apache Kafka added support for creating topics in 0.10.1.0, so this method works as expected with that and later versions.
 * With brokers older than 0.10.1.0, this method is unable to create topics and always returns an empty set.
 *
 * @param topics the specifications of the topics
 * @return the names of the topics that were created by this operation; never null but possibly empty
 * @throws ConnectException            if an error occurs, the operation takes too long, or the thread is interrupted while
 *                                     attempting to perform this operation
 */
public Set<String> createTopics(NewTopic... topics) {
    Map<String, NewTopic> topicsByName = new HashMap<>();
    if (topics != null) {
        for (NewTopic topic : topics) {
            if (topic != null)
                topicsByName.put(topic.name(), topic);
        }
    }
    if (topicsByName.isEmpty())
        return Collections.emptySet();
    String bootstrapServers = bootstrapServers();
    String topicNameList = Utils.join(topicsByName.keySet(), "', '");
    // Attempt to create any missing topics
    CreateTopicsOptions args = new CreateTopicsOptions().validateOnly(false);
    Map<String, KafkaFuture<Void>> newResults = admin.createTopics(topicsByName.values(), args).values();
    // Iterate over each future so that we can handle individual failures like when some topics already exist
    Set<String> newlyCreatedTopicNames = new HashSet<>();
    for (Map.Entry<String, KafkaFuture<Void>> entry : newResults.entrySet()) {
        String topic = entry.getKey();
        try {
            entry.getValue().get();
            log.info("Created topic {} on brokers at {}", topicsByName.get(topic), bootstrapServers);
            newlyCreatedTopicNames.add(topic);
        } catch (ExecutionException e) {
            Throwable cause = e.getCause();
            if (cause instanceof TopicExistsException) {
                log.debug("Found existing topic '{}' on the brokers at {}", topic, bootstrapServers);
                continue;
            }
            if (cause instanceof UnsupportedVersionException) {
                log.debug("Unable to create topic(s) '{}' since the brokers at {} do not support the CreateTopics API.", " Falling back to assume topic(s) exist or will be auto-created by the broker.", topicNameList, bootstrapServers);
                return Collections.emptySet();
            }
            if (cause instanceof ClusterAuthorizationException) {
                log.debug("Not authorized to create topic(s) '{}'." + " Falling back to assume topic(s) exist or will be auto-created by the broker.", topicNameList, bootstrapServers);
                return Collections.emptySet();
            }
            if (cause instanceof TimeoutException) {
                // Timed out waiting for the operation to complete
                throw new ConnectException("Timed out while checking for or creating topic(s) '" + topicNameList + "'." + " This could indicate a connectivity issue, unavailable topic partitions, or if" + " this is your first use of the topic it may have taken too long to create.", cause);
            }
            throw new ConnectException("Error while attempting to create/find topic(s) '" + topicNameList + "'", e);
        } catch (InterruptedException e) {
            Thread.interrupted();
            throw new ConnectException("Interrupted while attempting to create/find topic(s) '" + topicNameList + "'", e);
        }
    }
    return newlyCreatedTopicNames;
}