Examples with NewTopic org.apache.kafka.clients.admin.NewTopic used on opensource projects

Example 6 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project apache-kafka-on-k8s by banzaicloud.

the class TopicAdminTest method shouldCreateTopicWhenItDoesNotExist.

@Test
public void shouldCreateTopicWhenItDoesNotExist() {
    NewTopic newTopic = TopicAdmin.defineTopic("myTopic").partitions(1).compacted().build();
    Cluster cluster = createCluster(1);
    try (MockAdminClient mockAdminClient = new MockAdminClient(cluster.nodes(), cluster.nodeById(0))) {
        TopicAdmin admin = new TopicAdmin(null, mockAdminClient);
        assertTrue(admin.createTopic(newTopic));
    }
}

Example 7 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project apache-kafka-on-k8s by banzaicloud.

the class TopicAdminTest method returnNullWithApiVersionMismatch.

/**
 * 0.11.0.0 clients can talk with older brokers, but the CREATE_TOPIC API was added in 0.10.1.0. That means,
 * if our TopicAdmin talks to a pre 0.10.1 broker, it should receive an UnsupportedVersionException, should
 * create no topics, and return false.
 */
@Test
public void returnNullWithApiVersionMismatch() {
    final NewTopic newTopic = TopicAdmin.defineTopic("myTopic").partitions(1).compacted().build();
    Cluster cluster = createCluster(1);
    try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) {
        env.kafkaClient().setNode(cluster.controller());
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
        env.kafkaClient().prepareMetadataUpdate(env.cluster(), Collections.<String>emptySet());
        env.kafkaClient().prepareResponse(createTopicResponseWithUnsupportedVersion(newTopic));
        TopicAdmin admin = new TopicAdmin(null, env.adminClient());
        boolean created = admin.createTopic(newTopic);
        assertFalse(created);
    }
}

Example 8 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project apache-kafka-on-k8s by banzaicloud.

the class ClientCompatibilityTest method testAdminClient.

void testAdminClient() throws Throwable {
    Properties adminProps = new Properties();
    adminProps.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, testConfig.bootstrapServer);
    try (final AdminClient client = AdminClient.create(adminProps)) {
        while (true) {
            Collection<Node> nodes = client.describeCluster().nodes().get();
            if (nodes.size() == testConfig.numClusterNodes) {
                break;
            } else if (nodes.size() > testConfig.numClusterNodes) {
                throw new KafkaException("Expected to see " + testConfig.numClusterNodes + " nodes, but saw " + nodes.size());
            }
            Thread.sleep(1);
            log.info("Saw only {} cluster nodes.  Waiting to see {}.", nodes.size(), testConfig.numClusterNodes);
        }
        tryFeature("createTopics", testConfig.createTopicsSupported, new Invoker() {

            @Override
            public void invoke() throws Throwable {
                try {
                    client.createTopics(Collections.singleton(new NewTopic("newtopic", 1, (short) 1))).all().get();
                } catch (ExecutionException e) {
                    throw e.getCause();
                }
            }
        }, new ResultTester() {

            @Override
            public void test() throws Throwable {
                while (true) {
                    try {
                        client.describeTopics(Collections.singleton("newtopic")).all().get();
                        break;
                    } catch (ExecutionException e) {
                        if (e.getCause() instanceof UnknownTopicOrPartitionException)
                            continue;
                        throw e;
                    }
                }
            }
        });
        while (true) {
            Collection<TopicListing> listings = client.listTopics().listings().get();
            if (!testConfig.createTopicsSupported)
                break;
            boolean foundNewTopic = false;
            for (TopicListing listing : listings) {
                if (listing.name().equals("newtopic")) {
                    if (listing.isInternal())
                        throw new KafkaException("Did not expect newtopic to be an internal topic.");
                    foundNewTopic = true;
                }
            }
            if (foundNewTopic)
                break;
            Thread.sleep(1);
            log.info("Did not see newtopic.  Retrying listTopics...");
        }
        tryFeature("describeAclsSupported", testConfig.describeAclsSupported, new Invoker() {

            @Override
            public void invoke() throws Throwable {
                try {
                    client.describeAcls(AclBindingFilter.ANY).values().get();
                } catch (ExecutionException e) {
                    if (e.getCause() instanceof SecurityDisabledException)
                        return;
                    throw e.getCause();
                }
            }
        });
    }
}

Example 9 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project apache-kafka-on-k8s by banzaicloud.

the class InternalTopicManager method makeReady.

/**
 * Prepares a set of given internal topics.
 *
 * If a topic does not exist creates a new topic.
 * If a topic with the correct number of partitions exists ignores it.
 * If a topic exists already but has different number of partitions we fail and throw exception requesting user to reset the app before restarting again.
 */
public void makeReady(final Map<String, InternalTopicConfig> topics) {
    final Map<String, Integer> existingTopicPartitions = getNumPartitions(topics.keySet());
    final Set<InternalTopicConfig> topicsToBeCreated = validateTopicPartitions(topics.values(), existingTopicPartitions);
    if (topicsToBeCreated.size() > 0) {
        final Set<NewTopic> newTopics = new HashSet<>();
        for (final InternalTopicConfig internalTopicConfig : topicsToBeCreated) {
            final Map<String, String> topicConfig = internalTopicConfig.getProperties(defaultTopicConfigs, windowChangeLogAdditionalRetention);
            log.debug("Going to create topic {} with {} partitions and config {}.", internalTopicConfig.name(), internalTopicConfig.numberOfPartitions(), topicConfig);
            newTopics.add(new NewTopic(internalTopicConfig.name(), internalTopicConfig.numberOfPartitions(), replicationFactor).configs(topicConfig));
        }
        int remainingRetries = retries;
        boolean retry;
        do {
            retry = false;
            final CreateTopicsResult createTopicsResult = adminClient.createTopics(newTopics);
            final Set<String> createTopicNames = new HashSet<>();
            for (final Map.Entry<String, KafkaFuture<Void>> createTopicResult : createTopicsResult.values().entrySet()) {
                try {
                    createTopicResult.getValue().get();
                    createTopicNames.add(createTopicResult.getKey());
                } catch (final ExecutionException couldNotCreateTopic) {
                    final Throwable cause = couldNotCreateTopic.getCause();
                    final String topicName = createTopicResult.getKey();
                    if (cause instanceof TimeoutException) {
                        retry = true;
                        log.debug("Could not get number of partitions for topic {} due to timeout. " + "Will try again (remaining retries {}).", topicName, remainingRetries - 1);
                    } else if (cause instanceof TopicExistsException) {
                        createTopicNames.add(createTopicResult.getKey());
                        log.info(String.format("Topic %s exist already: %s", topicName, couldNotCreateTopic.getMessage()));
                    } else {
                        throw new StreamsException(String.format("Could not create topic %s.", topicName), couldNotCreateTopic);
                    }
                } catch (final InterruptedException fatalException) {
                    Thread.currentThread().interrupt();
                    log.error(INTERRUPTED_ERROR_MESSAGE, fatalException);
                    throw new IllegalStateException(INTERRUPTED_ERROR_MESSAGE, fatalException);
                }
            }
            if (retry) {
                final Iterator<NewTopic> it = newTopics.iterator();
                while (it.hasNext()) {
                    if (createTopicNames.contains(it.next().name())) {
                        it.remove();
                    }
                }
                continue;
            }
            return;
        } while (remainingRetries-- > 0);
        final String timeoutAndRetryError = "Could not create topics. " + "This can happen if the Kafka cluster is temporary not available. " + "You can increase admin client config `retries` to be resilient against this error.";
        log.error(timeoutAndRetryError);
        throw new StreamsException(timeoutAndRetryError);
    }
}

Example 10 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project strimzi by strimzi.

the class MockKafka method createTopic.

@Override
public void createTopic(Topic t, Handler<AsyncResult<Void>> handler) {
    NewTopic newTopic = TopicSerialization.toNewTopic(t, null);
    AsyncResult<Void> event = createTopicResponse.apply(newTopic.name());
    if (event.succeeded()) {
        Topic.Builder topicBuilder = new Topic.Builder().withTopicName(newTopic.name()).withNumPartitions(newTopic.numPartitions()).withNumReplicas(newTopic.replicationFactor());
        try {
            Field field = NewTopic.class.getDeclaredField("configs");
            field.setAccessible(true);
            topicBuilder.withConfig((Map) field.get(newTopic));
        } catch (ReflectiveOperationException e) {
            throw new RuntimeException(e);
        }
        Topic topic = topicBuilder.build();
        topics.put(topic.getTopicName(), topic);
    }
    handler.handle(event);
}