Examples with NewTopic org.apache.kafka.clients.admin.NewTopic used on opensource projects

Example 21 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class TopicBasedRemoteLogMetadataManager method initializeResources.

private void initializeResources() {
    log.info("Initializing the resources.");
    final NewTopic remoteLogMetadataTopicRequest = createRemoteLogMetadataTopicRequest();
    boolean topicCreated = false;
    long startTimeMs = time.milliseconds();
    AdminClient adminClient = null;
    try {
        adminClient = AdminClient.create(rlmmConfig.producerProperties());
        // Stop if it is already initialized or closing.
        while (!(initialized.get() || closing.get())) {
            // If it is timed out then raise an error to exit.
            if (time.milliseconds() - startTimeMs > rlmmConfig.initializationRetryMaxTimeoutMs()) {
                log.error("Timed out in initializing the resources, retried to initialize the resource for [{}] ms.", rlmmConfig.initializationRetryMaxTimeoutMs());
                initializationFailed = true;
                return;
            }
            if (!topicCreated) {
                topicCreated = createTopic(adminClient, remoteLogMetadataTopicRequest);
            }
            if (!topicCreated) {
                // Sleep for INITIALIZATION_RETRY_INTERVAL_MS before trying to create the topic again.
                log.info("Sleep for : {} ms before it is retried again.", rlmmConfig.initializationRetryIntervalMs());
                Utils.sleep(rlmmConfig.initializationRetryIntervalMs());
                continue;
            } else {
                // If topic is already created, validate the existing topic partitions.
                try {
                    String topicName = remoteLogMetadataTopicRequest.name();
                    // If the existing topic partition size is not same as configured, mark initialization as failed and exit.
                    if (!isPartitionsCountSameAsConfigured(adminClient, topicName)) {
                        initializationFailed = true;
                    }
                } catch (Exception e) {
                    log.info("Sleep for : {} ms before it is retried again.", rlmmConfig.initializationRetryIntervalMs());
                    Utils.sleep(rlmmConfig.initializationRetryIntervalMs());
                    continue;
                }
            }
            // Create producer and consumer managers.
            lock.writeLock().lock();
            try {
                producerManager = new ProducerManager(rlmmConfig, rlmmTopicPartitioner);
                consumerManager = new ConsumerManager(rlmmConfig, remotePartitionMetadataStore, rlmmTopicPartitioner, time);
                if (startConsumerThread) {
                    consumerManager.startConsumerThread();
                } else {
                    log.info("RLMM Consumer task thread is not configured to be started.");
                }
                if (!pendingAssignPartitions.isEmpty()) {
                    assignPartitions(pendingAssignPartitions);
                    pendingAssignPartitions.clear();
                }
                initialized.set(true);
                log.info("Initialized resources successfully.");
            } catch (Exception e) {
                log.error("Encountered error while initializing producer/consumer", e);
                return;
            } finally {
                lock.writeLock().unlock();
            }
        }
    } finally {
        if (adminClient != null) {
            try {
                adminClient.close(Duration.ofSeconds(10));
            } catch (Exception e) {
                // Ignore the error.
                log.debug("Error occurred while closing the admin client", e);
            }
        }
    }
}

Example 22 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class TopicAdminTest method shouldCreateTopicWithReplicationFactorWhenItDoesNotExist.

@Test
public void shouldCreateTopicWithReplicationFactorWhenItDoesNotExist() {
    for (int numBrokers = 1; numBrokers < 10; ++numBrokers) {
        int maxRf = Math.min(numBrokers, 5);
        int maxDefaultRf = Math.min(numBrokers, 5);
        for (short rf = 1; rf < maxRf; ++rf) {
            NewTopic newTopic = TopicAdmin.defineTopic("myTopic").replicationFactor(rf).compacted().build();
            // Try clusters with no default replication factor or default partitions
            assertTopicCreation(numBrokers, newTopic, null, null, rf, 1);
            // Try clusters with different default partitions
            for (int numPartitions = 1; numPartitions < 30; ++numPartitions) {
                assertTopicCreation(numBrokers, newTopic, numPartitions, null, rf, numPartitions);
            }
            // Try clusters with different default replication factors
            for (int defaultRF = 1; defaultRF < maxDefaultRf; ++defaultRF) {
                assertTopicCreation(numBrokers, newTopic, null, defaultRF, rf, 1);
            }
        }
    }
}

Example 23 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class TopicAdminTest method throwsWithClusterAuthorizationFailureOnDescribe.

@Test
public void throwsWithClusterAuthorizationFailureOnDescribe() {
    final NewTopic newTopic = TopicAdmin.defineTopic("myTopic").partitions(1).compacted().build();
    Cluster cluster = createCluster(1);
    try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(new MockTime(), cluster)) {
        env.kafkaClient().prepareResponse(describeTopicResponseWithClusterAuthorizationException(newTopic));
        TopicAdmin admin = new TopicAdmin(null, env.adminClient());
        Exception e = assertThrows(ConnectException.class, () -> admin.describeTopics(newTopic.name()));
        assertTrue(e.getCause() instanceof ClusterAuthorizationException);
    }
}

Example 24 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class TopicAdminTest method throwsWithApiVersionMismatchOnDescribe.

/**
 * 0.11.0.0 clients can talk with older brokers, but the DESCRIBE_TOPIC API was added in 0.10.0.0. That means,
 * if our TopicAdmin talks to a pre 0.10.0 broker, it should receive an UnsupportedVersionException, should
 * create no topics, and return false.
 */
@Test
public void throwsWithApiVersionMismatchOnDescribe() {
    final NewTopic newTopic = TopicAdmin.defineTopic("myTopic").partitions(1).compacted().build();
    Cluster cluster = createCluster(1);
    try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(new MockTime(), cluster)) {
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
        env.kafkaClient().prepareResponse(describeTopicResponseWithUnsupportedVersion(newTopic));
        TopicAdmin admin = new TopicAdmin(null, env.adminClient());
        Exception e = assertThrows(ConnectException.class, () -> admin.describeTopics(newTopic.name()));
        assertTrue(e.getCause() instanceof UnsupportedVersionException);
    }
}

Example 25 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class InternalTopicManagerTest method shouldThrowWhenCreateTopicsResultsDoNotContainTopic.

@Test
public void shouldThrowWhenCreateTopicsResultsDoNotContainTopic() {
    final AdminClient admin = EasyMock.createNiceMock(AdminClient.class);
    final StreamsConfig streamsConfig = new StreamsConfig(config);
    final InternalTopicManager topicManager = new InternalTopicManager(time, admin, streamsConfig);
    final InternalTopicConfig internalTopicConfig = setupRepartitionTopicConfig(topic1, 1);
    final NewTopic newTopic = newTopic(topic1, internalTopicConfig, streamsConfig);
    EasyMock.expect(admin.createTopics(mkSet(newTopic))).andStubAnswer(() -> new MockCreateTopicsResult(Collections.singletonMap(topic2, new KafkaFutureImpl<>())));
    EasyMock.replay(admin);
    assertThrows(IllegalStateException.class, () -> topicManager.setup(Collections.singletonMap(topic1, internalTopicConfig)));
}