Examples with AdminClient org.apache.kafka.clients.admin.AdminClient used on opensource projects

Example 11 with AdminClient

use of org.apache.kafka.clients.admin.AdminClient in project flink by apache.

the class KafkaSourceReaderTest method testCommitEmptyOffsets.

@Test
void testCommitEmptyOffsets() throws Exception {
    final String groupId = "testCommitEmptyOffsets";
    try (KafkaSourceReader<Integer> reader = (KafkaSourceReader<Integer>) createReader(Boundedness.CONTINUOUS_UNBOUNDED, groupId)) {
        reader.snapshotState(100L);
        reader.notifyCheckpointComplete(100L);
    }
    // Verify the committed offsets.
    try (AdminClient adminClient = KafkaSourceTestEnv.getAdminClient()) {
        Map<TopicPartition, OffsetAndMetadata> committedOffsets = adminClient.listConsumerGroupOffsets(groupId).partitionsToOffsetAndMetadata().get();
        assertThat(committedOffsets).isEmpty();
    }
}

Example 12 with AdminClient

use of org.apache.kafka.clients.admin.AdminClient in project flink by apache.

the class KafkaSourceReaderTest method setup.

@BeforeAll
public static void setup() throws Throwable {
    KafkaSourceTestEnv.setup();
    try (AdminClient adminClient = KafkaSourceTestEnv.getAdminClient()) {
        adminClient.createTopics(Collections.singleton(new NewTopic(TOPIC, NUM_PARTITIONS, (short) 1)));
        // Use the admin client to trigger the creation of internal __consumer_offsets topic.
        // This makes sure that we won't see unavailable coordinator in the tests.
        waitUtil(() -> {
            try {
                adminClient.listConsumerGroupOffsets("AnyGroup").partitionsToOffsetAndMetadata().get();
            } catch (Exception e) {
                return false;
            }
            return true;
        }, Duration.ofSeconds(60), "Waiting for offsets topic creation failed.");
    }
    KafkaSourceTestEnv.produceToKafka(getRecords(), StringSerializer.class, IntegerSerializer.class);
}

Example 13 with AdminClient

use of org.apache.kafka.clients.admin.AdminClient in project flink by apache.

the class KafkaContainerClient method createTopic.

public void createTopic(int replicationFactor, int numPartitions, String topic) {
    Map<String, Object> properties = new HashMap<>();
    properties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, container.getBootstrapServers());
    try (AdminClient admin = AdminClient.create(properties)) {
        admin.createTopics(Collections.singletonList(new NewTopic(topic, numPartitions, (short) replicationFactor)));
    }
}

Example 14 with AdminClient

use of org.apache.kafka.clients.admin.AdminClient in project kafka by apache.

the class TopicBasedRemoteLogMetadataManager method initializeResources.

private void initializeResources() {
    log.info("Initializing the resources.");
    final NewTopic remoteLogMetadataTopicRequest = createRemoteLogMetadataTopicRequest();
    boolean topicCreated = false;
    long startTimeMs = time.milliseconds();
    AdminClient adminClient = null;
    try {
        adminClient = AdminClient.create(rlmmConfig.producerProperties());
        // Stop if it is already initialized or closing.
        while (!(initialized.get() || closing.get())) {
            // If it is timed out then raise an error to exit.
            if (time.milliseconds() - startTimeMs > rlmmConfig.initializationRetryMaxTimeoutMs()) {
                log.error("Timed out in initializing the resources, retried to initialize the resource for [{}] ms.", rlmmConfig.initializationRetryMaxTimeoutMs());
                initializationFailed = true;
                return;
            }
            if (!topicCreated) {
                topicCreated = createTopic(adminClient, remoteLogMetadataTopicRequest);
            }
            if (!topicCreated) {
                // Sleep for INITIALIZATION_RETRY_INTERVAL_MS before trying to create the topic again.
                log.info("Sleep for : {} ms before it is retried again.", rlmmConfig.initializationRetryIntervalMs());
                Utils.sleep(rlmmConfig.initializationRetryIntervalMs());
                continue;
            } else {
                // If topic is already created, validate the existing topic partitions.
                try {
                    String topicName = remoteLogMetadataTopicRequest.name();
                    // If the existing topic partition size is not same as configured, mark initialization as failed and exit.
                    if (!isPartitionsCountSameAsConfigured(adminClient, topicName)) {
                        initializationFailed = true;
                    }
                } catch (Exception e) {
                    log.info("Sleep for : {} ms before it is retried again.", rlmmConfig.initializationRetryIntervalMs());
                    Utils.sleep(rlmmConfig.initializationRetryIntervalMs());
                    continue;
                }
            }
            // Create producer and consumer managers.
            lock.writeLock().lock();
            try {
                producerManager = new ProducerManager(rlmmConfig, rlmmTopicPartitioner);
                consumerManager = new ConsumerManager(rlmmConfig, remotePartitionMetadataStore, rlmmTopicPartitioner, time);
                if (startConsumerThread) {
                    consumerManager.startConsumerThread();
                } else {
                    log.info("RLMM Consumer task thread is not configured to be started.");
                }
                if (!pendingAssignPartitions.isEmpty()) {
                    assignPartitions(pendingAssignPartitions);
                    pendingAssignPartitions.clear();
                }
                initialized.set(true);
                log.info("Initialized resources successfully.");
            } catch (Exception e) {
                log.error("Encountered error while initializing producer/consumer", e);
                return;
            } finally {
                lock.writeLock().unlock();
            }
        }
    } finally {
        if (adminClient != null) {
            try {
                adminClient.close(Duration.ofSeconds(10));
            } catch (Exception e) {
                // Ignore the error.
                log.debug("Error occurred while closing the admin client", e);
            }
        }
    }
}

Example 15 with AdminClient

use of org.apache.kafka.clients.admin.AdminClient in project kafka by apache.

the class InternalTopicManagerTest method shouldThrowTimeoutExceptionWhenFuturesNeverCompleteDuringValidation.

@Test
public void shouldThrowTimeoutExceptionWhenFuturesNeverCompleteDuringValidation() {
    final AdminClient admin = EasyMock.createNiceMock(AdminClient.class);
    final MockTime time = new MockTime((Integer) config.get(StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG)) / 3);
    final InternalTopicManager topicManager = new InternalTopicManager(time, admin, new StreamsConfig(config));
    final KafkaFutureImpl<TopicDescription> topicDescriptionFutureThatNeverCompletes = new KafkaFutureImpl<>();
    EasyMock.expect(admin.describeTopics(Collections.singleton(topic1))).andStubAnswer(() -> new MockDescribeTopicsResult(mkMap(mkEntry(topic1, topicDescriptionFutureThatNeverCompletes))));
    final KafkaFutureImpl<Config> topicConfigSuccessfulFuture = new KafkaFutureImpl<>();
    topicConfigSuccessfulFuture.complete(new Config(repartitionTopicConfig().entrySet().stream().map(entry -> new ConfigEntry(entry.getKey(), entry.getValue())).collect(Collectors.toSet())));
    final ConfigResource topicResource = new ConfigResource(Type.TOPIC, topic1);
    EasyMock.expect(admin.describeConfigs(Collections.singleton(topicResource))).andStubAnswer(() -> new MockDescribeConfigsResult(mkMap(mkEntry(topicResource, topicConfigSuccessfulFuture))));
    EasyMock.replay(admin);
    final InternalTopicConfig internalTopicConfig = setupRepartitionTopicConfig(topic1, 1);
    assertThrows(TimeoutException.class, () -> topicManager.validate(Collections.singletonMap(topic1, internalTopicConfig)));
}