Examples with Admin org.apache.kafka.clients.admin.Admin used on opensource projects

Example 16 with Admin

use of org.apache.kafka.clients.admin.Admin in project kafka by apache.

the class MirrorConnectorsIntegrationBaseTest method testReplicationWithEmptyPartition.

@Test
public void testReplicationWithEmptyPartition() throws Exception {
    String consumerGroupName = "consumer-group-testReplicationWithEmptyPartition";
    Map<String, Object> consumerProps = Collections.singletonMap("group.id", consumerGroupName);
    // create topic
    String topic = "test-topic-with-empty-partition";
    primary.kafka().createTopic(topic, NUM_PARTITIONS);
    // produce to all test-topic-empty's partitions, except the last partition
    produceMessages(primary, topic, NUM_PARTITIONS - 1);
    // consume before starting the connectors so we don't need to wait for discovery
    int expectedRecords = NUM_RECORDS_PER_PARTITION * (NUM_PARTITIONS - 1);
    try (Consumer<byte[], byte[]> primaryConsumer = primary.kafka().createConsumerAndSubscribeTo(consumerProps, topic)) {
        waitForConsumingAllRecords(primaryConsumer, expectedRecords);
    }
    // one way replication from primary to backup
    mm2Props.put(BACKUP_CLUSTER_ALIAS + "->" + PRIMARY_CLUSTER_ALIAS + ".enabled", "false");
    mm2Config = new MirrorMakerConfig(mm2Props);
    waitUntilMirrorMakerIsRunning(backup, CONNECTOR_LIST, mm2Config, PRIMARY_CLUSTER_ALIAS, BACKUP_CLUSTER_ALIAS);
    // sleep few seconds to have MM2 finish replication so that "end" consumer will consume some record
    Thread.sleep(TimeUnit.SECONDS.toMillis(3));
    String backupTopic = PRIMARY_CLUSTER_ALIAS + "." + topic;
    // consume all records from backup cluster
    try (Consumer<byte[], byte[]> backupConsumer = backup.kafka().createConsumerAndSubscribeTo(consumerProps, backupTopic)) {
        waitForConsumingAllRecords(backupConsumer, expectedRecords);
    }
    try (Admin backupClient = backup.kafka().createAdminClient()) {
        // retrieve the consumer group offset from backup cluster
        Map<TopicPartition, OffsetAndMetadata> remoteOffsets = backupClient.listConsumerGroupOffsets(consumerGroupName).partitionsToOffsetAndMetadata().get();
        // pinpoint the offset of the last partition which does not receive records
        OffsetAndMetadata offset = remoteOffsets.get(new TopicPartition(backupTopic, NUM_PARTITIONS - 1));
        // offset of the last partition should exist, but its value should be 0
        assertNotNull(offset, "Offset of last partition was not replicated");
        assertEquals(0, offset.offset(), "Offset of last partition is not zero");
    }
}

Example 17 with Admin

use of org.apache.kafka.clients.admin.Admin in project kafka by apache.

the class MirrorConnectorsIntegrationBaseTest method getTopicConfig.

/*
     * retrieve the config value based on the input cluster, topic and config name
     */
protected static String getTopicConfig(EmbeddedKafkaCluster cluster, String topic, String configName) throws Exception {
    try (Admin client = cluster.createAdminClient()) {
        Collection<ConfigResource> cr = Collections.singleton(new ConfigResource(ConfigResource.Type.TOPIC, topic));
        DescribeConfigsResult configsResult = client.describeConfigs(cr);
        Config allConfigs = (Config) configsResult.all().get().values().toArray()[0];
        return allConfigs.get(configName).value();
    }
}

Example 18 with Admin

use of org.apache.kafka.clients.admin.Admin in project kafka by apache.

the class MirrorConnectorsIntegrationBaseTest method waitForConsumerGroupOffsetSync.

/*
     * given consumer group, topics and expected number of records, make sure the consumer group
     * offsets are eventually synced to the expected offset numbers
     */
protected static <T> void waitForConsumerGroupOffsetSync(EmbeddedConnectCluster connect, Consumer<T, T> consumer, List<String> topics, String consumerGroupId, int numRecords) throws InterruptedException {
    try (Admin adminClient = connect.kafka().createAdminClient()) {
        List<TopicPartition> tps = new ArrayList<>(NUM_PARTITIONS * topics.size());
        for (int partitionIndex = 0; partitionIndex < NUM_PARTITIONS; partitionIndex++) {
            for (String topic : topics) {
                tps.add(new TopicPartition(topic, partitionIndex));
            }
        }
        long expectedTotalOffsets = numRecords * topics.size();
        waitForCondition(() -> {
            Map<TopicPartition, OffsetAndMetadata> consumerGroupOffsets = adminClient.listConsumerGroupOffsets(consumerGroupId).partitionsToOffsetAndMetadata().get();
            long consumerGroupOffsetTotal = consumerGroupOffsets.values().stream().mapToLong(OffsetAndMetadata::offset).sum();
            Map<TopicPartition, Long> offsets = consumer.endOffsets(tps, CONSUMER_POLL_TIMEOUT_MS);
            long totalOffsets = offsets.values().stream().mapToLong(l -> l).sum();
            // make sure the consumer group offsets are synced to expected number
            return totalOffsets == expectedTotalOffsets && consumerGroupOffsetTotal > 0;
        }, OFFSET_SYNC_DURATION_MS, "Consumer group offset sync is not complete in time");
    }
}

Example 19 with Admin

use of org.apache.kafka.clients.admin.Admin in project kafka by apache.

the class MirrorConnectorsIntegrationBaseTest method deleteAllTopics.

/*
     * delete all topics of the input kafka cluster
     */
private static void deleteAllTopics(EmbeddedKafkaCluster cluster) throws Exception {
    try (final Admin adminClient = cluster.createAdminClient()) {
        Set<String> topicsToBeDeleted = adminClient.listTopics().names().get();
        log.debug("Deleting topics: {} ", topicsToBeDeleted);
        adminClient.deleteTopics(topicsToBeDeleted).all().get();
    }
}

Example 20 with Admin

use of org.apache.kafka.clients.admin.Admin in project kafka by apache.

the class IdentityReplicationIntegrationTest method testReplicationWithEmptyPartition.

@Test
public void testReplicationWithEmptyPartition() throws Exception {
    String consumerGroupName = "consumer-group-testReplicationWithEmptyPartition";
    Map<String, Object> consumerProps = Collections.singletonMap("group.id", consumerGroupName);
    // create topic
    String topic = "test-topic-with-empty-partition";
    primary.kafka().createTopic(topic, NUM_PARTITIONS);
    // produce to all test-topic-empty's partitions, except the last partition
    produceMessages(primary, topic, NUM_PARTITIONS - 1);
    // consume before starting the connectors so we don't need to wait for discovery
    int expectedRecords = NUM_RECORDS_PER_PARTITION * (NUM_PARTITIONS - 1);
    try (Consumer<byte[], byte[]> primaryConsumer = primary.kafka().createConsumerAndSubscribeTo(consumerProps, topic)) {
        waitForConsumingAllRecords(primaryConsumer, expectedRecords);
    }
    // one way replication from primary to backup
    mm2Props.put(BACKUP_CLUSTER_ALIAS + "->" + PRIMARY_CLUSTER_ALIAS + ".enabled", "false");
    mm2Config = new MirrorMakerConfig(mm2Props);
    waitUntilMirrorMakerIsRunning(backup, CONNECTOR_LIST, mm2Config, PRIMARY_CLUSTER_ALIAS, BACKUP_CLUSTER_ALIAS);
    // sleep few seconds to have MM2 finish replication so that "end" consumer will consume some record
    Thread.sleep(TimeUnit.SECONDS.toMillis(3));
    // note that with IdentityReplicationPolicy, topics on the backup are NOT renamed to PRIMARY_CLUSTER_ALIAS + "." + topic
    String backupTopic = topic;
    // consume all records from backup cluster
    try (Consumer<byte[], byte[]> backupConsumer = backup.kafka().createConsumerAndSubscribeTo(consumerProps, backupTopic)) {
        waitForConsumingAllRecords(backupConsumer, expectedRecords);
    }
    try (Admin backupClient = backup.kafka().createAdminClient()) {
        // retrieve the consumer group offset from backup cluster
        Map<TopicPartition, OffsetAndMetadata> remoteOffsets = backupClient.listConsumerGroupOffsets(consumerGroupName).partitionsToOffsetAndMetadata().get();
        // pinpoint the offset of the last partition which does not receive records
        OffsetAndMetadata offset = remoteOffsets.get(new TopicPartition(backupTopic, NUM_PARTITIONS - 1));
        // offset of the last partition should exist, but its value should be 0
        assertNotNull(offset, "Offset of last partition was not replicated");
        assertEquals(0, offset.offset(), "Offset of last partition is not zero");
    }
}