Examples with TopicDescription org.apache.kafka.clients.admin.TopicDescription used on opensource projects

Example 6 with TopicDescription

use of org.apache.kafka.clients.admin.TopicDescription in project apache-kafka-on-k8s by banzaicloud.

the class WorkerUtils method verifyTopics.

/**
 * Verifies that topics in 'topicsToVerify' list have the same number of partitions as
 * described in 'topicsInfo'
 * @param log                The logger to use
 * @param adminClient        AdminClient
 * @param topicsToVerify     List of topics to verify
 * @param topicsInfo         Map of topic name to topic description, which includes topics in
 *                           'topicsToVerify' list.
 * @throws RuntimeException  If one or more topics have different number of partitions than
 * described in 'topicsInfo'
 */
private static void verifyTopics(Logger log, AdminClient adminClient, Collection<String> topicsToVerify, Map<String, NewTopic> topicsInfo) throws Throwable {
    DescribeTopicsResult topicsResult = adminClient.describeTopics(topicsToVerify, new DescribeTopicsOptions().timeoutMs(CREATE_TOPICS_REQUEST_TIMEOUT));
    Map<String, TopicDescription> topicDescriptionMap = topicsResult.all().get();
    for (TopicDescription desc : topicDescriptionMap.values()) {
        // map will always contain the topic since all topics in 'topicsExists' are in given
        // 'topics' map
        int partitions = topicsInfo.get(desc.name()).numPartitions();
        if (desc.partitions().size() != partitions) {
            String str = "Topic '" + desc.name() + "' exists, but has " + desc.partitions().size() + " partitions, while requested " + " number of partitions is " + partitions;
            log.warn(str);
            throw new RuntimeException(str);
        }
    }
}

Example 7 with TopicDescription

use of org.apache.kafka.clients.admin.TopicDescription in project strimzi by strimzi.

the class Utils method getTopicMetadata.

public static TopicMetadata getTopicMetadata(Topic kubeTopic) {
    List<Node> nodes = new ArrayList<>();
    for (int nodeId = 0; nodeId < kubeTopic.getNumReplicas(); nodeId++) {
        nodes.add(new Node(nodeId, "localhost", 9092 + nodeId));
    }
    List<TopicPartitionInfo> partitions = new ArrayList<>();
    for (int partitionId = 0; partitionId < kubeTopic.getNumPartitions(); partitionId++) {
        partitions.add(new TopicPartitionInfo(partitionId, nodes.get(0), nodes, nodes));
    }
    List<ConfigEntry> configs = new ArrayList<>();
    for (Map.Entry<String, String> entry : kubeTopic.getConfig().entrySet()) {
        configs.add(new ConfigEntry(entry.getKey(), entry.getValue()));
    }
    return new TopicMetadata(new TopicDescription(kubeTopic.getTopicName().toString(), false, partitions), new Config(configs));
}

Example 8 with TopicDescription

use of org.apache.kafka.clients.admin.TopicDescription in project strimzi by strimzi.

the class Utils method getTopicMetadata.

public static TopicMetadata getTopicMetadata(String topicName, Config config) {
    Node node0 = new Node(0, "host0", 1234);
    Node node1 = new Node(1, "host1", 1234);
    Node node2 = new Node(2, "host2", 1234);
    List<Node> nodes02 = asList(node0, node1, node2);
    TopicDescription desc = new TopicDescription(topicName, false, asList(new TopicPartitionInfo(0, node0, nodes02, nodes02), new TopicPartitionInfo(1, node0, nodes02, nodes02)));
    // org.apache.kafka.clients.admin.Config config = new Config(configs);
    return new TopicMetadata(desc, config);
}

Example 9 with TopicDescription

use of org.apache.kafka.clients.admin.TopicDescription in project cruise-control by linkedin.

the class ExecutorTest method testMoveNonExistingPartition.

@Test
public void testMoveNonExistingPartition() throws InterruptedException {
    ZkUtils zkUtils = KafkaCruiseControlUnitTestUtils.zkUtils(zookeeper().getConnectionString());
    AdminClient adminClient = getAdminClient(broker(0).getPlaintextAddr());
    adminClient.createTopics(Arrays.asList(new NewTopic(TOPIC_0, 1, (short) 1), new NewTopic(TOPIC_1, 1, (short) 2)));
    Map<String, TopicDescription> topicDescriptions = createTopics();
    int initialLeader0 = topicDescriptions.get(TOPIC_0).partitions().get(0).leader().id();
    int initialLeader1 = topicDescriptions.get(TOPIC_1).partitions().get(0).leader().id();
    ExecutionProposal proposal0 = new ExecutionProposal(TP0, 0, initialLeader0, Collections.singletonList(initialLeader0), Collections.singletonList(initialLeader0 == 0 ? 1 : 0));
    ExecutionProposal proposal1 = new ExecutionProposal(TP1, 0, initialLeader1, Arrays.asList(initialLeader1, initialLeader1 == 0 ? 1 : 0), Arrays.asList(initialLeader1 == 0 ? 1 : 0, initialLeader1));
    ExecutionProposal proposal2 = new ExecutionProposal(TP2, 0, initialLeader0, Collections.singletonList(initialLeader0), Collections.singletonList(initialLeader0 == 0 ? 1 : 0));
    ExecutionProposal proposal3 = new ExecutionProposal(TP3, 0, initialLeader1, Arrays.asList(initialLeader1, initialLeader1 == 0 ? 1 : 0), Arrays.asList(initialLeader1 == 0 ? 1 : 0, initialLeader1));
    Collection<ExecutionProposal> proposalsToExecute = Arrays.asList(proposal0, proposal1, proposal2, proposal3);
    Collection<ExecutionProposal> proposalsToCheck = Arrays.asList(proposal0, proposal1);
    executeAndVerifyProposals(zkUtils, proposalsToExecute, proposalsToCheck);
}

Example 10 with TopicDescription

use of org.apache.kafka.clients.admin.TopicDescription in project cruise-control by linkedin.

the class ExecutorTest method createTopics.

private Map<String, TopicDescription> createTopics() throws InterruptedException {
    AdminClient adminClient = getAdminClient(broker(0).getPlaintextAddr());
    adminClient.createTopics(Arrays.asList(new NewTopic(TOPIC_0, 1, (short) 1), new NewTopic(TOPIC_1, 1, (short) 2)));
    // We need to use the admin clients to query the metadata from two different brokers to make sure that
    // both brokers have the latest metadata. Otherwise the Executor may get confused when it does not
    // see expected topics in the metadata.
    Map<String, TopicDescription> topicDescriptions0 = null;
    Map<String, TopicDescription> topicDescriptions1 = null;
    do {
        try (AdminClient adminClient0 = getAdminClient(broker(0).getPlaintextAddr());
            AdminClient adminClient1 = getAdminClient(broker(1).getPlaintextAddr())) {
            topicDescriptions0 = adminClient0.describeTopics(Arrays.asList(TOPIC_0, TOPIC_1)).all().get();
            topicDescriptions1 = adminClient1.describeTopics(Arrays.asList(TOPIC_0, TOPIC_1)).all().get();
            try {
                Thread.sleep(100);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        } catch (ExecutionException ee) {
        // Let it go.
        }
    } while (topicDescriptions0 == null || topicDescriptions0.size() < 2 || topicDescriptions1 == null || topicDescriptions1.size() < 2);
    return topicDescriptions0;
}