Examples with NewTopic org.apache.kafka.clients.admin.NewTopic used on opensource projects

Example 56 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project cruise-control by linkedin.

the class CruiseControlMetricsReporterAutoCreateTopicTest method setUp.

/**
 * Setup the unit test.
 */
@Before
public void setUp() {
    super.setUp();
    // creating the "TestTopic" explicitly because the topic auto-creation is disabled on the broker
    Properties adminProps = new Properties();
    adminProps.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers());
    AdminClient adminClient = AdminClient.create(adminProps);
    NewTopic testTopic = new NewTopic(TEST_TOPIC, 1, (short) 1);
    CreateTopicsResult createTopicsResult = adminClient.createTopics(Collections.singleton(testTopic));
    AtomicInteger adminFailed = new AtomicInteger(0);
    createTopicsResult.all().whenComplete((v, e) -> {
        if (e != null) {
            adminFailed.incrementAndGet();
        }
    });
    assertEquals(0, adminFailed.get());
    // starting producer to verify that Kafka cluster is working fine
    Properties producerProps = new Properties();
    producerProps.setProperty(ProducerConfig.ACKS_CONFIG, "-1");
    AtomicInteger producerFailed = new AtomicInteger(0);
    try (Producer<String, String> producer = createProducer(producerProps)) {
        for (int i = 0; i < 10; i++) {
            producer.send(new ProducerRecord<>(TEST_TOPIC, Integer.toString(i)), (m, e) -> {
                if (e != null) {
                    producerFailed.incrementAndGet();
                }
            });
        }
    }
    assertEquals(0, producerFailed.get());
}

Example 57 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project cruise-control by linkedin.

the class DiskFailureIntegrationTest method testDiskFailure.

@Test
public void testDiskFailure() throws IOException {
    AdminClient adminClient = KafkaCruiseControlUtils.createAdminClient(Collections.singletonMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, broker(0).plaintextAddr()));
    try {
        adminClient.createTopics(Collections.singleton(new NewTopic(TOPIC0, PARTITION_COUNT, TOPIC0_REPLICATION_FACTOR)));
    } finally {
        KafkaCruiseControlUtils.closeAdminClientWithTimeout(adminClient);
    }
    waitForMetadataPropagates();
    KafkaCruiseControlIntegrationTestUtils.produceRandomDataToTopic(TOPIC0, 5, 400, KafkaCruiseControlIntegrationTestUtils.getDefaultProducerProperties(bootstrapServers()));
    waitForProposal();
    Entry<File, File> entry = _brokerLogDirs.get(BROKER_ID_TO_CAUSE_DISK_FAILURE);
    FileUtils.deleteDirectory(entry.getKey());
    LOG.info("Disk failure injected");
    waitForOfflineReplicaDetection();
    waitForDiskFailureFixStarted();
    waitForOfflineReplicasFixed();
}

Example 58 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project cruise-control by linkedin.

the class ExecutorTest method testMoveNonExistingPartition.

@Test
public void testMoveNonExistingPartition() throws InterruptedException {
    ZkUtils zkUtils = KafkaCruiseControlUnitTestUtils.zkUtils(zookeeper().getConnectionString());
    AdminClient adminClient = getAdminClient(broker(0).getPlaintextAddr());
    adminClient.createTopics(Arrays.asList(new NewTopic(TOPIC_0, 1, (short) 1), new NewTopic(TOPIC_1, 1, (short) 2)));
    Map<String, TopicDescription> topicDescriptions = createTopics();
    int initialLeader0 = topicDescriptions.get(TOPIC_0).partitions().get(0).leader().id();
    int initialLeader1 = topicDescriptions.get(TOPIC_1).partitions().get(0).leader().id();
    ExecutionProposal proposal0 = new ExecutionProposal(TP0, 0, initialLeader0, Collections.singletonList(initialLeader0), Collections.singletonList(initialLeader0 == 0 ? 1 : 0));
    ExecutionProposal proposal1 = new ExecutionProposal(TP1, 0, initialLeader1, Arrays.asList(initialLeader1, initialLeader1 == 0 ? 1 : 0), Arrays.asList(initialLeader1 == 0 ? 1 : 0, initialLeader1));
    ExecutionProposal proposal2 = new ExecutionProposal(TP2, 0, initialLeader0, Collections.singletonList(initialLeader0), Collections.singletonList(initialLeader0 == 0 ? 1 : 0));
    ExecutionProposal proposal3 = new ExecutionProposal(TP3, 0, initialLeader1, Arrays.asList(initialLeader1, initialLeader1 == 0 ? 1 : 0), Arrays.asList(initialLeader1 == 0 ? 1 : 0, initialLeader1));
    Collection<ExecutionProposal> proposalsToExecute = Arrays.asList(proposal0, proposal1, proposal2, proposal3);
    Collection<ExecutionProposal> proposalsToCheck = Arrays.asList(proposal0, proposal1);
    executeAndVerifyProposals(zkUtils, proposalsToExecute, proposalsToCheck);
}

Example 59 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project cruise-control by linkedin.

the class ExecutorTest method createTopics.

private Map<String, TopicDescription> createTopics() throws InterruptedException {
    AdminClient adminClient = getAdminClient(broker(0).getPlaintextAddr());
    adminClient.createTopics(Arrays.asList(new NewTopic(TOPIC_0, 1, (short) 1), new NewTopic(TOPIC_1, 1, (short) 2)));
    // We need to use the admin clients to query the metadata from two different brokers to make sure that
    // both brokers have the latest metadata. Otherwise the Executor may get confused when it does not
    // see expected topics in the metadata.
    Map<String, TopicDescription> topicDescriptions0 = null;
    Map<String, TopicDescription> topicDescriptions1 = null;
    do {
        try (AdminClient adminClient0 = getAdminClient(broker(0).getPlaintextAddr());
            AdminClient adminClient1 = getAdminClient(broker(1).getPlaintextAddr())) {
            topicDescriptions0 = adminClient0.describeTopics(Arrays.asList(TOPIC_0, TOPIC_1)).all().get();
            topicDescriptions1 = adminClient1.describeTopics(Arrays.asList(TOPIC_0, TOPIC_1)).all().get();
            try {
                Thread.sleep(100);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        } catch (ExecutionException ee) {
        // Let it go.
        }
    } while (topicDescriptions0 == null || topicDescriptions0.size() < 2 || topicDescriptions1 == null || topicDescriptions1.size() < 2);
    return topicDescriptions0;
}

Example 60 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project strimzi by strimzi.

the class TopicSerializationTest method testToNewTopic.

@Test
public void testToNewTopic() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
    Topic topic = new Topic.Builder().withTopicName("test-topic").withConfigEntry("foo", "bar").withNumPartitions(3).withNumReplicas((short) 2).withMapName("gee").build();
    NewTopic newTopic = TopicSerialization.toNewTopic(topic, null);
    assertEquals("test-topic", newTopic.name());
    assertEquals(3, newTopic.numPartitions());
    assertEquals(2, newTopic.replicationFactor());
    assertEquals(null, newTopic.replicasAssignments());
    // For some reason Kafka doesn't provide an accessor for the config
    // and only provides a package-access method to convert to topic details
    Method m = NewTopic.class.getDeclaredMethod("convertToTopicDetails");
    m.setAccessible(true);
    CreateTopicsRequest.TopicDetails details = (CreateTopicsRequest.TopicDetails) m.invoke(newTopic);
    assertEquals(singletonMap("foo", "bar"), details.configs);
}