Examples with NewTopic org.apache.kafka.clients.admin.NewTopic used on opensource projects

Example 31 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class InternalTopicManagerTest method setupCleanUpScenario.

private void setupCleanUpScenario(final AdminClient admin, final StreamsConfig streamsConfig, final InternalTopicConfig internalTopicConfig1, final InternalTopicConfig internalTopicConfig2) {
    final KafkaFutureImpl<TopicMetadataAndConfig> createTopicFailFuture1 = new KafkaFutureImpl<>();
    createTopicFailFuture1.completeExceptionally(new TopicExistsException("exists"));
    final KafkaFutureImpl<TopicMetadataAndConfig> createTopicFailFuture2 = new KafkaFutureImpl<>();
    createTopicFailFuture2.completeExceptionally(new IllegalStateException("Nobody expects the Spanish inquisition"));
    final KafkaFutureImpl<TopicMetadataAndConfig> createTopicSuccessfulFuture = new KafkaFutureImpl<>();
    createTopicSuccessfulFuture.complete(new TopicMetadataAndConfig(Uuid.randomUuid(), 1, 1, new Config(Collections.emptyList())));
    final NewTopic newTopic1 = newTopic(topic1, internalTopicConfig1, streamsConfig);
    final NewTopic newTopic2 = newTopic(topic2, internalTopicConfig2, streamsConfig);
    EasyMock.expect(admin.createTopics(mkSet(newTopic1, newTopic2))).andAnswer(() -> new MockCreateTopicsResult(mkMap(mkEntry(topic1, createTopicSuccessfulFuture), mkEntry(topic2, createTopicFailFuture1))));
    EasyMock.expect(admin.createTopics(mkSet(newTopic2))).andAnswer(() -> new MockCreateTopicsResult(mkMap(mkEntry(topic2, createTopicFailFuture2))));
}

Example 32 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class InternalTopicManagerTest method shouldCleanUpWhenCreateTopicsTimesOut.

@Test
public void shouldCleanUpWhenCreateTopicsTimesOut() {
    final AdminClient admin = EasyMock.createNiceMock(AdminClient.class);
    final StreamsConfig streamsConfig = new StreamsConfig(config);
    final MockTime time = new MockTime((Integer) config.get(StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG)) / 3);
    final InternalTopicManager topicManager = new InternalTopicManager(time, admin, streamsConfig);
    final InternalTopicConfig internalTopicConfig1 = setupRepartitionTopicConfig(topic1, 1);
    final InternalTopicConfig internalTopicConfig2 = setupRepartitionTopicConfig(topic2, 1);
    final KafkaFutureImpl<TopicMetadataAndConfig> createTopicFailFuture1 = new KafkaFutureImpl<>();
    createTopicFailFuture1.completeExceptionally(new TopicExistsException("exists"));
    final KafkaFutureImpl<TopicMetadataAndConfig> createTopicSuccessfulFuture = new KafkaFutureImpl<>();
    createTopicSuccessfulFuture.complete(new TopicMetadataAndConfig(Uuid.randomUuid(), 1, 1, new Config(Collections.emptyList())));
    final NewTopic newTopic1 = newTopic(topic1, internalTopicConfig1, streamsConfig);
    final NewTopic newTopic2 = newTopic(topic2, internalTopicConfig2, streamsConfig);
    EasyMock.expect(admin.createTopics(mkSet(newTopic1, newTopic2))).andAnswer(() -> new MockCreateTopicsResult(mkMap(mkEntry(topic1, createTopicSuccessfulFuture), mkEntry(topic2, createTopicFailFuture1))));
    final KafkaFutureImpl<TopicMetadataAndConfig> createTopicFutureThatNeverCompletes = new KafkaFutureImpl<>();
    EasyMock.expect(admin.createTopics(mkSet(newTopic2))).andStubAnswer(() -> new MockCreateTopicsResult(mkMap(mkEntry(topic2, createTopicFutureThatNeverCompletes))));
    final KafkaFutureImpl<Void> deleteTopicSuccessfulFuture = new KafkaFutureImpl<>();
    deleteTopicSuccessfulFuture.complete(null);
    EasyMock.expect(admin.deleteTopics(mkSet(topic1))).andAnswer(() -> new MockDeleteTopicsResult(mkMap(mkEntry(topic1, deleteTopicSuccessfulFuture))));
    EasyMock.replay(admin);
    assertThrows(TimeoutException.class, () -> topicManager.setup(mkMap(mkEntry(topic1, internalTopicConfig1), mkEntry(topic2, internalTopicConfig2))));
    EasyMock.verify(admin);
}

Example 33 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class WorkerUtilsTest method testExistingTopicsNotCreated.

@Test
public void testExistingTopicsNotCreated() throws Throwable {
    final String existingTopic = "existing-topic";
    List<TopicPartitionInfo> tpInfo = new ArrayList<>();
    tpInfo.add(new TopicPartitionInfo(0, broker1, singleReplica, Collections.<Node>emptyList()));
    tpInfo.add(new TopicPartitionInfo(1, broker2, singleReplica, Collections.<Node>emptyList()));
    tpInfo.add(new TopicPartitionInfo(2, broker3, singleReplica, Collections.<Node>emptyList()));
    adminClient.addTopic(false, existingTopic, tpInfo, null);
    WorkerUtils.createTopics(log, adminClient, Collections.singletonMap(existingTopic, new NewTopic(existingTopic, tpInfo.size(), TEST_REPLICATION_FACTOR)), false);
    assertEquals(Collections.singleton(existingTopic), adminClient.listTopics().names().get());
}

Example 34 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class WorkerUtilsTest method testCreateTopicsFailsIfAtLeastOneTopicExists.

@Test
public void testCreateTopicsFailsIfAtLeastOneTopicExists() throws Throwable {
    adminClient.addTopic(false, TEST_TOPIC, Collections.singletonList(new TopicPartitionInfo(0, broker1, singleReplica, Collections.<Node>emptyList())), null);
    Map<String, NewTopic> newTopics = new HashMap<>();
    newTopics.put(TEST_TOPIC, NEW_TEST_TOPIC);
    newTopics.put("another-topic", new NewTopic("another-topic", TEST_PARTITIONS, TEST_REPLICATION_FACTOR));
    newTopics.put("one-more-topic", new NewTopic("one-more-topic", TEST_PARTITIONS, TEST_REPLICATION_FACTOR));
    assertThrows(TopicExistsException.class, () -> WorkerUtils.createTopics(log, adminClient, newTopics, true));
}

Example 35 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class WorkerSourceTaskTest method testSendRecordsTopicCreateRetries.

@Test
public void testSendRecordsTopicCreateRetries() throws Exception {
    if (!enableTopicCreation)
        // should only test with topic creation enabled
        return;
    createWorkerTask();
    SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD);
    SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD);
    // First call to describe the topic times out
    expectPreliminaryCalls();
    EasyMock.expect(admin.describeTopics(TOPIC)).andReturn(Collections.emptyMap());
    Capture<NewTopic> newTopicCapture = EasyMock.newCapture();
    EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))).andThrow(new RetriableException(new TimeoutException("timeout")));
    // Second round
    expectTopicCreation(TOPIC);
    expectSendRecordTaskCommitRecordSucceed(false);
    expectSendRecordTaskCommitRecordSucceed(false);
    PowerMock.replayAll();
    Whitebox.setInternalState(workerTask, "toSend", Arrays.asList(record1, record2));
    Whitebox.invokeMethod(workerTask, "sendRecords");
    assertEquals(Arrays.asList(record1, record2), Whitebox.getInternalState(workerTask, "toSend"));
    // Next they all succeed
    Whitebox.invokeMethod(workerTask, "sendRecords");
    assertNull(Whitebox.getInternalState(workerTask, "toSend"));
}