Examples with NewTopic org.apache.kafka.clients.admin.NewTopic used on opensource projects

Example 81 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class InternalTopicManagerTest method shouldThrowTimeoutExceptionWhenFuturesNeverCompleteDuringSetup.

@Test
public void shouldThrowTimeoutExceptionWhenFuturesNeverCompleteDuringSetup() {
    final AdminClient admin = EasyMock.createNiceMock(AdminClient.class);
    final MockTime time = new MockTime((Integer) config.get(StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG)) / 3);
    final StreamsConfig streamsConfig = new StreamsConfig(config);
    final InternalTopicManager topicManager = new InternalTopicManager(time, admin, streamsConfig);
    final KafkaFutureImpl<TopicMetadataAndConfig> createTopicFutureThatNeverCompletes = new KafkaFutureImpl<>();
    final InternalTopicConfig internalTopicConfig = setupRepartitionTopicConfig(topic1, 1);
    final NewTopic newTopic = newTopic(topic1, internalTopicConfig, streamsConfig);
    EasyMock.expect(admin.createTopics(mkSet(newTopic))).andStubAnswer(() -> new MockCreateTopicsResult(mkMap(mkEntry(topic1, createTopicFutureThatNeverCompletes))));
    EasyMock.replay(admin);
    assertThrows(TimeoutException.class, () -> topicManager.setup(Collections.singletonMap(topic1, internalTopicConfig)));
}

Example 82 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class InternalTopicManagerTest method shouldCompleteTopicValidationOnRetry.

@Test
public void shouldCompleteTopicValidationOnRetry() {
    final AdminClient admin = EasyMock.createNiceMock(AdminClient.class);
    final InternalTopicManager topicManager = new InternalTopicManager(time, admin, new StreamsConfig(config));
    final TopicPartitionInfo partitionInfo = new TopicPartitionInfo(0, broker1, Collections.singletonList(broker1), Collections.singletonList(broker1));
    final KafkaFutureImpl<TopicDescription> topicDescriptionSuccessFuture = new KafkaFutureImpl<>();
    final KafkaFutureImpl<TopicDescription> topicDescriptionFailFuture = new KafkaFutureImpl<>();
    topicDescriptionSuccessFuture.complete(new TopicDescription(topic1, false, Collections.singletonList(partitionInfo), Collections.emptySet()));
    topicDescriptionFailFuture.completeExceptionally(new UnknownTopicOrPartitionException("KABOOM!"));
    final KafkaFutureImpl<CreateTopicsResult.TopicMetadataAndConfig> topicCreationFuture = new KafkaFutureImpl<>();
    topicCreationFuture.completeExceptionally(new TopicExistsException("KABOOM!"));
    // let the first describe succeed on topic, and fail on topic2, and then let creation throws topics-existed;
    // it should retry with just topic2 and then let it succeed
    EasyMock.expect(admin.describeTopics(mkSet(topic1, topic2))).andReturn(new MockDescribeTopicsResult(mkMap(mkEntry(topic1, topicDescriptionSuccessFuture), mkEntry(topic2, topicDescriptionFailFuture)))).once();
    EasyMock.expect(admin.createTopics(Collections.singleton(new NewTopic(topic2, Optional.of(1), Optional.of((short) 1)).configs(mkMap(mkEntry(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT), mkEntry(TopicConfig.MESSAGE_TIMESTAMP_TYPE_CONFIG, "CreateTime")))))).andReturn(new MockCreateTopicsResult(Collections.singletonMap(topic2, topicCreationFuture))).once();
    EasyMock.expect(admin.describeTopics(Collections.singleton(topic2))).andReturn(new MockDescribeTopicsResult(Collections.singletonMap(topic2, topicDescriptionSuccessFuture)));
    EasyMock.replay(admin);
    final InternalTopicConfig topicConfig = new UnwindowedChangelogTopicConfig(topic1, Collections.emptyMap());
    topicConfig.setNumberOfPartitions(1);
    final InternalTopicConfig topic2Config = new UnwindowedChangelogTopicConfig(topic2, Collections.emptyMap());
    topic2Config.setNumberOfPartitions(1);
    topicManager.makeReady(mkMap(mkEntry(topic1, topicConfig), mkEntry(topic2, topic2Config)));
    EasyMock.verify(admin);
}

Example 83 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class InternalTopicManagerTest method shouldCreateTopicWhenTopicLeaderNotAvailableAndThenTopicNotFound.

@Test
public void shouldCreateTopicWhenTopicLeaderNotAvailableAndThenTopicNotFound() {
    final AdminClient admin = EasyMock.createNiceMock(AdminClient.class);
    final InternalTopicManager topicManager = new InternalTopicManager(time, admin, new StreamsConfig(config));
    final KafkaFutureImpl<TopicDescription> topicDescriptionLeaderNotAvailableFuture = new KafkaFutureImpl<>();
    topicDescriptionLeaderNotAvailableFuture.completeExceptionally(new LeaderNotAvailableException("Leader Not Available!"));
    final KafkaFutureImpl<TopicDescription> topicDescriptionUnknownTopicFuture = new KafkaFutureImpl<>();
    topicDescriptionUnknownTopicFuture.completeExceptionally(new UnknownTopicOrPartitionException("Unknown Topic!"));
    final KafkaFutureImpl<CreateTopicsResult.TopicMetadataAndConfig> topicCreationFuture = new KafkaFutureImpl<>();
    topicCreationFuture.complete(EasyMock.createNiceMock(CreateTopicsResult.TopicMetadataAndConfig.class));
    EasyMock.expect(admin.describeTopics(Collections.singleton(topic1))).andReturn(new MockDescribeTopicsResult(Collections.singletonMap(topic1, topicDescriptionLeaderNotAvailableFuture))).once();
    // we would not need to call create-topics for the first time
    EasyMock.expect(admin.describeTopics(Collections.singleton(topic1))).andReturn(new MockDescribeTopicsResult(Collections.singletonMap(topic1, topicDescriptionUnknownTopicFuture))).once();
    EasyMock.expect(admin.createTopics(Collections.singleton(new NewTopic(topic1, Optional.of(1), Optional.of((short) 1)).configs(mkMap(mkEntry(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE), mkEntry(TopicConfig.MESSAGE_TIMESTAMP_TYPE_CONFIG, "CreateTime"), mkEntry(TopicConfig.SEGMENT_BYTES_CONFIG, "52428800"), mkEntry(TopicConfig.RETENTION_MS_CONFIG, "-1")))))).andReturn(new MockCreateTopicsResult(Collections.singletonMap(topic1, topicCreationFuture))).once();
    EasyMock.replay(admin);
    final InternalTopicConfig internalTopicConfig = new RepartitionTopicConfig(topic1, Collections.emptyMap());
    internalTopicConfig.setNumberOfPartitions(1);
    topicManager.makeReady(Collections.singletonMap(topic1, internalTopicConfig));
    EasyMock.verify(admin);
}

Example 84 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class TopicCreationTest method withDefaultTopicCreation.

@Test
public void withDefaultTopicCreation() {
    sourceProps = defaultConnectorPropsWithTopicCreation();
    // Setting here but they should be ignored for the default group
    sourceProps.put(TOPIC_CREATION_PREFIX + DEFAULT_TOPIC_CREATION_GROUP + "." + INCLUDE_REGEX_CONFIG, FOO_REGEX);
    sourceProps.put(TOPIC_CREATION_PREFIX + DEFAULT_TOPIC_CREATION_GROUP + "." + EXCLUDE_REGEX_CONFIG, BAR_REGEX);
    // verify config creation
    sourceConfig = new SourceConnectorConfig(MOCK_PLUGINS, sourceProps, true);
    assertTrue(sourceConfig.usesTopicCreation());
    assertEquals(DEFAULT_REPLICATION_FACTOR, (short) sourceConfig.topicCreationReplicationFactor(DEFAULT_TOPIC_CREATION_GROUP));
    assertEquals(DEFAULT_PARTITIONS, (int) sourceConfig.topicCreationPartitions(DEFAULT_TOPIC_CREATION_GROUP));
    assertThat(sourceConfig.topicCreationInclude(DEFAULT_TOPIC_CREATION_GROUP), is(Collections.singletonList(".*")));
    assertThat(sourceConfig.topicCreationExclude(DEFAULT_TOPIC_CREATION_GROUP), is(Collections.emptyList()));
    assertThat(sourceConfig.topicCreationOtherConfigs(DEFAULT_TOPIC_CREATION_GROUP), is(Collections.emptyMap()));
    // verify topic creation group is instantiated correctly
    Map<String, TopicCreationGroup> groups = TopicCreationGroup.configuredGroups(sourceConfig);
    assertEquals(1, groups.size());
    assertThat(groups.keySet(), hasItem(DEFAULT_TOPIC_CREATION_GROUP));
    // verify topic creation
    TopicCreation topicCreation = TopicCreation.newTopicCreation(workerConfig, groups);
    TopicCreationGroup group = topicCreation.defaultTopicGroup();
    // Default group will match all topics besides empty string
    assertTrue(group.matches(" "));
    assertTrue(group.matches(FOO_TOPIC));
    assertEquals(DEFAULT_TOPIC_CREATION_GROUP, group.name());
    assertTrue(topicCreation.isTopicCreationEnabled());
    assertTrue(topicCreation.isTopicCreationRequired(FOO_TOPIC));
    assertThat(topicCreation.topicGroups(), is(Collections.emptyMap()));
    assertEquals(topicCreation.defaultTopicGroup(), topicCreation.findFirstGroup(FOO_TOPIC));
    topicCreation.addTopic(FOO_TOPIC);
    assertFalse(topicCreation.isTopicCreationRequired(FOO_TOPIC));
    // verify new topic properties
    NewTopic topicSpec = topicCreation.findFirstGroup(FOO_TOPIC).newTopic(FOO_TOPIC);
    assertEquals(FOO_TOPIC, topicSpec.name());
    assertEquals(DEFAULT_REPLICATION_FACTOR, topicSpec.replicationFactor());
    assertEquals(DEFAULT_PARTITIONS, topicSpec.numPartitions());
    assertThat(topicSpec.configs(), is(Collections.emptyMap()));
}

Example 85 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class TopicCreationTest method topicCreationWithTwoGroups.

@Test
public void topicCreationWithTwoGroups() {
    short fooReplicas = 3;
    int partitions = 5;
    int barPartitions = 1;
    sourceProps = defaultConnectorPropsWithTopicCreation();
    sourceProps.put(TOPIC_CREATION_GROUPS_CONFIG, String.join(",", FOO_GROUP, BAR_GROUP));
    sourceProps.put(DEFAULT_TOPIC_CREATION_PREFIX + PARTITIONS_CONFIG, String.valueOf(partitions));
    // Setting here but they should be ignored for the default group
    sourceProps.put(TOPIC_CREATION_PREFIX + FOO_GROUP + "." + INCLUDE_REGEX_CONFIG, FOO_TOPIC);
    sourceProps.put(TOPIC_CREATION_PREFIX + FOO_GROUP + "." + REPLICATION_FACTOR_CONFIG, String.valueOf(fooReplicas));
    sourceProps.put(TOPIC_CREATION_PREFIX + BAR_GROUP + "." + INCLUDE_REGEX_CONFIG, BAR_REGEX);
    sourceProps.put(TOPIC_CREATION_PREFIX + BAR_GROUP + "." + PARTITIONS_CONFIG, String.valueOf(barPartitions));
    Map<String, String> fooTopicProps = new HashMap<>();
    fooTopicProps.put(RETENTION_MS_CONFIG, String.valueOf(TimeUnit.DAYS.toMillis(30)));
    fooTopicProps.forEach((k, v) -> sourceProps.put(TOPIC_CREATION_PREFIX + FOO_GROUP + "." + k, v));
    Map<String, String> barTopicProps = new HashMap<>();
    barTopicProps.put(CLEANUP_POLICY_CONFIG, CLEANUP_POLICY_COMPACT);
    barTopicProps.forEach((k, v) -> sourceProps.put(TOPIC_CREATION_PREFIX + BAR_GROUP + "." + k, v));
    // verify config creation
    sourceConfig = new SourceConnectorConfig(MOCK_PLUGINS, sourceProps, true);
    assertTrue(sourceConfig.usesTopicCreation());
    assertEquals(DEFAULT_REPLICATION_FACTOR, (short) sourceConfig.topicCreationReplicationFactor(DEFAULT_TOPIC_CREATION_GROUP));
    assertEquals(partitions, (int) sourceConfig.topicCreationPartitions(DEFAULT_TOPIC_CREATION_GROUP));
    assertThat(sourceConfig.topicCreationInclude(DEFAULT_TOPIC_CREATION_GROUP), is(Collections.singletonList(".*")));
    assertThat(sourceConfig.topicCreationExclude(DEFAULT_TOPIC_CREATION_GROUP), is(Collections.emptyList()));
    assertThat(sourceConfig.topicCreationOtherConfigs(DEFAULT_TOPIC_CREATION_GROUP), is(Collections.emptyMap()));
    // verify topic creation group is instantiated correctly
    Map<String, TopicCreationGroup> groups = TopicCreationGroup.configuredGroups(sourceConfig);
    assertEquals(3, groups.size());
    assertThat(groups.keySet(), hasItems(DEFAULT_TOPIC_CREATION_GROUP, FOO_GROUP, BAR_GROUP));
    // verify topic creation
    TopicCreation topicCreation = TopicCreation.newTopicCreation(workerConfig, groups);
    TopicCreationGroup defaultGroup = topicCreation.defaultTopicGroup();
    // Default group will match all topics besides empty string
    assertTrue(defaultGroup.matches(" "));
    assertTrue(defaultGroup.matches(FOO_TOPIC));
    assertTrue(defaultGroup.matches(BAR_TOPIC));
    assertEquals(DEFAULT_TOPIC_CREATION_GROUP, defaultGroup.name());
    TopicCreationGroup fooGroup = groups.get(FOO_GROUP);
    assertFalse(fooGroup.matches(" "));
    assertTrue(fooGroup.matches(FOO_TOPIC));
    assertFalse(fooGroup.matches(BAR_TOPIC));
    assertEquals(FOO_GROUP, fooGroup.name());
    TopicCreationGroup barGroup = groups.get(BAR_GROUP);
    assertTrue(barGroup.matches(BAR_TOPIC));
    assertFalse(barGroup.matches(FOO_TOPIC));
    assertEquals(BAR_GROUP, barGroup.name());
    assertTrue(topicCreation.isTopicCreationEnabled());
    assertTrue(topicCreation.isTopicCreationRequired(FOO_TOPIC));
    assertTrue(topicCreation.isTopicCreationRequired(BAR_TOPIC));
    assertEquals(2, topicCreation.topicGroups().size());
    assertThat(topicCreation.topicGroups().keySet(), hasItems(FOO_GROUP, BAR_GROUP));
    assertEquals(fooGroup, topicCreation.findFirstGroup(FOO_TOPIC));
    assertEquals(barGroup, topicCreation.findFirstGroup(BAR_TOPIC));
    topicCreation.addTopic(FOO_TOPIC);
    topicCreation.addTopic(BAR_TOPIC);
    assertFalse(topicCreation.isTopicCreationRequired(FOO_TOPIC));
    assertFalse(topicCreation.isTopicCreationRequired(BAR_TOPIC));
    // verify new topic properties
    String otherTopic = "any-other-topic";
    NewTopic defaultTopicSpec = topicCreation.findFirstGroup(otherTopic).newTopic(otherTopic);
    assertEquals(otherTopic, defaultTopicSpec.name());
    assertEquals(DEFAULT_REPLICATION_FACTOR, defaultTopicSpec.replicationFactor());
    assertEquals(partitions, defaultTopicSpec.numPartitions());
    assertThat(defaultTopicSpec.configs(), is(Collections.emptyMap()));
    NewTopic fooTopicSpec = topicCreation.findFirstGroup(FOO_TOPIC).newTopic(FOO_TOPIC);
    assertEquals(FOO_TOPIC, fooTopicSpec.name());
    assertEquals(fooReplicas, fooTopicSpec.replicationFactor());
    assertEquals(partitions, fooTopicSpec.numPartitions());
    assertThat(fooTopicSpec.configs(), is(fooTopicProps));
    NewTopic barTopicSpec = topicCreation.findFirstGroup(BAR_TOPIC).newTopic(BAR_TOPIC);
    assertEquals(BAR_TOPIC, barTopicSpec.name());
    assertEquals(DEFAULT_REPLICATION_FACTOR, barTopicSpec.replicationFactor());
    assertEquals(barPartitions, barTopicSpec.numPartitions());
    assertThat(barTopicSpec.configs(), is(barTopicProps));
}