use of org.apache.kafka.streams.processor.internals.InternalTopicManager.ValidationResult in project kafka by apache.
the class InternalTopicManagerTest method shouldReportMisconfigurationsOfCleanupPolicyForRepartitionTopics.
@Test
public void shouldReportMisconfigurationsOfCleanupPolicyForRepartitionTopics() {
final long retentionMs = 1000;
setupTopicInMockAdminClient(topic1, repartitionTopicConfig());
final Map<String, String> repartitionTopicConfigCleanupPolicyCompact = repartitionTopicConfig();
repartitionTopicConfigCleanupPolicyCompact.put(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT);
setupTopicInMockAdminClient(topic2, repartitionTopicConfigCleanupPolicyCompact);
final Map<String, String> repartitionTopicConfigCleanupPolicyCompactAndDelete = repartitionTopicConfig();
repartitionTopicConfigCleanupPolicyCompactAndDelete.put(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT + "," + TopicConfig.CLEANUP_POLICY_DELETE);
setupTopicInMockAdminClient(topic3, repartitionTopicConfigCleanupPolicyCompactAndDelete);
final Map<String, String> repartitionTopicConfigWithFiniteRetentionMs = repartitionTopicConfig();
repartitionTopicConfigWithFiniteRetentionMs.put(TopicConfig.RETENTION_MS_CONFIG, String.valueOf(retentionMs));
setupTopicInMockAdminClient(topic4, repartitionTopicConfigWithFiniteRetentionMs);
final Map<String, String> repartitionTopicConfigWithRetentionBytesSet = repartitionTopicConfig();
repartitionTopicConfigWithRetentionBytesSet.put(TopicConfig.RETENTION_BYTES_CONFIG, "1024");
setupTopicInMockAdminClient(topic5, repartitionTopicConfigWithRetentionBytesSet);
final InternalTopicConfig internalTopicConfig1 = setupRepartitionTopicConfig(topic1, 1);
final InternalTopicConfig internalTopicConfig2 = setupRepartitionTopicConfig(topic2, 1);
final InternalTopicConfig internalTopicConfig3 = setupRepartitionTopicConfig(topic3, 1);
final InternalTopicConfig internalTopicConfig4 = setupRepartitionTopicConfig(topic4, 1);
final InternalTopicConfig internalTopicConfig5 = setupRepartitionTopicConfig(topic5, 1);
final ValidationResult validationResult = internalTopicManager.validate(mkMap(mkEntry(topic1, internalTopicConfig1), mkEntry(topic2, internalTopicConfig2), mkEntry(topic3, internalTopicConfig3), mkEntry(topic4, internalTopicConfig4), mkEntry(topic5, internalTopicConfig5)));
final Map<String, List<String>> misconfigurationsForTopics = validationResult.misconfigurationsForTopics();
assertThat(validationResult.missingTopics(), empty());
assertThat(misconfigurationsForTopics.size(), is(4));
assertThat(misconfigurationsForTopics, hasKey(topic2));
assertThat(misconfigurationsForTopics.get(topic2).size(), is(1));
assertThat(misconfigurationsForTopics.get(topic2).get(0), is("Cleanup policy (" + TopicConfig.CLEANUP_POLICY_CONFIG + ") of existing internal topic " + topic2 + " should not contain \"" + TopicConfig.CLEANUP_POLICY_COMPACT + "\"."));
assertThat(misconfigurationsForTopics, hasKey(topic3));
assertThat(misconfigurationsForTopics.get(topic3).size(), is(1));
assertThat(misconfigurationsForTopics.get(topic3).get(0), is("Cleanup policy (" + TopicConfig.CLEANUP_POLICY_CONFIG + ") of existing internal topic " + topic3 + " should not contain \"" + TopicConfig.CLEANUP_POLICY_COMPACT + "\"."));
assertThat(misconfigurationsForTopics, hasKey(topic4));
assertThat(misconfigurationsForTopics.get(topic4).size(), is(1));
assertThat(misconfigurationsForTopics.get(topic4).get(0), is("Retention time (" + TopicConfig.RETENTION_MS_CONFIG + ") of existing internal topic " + topic4 + " is " + retentionMs + " but should be -1."));
assertThat(misconfigurationsForTopics, hasKey(topic5));
assertThat(misconfigurationsForTopics.get(topic5).size(), is(1));
assertThat(misconfigurationsForTopics.get(topic5).get(0), is("Retention byte (" + TopicConfig.RETENTION_BYTES_CONFIG + ") of existing internal topic " + topic5 + " is set but it should be unset."));
}
use of org.apache.kafka.streams.processor.internals.InternalTopicManager.ValidationResult in project kafka by apache.
the class InternalTopicManagerTest method shouldReportMisconfigurationsOfCleanupPolicyForUnwindowedChangelogTopics.
@Test
public void shouldReportMisconfigurationsOfCleanupPolicyForUnwindowedChangelogTopics() {
final Map<String, String> unwindowedChangelogConfigWithDeleteCleanupPolicy = unwindowedChangelogConfig();
unwindowedChangelogConfigWithDeleteCleanupPolicy.put(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE);
setupTopicInMockAdminClient(topic1, unwindowedChangelogConfigWithDeleteCleanupPolicy);
final Map<String, String> unwindowedChangelogConfigWithDeleteCompactCleanupPolicy = unwindowedChangelogConfig();
unwindowedChangelogConfigWithDeleteCompactCleanupPolicy.put(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT + "," + TopicConfig.CLEANUP_POLICY_DELETE);
setupTopicInMockAdminClient(topic2, unwindowedChangelogConfigWithDeleteCompactCleanupPolicy);
setupTopicInMockAdminClient(topic3, unwindowedChangelogConfig());
final InternalTopicConfig internalTopicConfig1 = setupUnwindowedChangelogTopicConfig(topic1, 1);
final InternalTopicConfig internalTopicConfig2 = setupUnwindowedChangelogTopicConfig(topic2, 1);
final InternalTopicConfig internalTopicConfig3 = setupUnwindowedChangelogTopicConfig(topic3, 1);
final ValidationResult validationResult = internalTopicManager.validate(mkMap(mkEntry(topic1, internalTopicConfig1), mkEntry(topic2, internalTopicConfig2), mkEntry(topic3, internalTopicConfig3)));
final Map<String, List<String>> misconfigurationsForTopics = validationResult.misconfigurationsForTopics();
assertThat(validationResult.missingTopics(), empty());
assertThat(misconfigurationsForTopics.size(), is(2));
assertThat(misconfigurationsForTopics, hasKey(topic1));
assertThat(misconfigurationsForTopics.get(topic1).size(), is(1));
assertThat(misconfigurationsForTopics.get(topic1).get(0), is("Cleanup policy (" + TopicConfig.CLEANUP_POLICY_CONFIG + ") of existing internal topic " + topic1 + " should not contain \"" + TopicConfig.CLEANUP_POLICY_DELETE + "\"."));
assertThat(misconfigurationsForTopics, hasKey(topic2));
assertThat(misconfigurationsForTopics.get(topic2).size(), is(1));
assertThat(misconfigurationsForTopics.get(topic2).get(0), is("Cleanup policy (" + TopicConfig.CLEANUP_POLICY_CONFIG + ") of existing internal topic " + topic2 + " should not contain \"" + TopicConfig.CLEANUP_POLICY_DELETE + "\"."));
assertThat(misconfigurationsForTopics, not(hasKey(topic3)));
}
use of org.apache.kafka.streams.processor.internals.InternalTopicManager.ValidationResult in project kafka by apache.
the class InternalTopicManagerTest method shouldRetryWhenCallsThrowTimeoutExceptionDuringValidation.
@Test
public void shouldRetryWhenCallsThrowTimeoutExceptionDuringValidation() {
setupTopicInMockAdminClient(topic1, repartitionTopicConfig());
mockAdminClient.timeoutNextRequest(2);
final InternalTopicConfig internalTopicConfig = setupRepartitionTopicConfig(topic1, 1);
final ValidationResult validationResult = internalTopicManager.validate(Collections.singletonMap(topic1, internalTopicConfig));
assertThat(validationResult.missingTopics(), empty());
assertThat(validationResult.misconfigurationsForTopics(), anEmptyMap());
}
use of org.apache.kafka.streams.processor.internals.InternalTopicManager.ValidationResult in project kafka by apache.
the class InternalTopicManagerTest method shouldReportMisconfigurationsOfPartitionCount.
@Test
public void shouldReportMisconfigurationsOfPartitionCount() {
setupTopicInMockAdminClient(topic1, repartitionTopicConfig());
setupTopicInMockAdminClient(topic2, repartitionTopicConfig());
setupTopicInMockAdminClient(topic3, repartitionTopicConfig());
final InternalTopicConfig internalTopicConfig1 = setupRepartitionTopicConfig(topic1, 2);
final InternalTopicConfig internalTopicConfig2 = setupRepartitionTopicConfig(topic2, 3);
final InternalTopicConfig internalTopicConfig3 = setupRepartitionTopicConfig(topic3, 1);
final ValidationResult validationResult = internalTopicManager.validate(mkMap(mkEntry(topic1, internalTopicConfig1), mkEntry(topic2, internalTopicConfig2), mkEntry(topic3, internalTopicConfig3)));
final Map<String, List<String>> misconfigurationsForTopics = validationResult.misconfigurationsForTopics();
assertThat(validationResult.missingTopics(), empty());
assertThat(misconfigurationsForTopics.size(), is(2));
assertThat(misconfigurationsForTopics, hasKey(topic1));
assertThat(misconfigurationsForTopics.get(topic1).size(), is(1));
assertThat(misconfigurationsForTopics.get(topic1).get(0), is("Internal topic " + topic1 + " requires 2 partitions, but the existing topic on the broker has 1 partitions."));
assertThat(misconfigurationsForTopics, hasKey(topic2));
assertThat(misconfigurationsForTopics.get(topic2).size(), is(1));
assertThat(misconfigurationsForTopics.get(topic2).get(0), is("Internal topic " + topic2 + " requires 3 partitions, but the existing topic on the broker has 1 partitions."));
assertThat(misconfigurationsForTopics, not(hasKey(topic3)));
}
use of org.apache.kafka.streams.processor.internals.InternalTopicManager.ValidationResult in project kafka by apache.
the class InternalTopicManagerTest method shouldOnlyRetryDescribeTopicsWhenDescribeTopicsThrowsLeaderNotAvailableExceptionDuringValidation.
@Test
public void shouldOnlyRetryDescribeTopicsWhenDescribeTopicsThrowsLeaderNotAvailableExceptionDuringValidation() {
final AdminClient admin = EasyMock.createNiceMock(AdminClient.class);
final InternalTopicManager topicManager = new InternalTopicManager(time, admin, new StreamsConfig(config));
final KafkaFutureImpl<TopicDescription> topicDescriptionFailFuture = new KafkaFutureImpl<>();
topicDescriptionFailFuture.completeExceptionally(new LeaderNotAvailableException("Leader Not Available!"));
final KafkaFutureImpl<TopicDescription> topicDescriptionSuccessfulFuture = new KafkaFutureImpl<>();
topicDescriptionSuccessfulFuture.complete(new TopicDescription(topic1, false, Collections.singletonList(new TopicPartitionInfo(0, broker1, cluster, Collections.emptyList()))));
EasyMock.expect(admin.describeTopics(Collections.singleton(topic1))).andReturn(new MockDescribeTopicsResult(mkMap(mkEntry(topic1, topicDescriptionFailFuture)))).andReturn(new MockDescribeTopicsResult(mkMap(mkEntry(topic1, topicDescriptionSuccessfulFuture))));
final KafkaFutureImpl<Config> topicConfigSuccessfulFuture = new KafkaFutureImpl<>();
topicConfigSuccessfulFuture.complete(new Config(repartitionTopicConfig().entrySet().stream().map(entry -> new ConfigEntry(entry.getKey(), entry.getValue())).collect(Collectors.toSet())));
final ConfigResource topicResource = new ConfigResource(Type.TOPIC, topic1);
EasyMock.expect(admin.describeConfigs(Collections.singleton(topicResource))).andReturn(new MockDescribeConfigsResult(mkMap(mkEntry(topicResource, topicConfigSuccessfulFuture))));
EasyMock.replay(admin);
final InternalTopicConfig internalTopicConfig = setupRepartitionTopicConfig(topic1, 1);
final ValidationResult validationResult = topicManager.validate(Collections.singletonMap(topic1, internalTopicConfig));
assertThat(validationResult.missingTopics(), empty());
assertThat(validationResult.misconfigurationsForTopics(), anEmptyMap());
EasyMock.verify(admin);
}
Aggregations