use of org.apache.kafka.common.utils.Time in project kafka by apache.
the class MetadataTest method testMetadataMerge.
@Test
public void testMetadataMerge() {
Time time = new MockTime();
Map<String, Uuid> topicIds = new HashMap<>();
final AtomicReference<Set<String>> retainTopics = new AtomicReference<>(new HashSet<>());
metadata = new Metadata(refreshBackoffMs, metadataExpireMs, new LogContext(), new ClusterResourceListeners()) {
@Override
protected boolean retainTopic(String topic, boolean isInternal, long nowMs) {
return retainTopics.get().contains(topic);
}
};
// Initialize a metadata instance with two topic variants "old" and "keep". Both will be retained.
String oldClusterId = "oldClusterId";
int oldNodes = 2;
Map<String, Errors> oldTopicErrors = new HashMap<>();
oldTopicErrors.put("oldInvalidTopic", Errors.INVALID_TOPIC_EXCEPTION);
oldTopicErrors.put("keepInvalidTopic", Errors.INVALID_TOPIC_EXCEPTION);
oldTopicErrors.put("oldUnauthorizedTopic", Errors.TOPIC_AUTHORIZATION_FAILED);
oldTopicErrors.put("keepUnauthorizedTopic", Errors.TOPIC_AUTHORIZATION_FAILED);
Map<String, Integer> oldTopicPartitionCounts = new HashMap<>();
oldTopicPartitionCounts.put("oldValidTopic", 2);
oldTopicPartitionCounts.put("keepValidTopic", 3);
retainTopics.set(Utils.mkSet("oldInvalidTopic", "keepInvalidTopic", "oldUnauthorizedTopic", "keepUnauthorizedTopic", "oldValidTopic", "keepValidTopic"));
topicIds.put("oldValidTopic", Uuid.randomUuid());
topicIds.put("keepValidTopic", Uuid.randomUuid());
MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWithIds(oldClusterId, oldNodes, oldTopicErrors, oldTopicPartitionCounts, _tp -> 100, topicIds);
metadata.updateWithCurrentRequestVersion(metadataResponse, true, time.milliseconds());
Map<String, Uuid> metadataTopicIds1 = metadata.topicIds();
retainTopics.get().forEach(topic -> assertEquals(metadataTopicIds1.get(topic), topicIds.get(topic)));
// Update the metadata to add a new topic variant, "new", which will be retained with "keep". Note this
// means that all of the "old" topics should be dropped.
Cluster cluster = metadata.fetch();
assertEquals(cluster.clusterResource().clusterId(), oldClusterId);
assertEquals(cluster.nodes().size(), oldNodes);
assertEquals(cluster.invalidTopics(), new HashSet<>(Arrays.asList("oldInvalidTopic", "keepInvalidTopic")));
assertEquals(cluster.unauthorizedTopics(), new HashSet<>(Arrays.asList("oldUnauthorizedTopic", "keepUnauthorizedTopic")));
assertEquals(cluster.topics(), new HashSet<>(Arrays.asList("oldValidTopic", "keepValidTopic")));
assertEquals(cluster.partitionsForTopic("oldValidTopic").size(), 2);
assertEquals(cluster.partitionsForTopic("keepValidTopic").size(), 3);
assertEquals(new HashSet<>(cluster.topicIds()), new HashSet<>(topicIds.values()));
String newClusterId = "newClusterId";
int newNodes = oldNodes + 1;
Map<String, Errors> newTopicErrors = new HashMap<>();
newTopicErrors.put("newInvalidTopic", Errors.INVALID_TOPIC_EXCEPTION);
newTopicErrors.put("newUnauthorizedTopic", Errors.TOPIC_AUTHORIZATION_FAILED);
Map<String, Integer> newTopicPartitionCounts = new HashMap<>();
newTopicPartitionCounts.put("keepValidTopic", 2);
newTopicPartitionCounts.put("newValidTopic", 4);
retainTopics.set(Utils.mkSet("keepInvalidTopic", "newInvalidTopic", "keepUnauthorizedTopic", "newUnauthorizedTopic", "keepValidTopic", "newValidTopic"));
topicIds.put("newValidTopic", Uuid.randomUuid());
metadataResponse = RequestTestUtils.metadataUpdateWithIds(newClusterId, newNodes, newTopicErrors, newTopicPartitionCounts, _tp -> 200, topicIds);
metadata.updateWithCurrentRequestVersion(metadataResponse, true, time.milliseconds());
topicIds.remove("oldValidTopic");
Map<String, Uuid> metadataTopicIds2 = metadata.topicIds();
retainTopics.get().forEach(topic -> assertEquals(metadataTopicIds2.get(topic), topicIds.get(topic)));
assertNull(metadataTopicIds2.get("oldValidTopic"));
cluster = metadata.fetch();
assertEquals(cluster.clusterResource().clusterId(), newClusterId);
assertEquals(cluster.nodes().size(), newNodes);
assertEquals(cluster.invalidTopics(), new HashSet<>(Arrays.asList("keepInvalidTopic", "newInvalidTopic")));
assertEquals(cluster.unauthorizedTopics(), new HashSet<>(Arrays.asList("keepUnauthorizedTopic", "newUnauthorizedTopic")));
assertEquals(cluster.topics(), new HashSet<>(Arrays.asList("keepValidTopic", "newValidTopic")));
assertEquals(cluster.partitionsForTopic("keepValidTopic").size(), 2);
assertEquals(cluster.partitionsForTopic("newValidTopic").size(), 4);
assertEquals(new HashSet<>(cluster.topicIds()), new HashSet<>(topicIds.values()));
// Perform another metadata update, but this time all topic metadata should be cleared.
retainTopics.set(Collections.emptySet());
metadataResponse = RequestTestUtils.metadataUpdateWithIds(newClusterId, newNodes, newTopicErrors, newTopicPartitionCounts, _tp -> 300, topicIds);
metadata.updateWithCurrentRequestVersion(metadataResponse, true, time.milliseconds());
Map<String, Uuid> metadataTopicIds3 = metadata.topicIds();
topicIds.forEach((topicName, topicId) -> assertNull(metadataTopicIds3.get(topicName)));
cluster = metadata.fetch();
assertEquals(cluster.clusterResource().clusterId(), newClusterId);
assertEquals(cluster.nodes().size(), newNodes);
assertEquals(cluster.invalidTopics(), Collections.emptySet());
assertEquals(cluster.unauthorizedTopics(), Collections.emptySet());
assertEquals(cluster.topics(), Collections.emptySet());
assertTrue(cluster.topicIds().isEmpty());
}
use of org.apache.kafka.common.utils.Time in project kafka by apache.
the class MetadataTest method testInvalidTopicError.
@Test
public void testInvalidTopicError() {
Time time = new MockTime();
String invalidTopic = "topic dfsa";
MetadataResponse invalidTopicResponse = RequestTestUtils.metadataUpdateWith("clusterId", 1, Collections.singletonMap(invalidTopic, Errors.INVALID_TOPIC_EXCEPTION), Collections.emptyMap());
metadata.updateWithCurrentRequestVersion(invalidTopicResponse, false, time.milliseconds());
InvalidTopicException e = assertThrows(InvalidTopicException.class, () -> metadata.maybeThrowAnyException());
assertEquals(Collections.singleton(invalidTopic), e.invalidTopics());
// We clear the exception once it has been raised to the user
metadata.maybeThrowAnyException();
// Reset the invalid topic error
metadata.updateWithCurrentRequestVersion(invalidTopicResponse, false, time.milliseconds());
// If we get a good update, the error should clear even if we haven't had a chance to raise it to the user
metadata.updateWithCurrentRequestVersion(emptyMetadataResponse(), false, time.milliseconds());
metadata.maybeThrowAnyException();
}
use of org.apache.kafka.common.utils.Time in project kafka by apache.
the class MetadataTest method testTopicAuthorizationError.
@Test
public void testTopicAuthorizationError() {
Time time = new MockTime();
String invalidTopic = "foo";
MetadataResponse unauthorizedTopicResponse = RequestTestUtils.metadataUpdateWith("clusterId", 1, Collections.singletonMap(invalidTopic, Errors.TOPIC_AUTHORIZATION_FAILED), Collections.emptyMap());
metadata.updateWithCurrentRequestVersion(unauthorizedTopicResponse, false, time.milliseconds());
TopicAuthorizationException e = assertThrows(TopicAuthorizationException.class, () -> metadata.maybeThrowAnyException());
assertEquals(Collections.singleton(invalidTopic), e.unauthorizedTopics());
// We clear the exception once it has been raised to the user
metadata.maybeThrowAnyException();
// Reset the unauthorized topic error
metadata.updateWithCurrentRequestVersion(unauthorizedTopicResponse, false, time.milliseconds());
// If we get a good update, the error should clear even if we haven't had a chance to raise it to the user
metadata.updateWithCurrentRequestVersion(emptyMetadataResponse(), false, time.milliseconds());
metadata.maybeThrowAnyException();
}
use of org.apache.kafka.common.utils.Time in project kafka by apache.
the class MetadataTest method testMetadataMergeOnIdDowngrade.
@Test
public void testMetadataMergeOnIdDowngrade() {
Time time = new MockTime();
Map<String, Uuid> topicIds = new HashMap<>();
final AtomicReference<Set<String>> retainTopics = new AtomicReference<>(new HashSet<>());
metadata = new Metadata(refreshBackoffMs, metadataExpireMs, new LogContext(), new ClusterResourceListeners()) {
@Override
protected boolean retainTopic(String topic, boolean isInternal, long nowMs) {
return retainTopics.get().contains(topic);
}
};
// Initialize a metadata instance with two topics. Both will be retained.
String clusterId = "clusterId";
int nodes = 2;
Map<String, Integer> topicPartitionCounts = new HashMap<>();
topicPartitionCounts.put("validTopic1", 2);
topicPartitionCounts.put("validTopic2", 3);
retainTopics.set(Utils.mkSet("validTopic1", "validTopic2"));
topicIds.put("validTopic1", Uuid.randomUuid());
topicIds.put("validTopic2", Uuid.randomUuid());
MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWithIds(clusterId, nodes, Collections.emptyMap(), topicPartitionCounts, _tp -> 100, topicIds);
metadata.updateWithCurrentRequestVersion(metadataResponse, true, time.milliseconds());
Map<String, Uuid> metadataTopicIds1 = metadata.topicIds();
retainTopics.get().forEach(topic -> assertEquals(metadataTopicIds1.get(topic), topicIds.get(topic)));
// Try removing the topic ID from keepValidTopic (simulating receiving a request from a controller with an older IBP)
topicIds.remove("validTopic1");
metadataResponse = RequestTestUtils.metadataUpdateWithIds(clusterId, nodes, Collections.emptyMap(), topicPartitionCounts, _tp -> 200, topicIds);
metadata.updateWithCurrentRequestVersion(metadataResponse, true, time.milliseconds());
Map<String, Uuid> metadataTopicIds2 = metadata.topicIds();
retainTopics.get().forEach(topic -> assertEquals(metadataTopicIds2.get(topic), topicIds.get(topic)));
Cluster cluster = metadata.fetch();
// We still have the topic, but it just doesn't have an ID.
assertEquals(Utils.mkSet("validTopic1", "validTopic2"), cluster.topics());
assertEquals(2, cluster.partitionsForTopic("validTopic1").size());
assertEquals(new HashSet<>(topicIds.values()), new HashSet<>(cluster.topicIds()));
assertEquals(Uuid.ZERO_UUID, cluster.topicId("validTopic1"));
}
use of org.apache.kafka.common.utils.Time in project kafka by apache.
the class KafkaAdminClientTest method testOffsetCommitNumRetries.
@Test
public void testOffsetCommitNumRetries() throws Exception {
final Cluster cluster = mockCluster(3, 0);
final Time time = new MockTime();
try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(time, cluster, AdminClientConfig.RETRIES_CONFIG, "0")) {
env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
final TopicPartition tp1 = new TopicPartition("foo", 0);
env.kafkaClient().prepareResponse(prepareFindCoordinatorResponse(Errors.NONE, env.cluster().controller()));
env.kafkaClient().prepareResponse(prepareOffsetCommitResponse(tp1, Errors.NOT_COORDINATOR));
env.kafkaClient().prepareResponse(prepareFindCoordinatorResponse(Errors.NONE, env.cluster().controller()));
Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
offsets.put(tp1, new OffsetAndMetadata(123L));
final AlterConsumerGroupOffsetsResult result = env.adminClient().alterConsumerGroupOffsets(GROUP_ID, offsets);
TestUtils.assertFutureError(result.all(), TimeoutException.class);
}
}
Aggregations