use of org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse in project kafka by apache.
the class ListOffsetsRequest method getErrorResponse.
@Override
public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
short versionId = version();
short errorCode = Errors.forException(e).code();
List<ListOffsetsTopicResponse> responses = new ArrayList<>();
for (ListOffsetsTopic topic : data.topics()) {
ListOffsetsTopicResponse topicResponse = new ListOffsetsTopicResponse().setName(topic.name());
List<ListOffsetsPartitionResponse> partitions = new ArrayList<>();
for (ListOffsetsPartition partition : topic.partitions()) {
ListOffsetsPartitionResponse partitionResponse = new ListOffsetsPartitionResponse().setErrorCode(errorCode).setPartitionIndex(partition.partitionIndex());
if (versionId == 0) {
partitionResponse.setOldStyleOffsets(Collections.emptyList());
} else {
partitionResponse.setOffset(ListOffsetsResponse.UNKNOWN_OFFSET).setTimestamp(ListOffsetsResponse.UNKNOWN_TIMESTAMP);
}
partitions.add(partitionResponse);
}
topicResponse.setPartitions(partitions);
responses.add(topicResponse);
}
ListOffsetsResponseData responseData = new ListOffsetsResponseData().setThrottleTimeMs(throttleTimeMs).setTopics(responses);
return new ListOffsetsResponse(responseData);
}
use of org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse in project kafka by apache.
the class KafkaAdminClient method getListOffsetsCalls.
// visible for benchmark
List<Call> getListOffsetsCalls(MetadataOperationContext<ListOffsetsResultInfo, ListOffsetsOptions> context, Map<TopicPartition, OffsetSpec> topicPartitionOffsets, Map<TopicPartition, KafkaFutureImpl<ListOffsetsResultInfo>> futures) {
MetadataResponse mr = context.response().orElseThrow(() -> new IllegalStateException("No Metadata response"));
Cluster clusterSnapshot = mr.buildCluster();
List<Call> calls = new ArrayList<>();
// grouping topic partitions per leader
Map<Node, Map<String, ListOffsetsTopic>> leaders = new HashMap<>();
for (Map.Entry<TopicPartition, OffsetSpec> entry : topicPartitionOffsets.entrySet()) {
OffsetSpec offsetSpec = entry.getValue();
TopicPartition tp = entry.getKey();
KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
long offsetQuery = getOffsetFromOffsetSpec(offsetSpec);
// avoid sending listOffsets request for topics with errors
if (!mr.errors().containsKey(tp.topic())) {
Node node = clusterSnapshot.leaderFor(tp);
if (node != null) {
Map<String, ListOffsetsTopic> leadersOnNode = leaders.computeIfAbsent(node, k -> new HashMap<>());
ListOffsetsTopic topic = leadersOnNode.computeIfAbsent(tp.topic(), k -> new ListOffsetsTopic().setName(tp.topic()));
topic.partitions().add(new ListOffsetsPartition().setPartitionIndex(tp.partition()).setTimestamp(offsetQuery));
} else {
future.completeExceptionally(Errors.LEADER_NOT_AVAILABLE.exception());
}
} else {
future.completeExceptionally(mr.errors().get(tp.topic()).exception());
}
}
for (final Map.Entry<Node, Map<String, ListOffsetsTopic>> entry : leaders.entrySet()) {
final int brokerId = entry.getKey().id();
calls.add(new Call("listOffsets on broker " + brokerId, context.deadline(), new ConstantNodeIdProvider(brokerId)) {
final List<ListOffsetsTopic> partitionsToQuery = new ArrayList<>(entry.getValue().values());
private boolean supportsMaxTimestamp = partitionsToQuery.stream().flatMap(t -> t.partitions().stream()).anyMatch(p -> p.timestamp() == ListOffsetsRequest.MAX_TIMESTAMP);
@Override
ListOffsetsRequest.Builder createRequest(int timeoutMs) {
return ListOffsetsRequest.Builder.forConsumer(true, context.options().isolationLevel(), supportsMaxTimestamp).setTargetTimes(partitionsToQuery);
}
@Override
void handleResponse(AbstractResponse abstractResponse) {
ListOffsetsResponse response = (ListOffsetsResponse) abstractResponse;
Map<TopicPartition, OffsetSpec> retryTopicPartitionOffsets = new HashMap<>();
for (ListOffsetsTopicResponse topic : response.topics()) {
for (ListOffsetsPartitionResponse partition : topic.partitions()) {
TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
Errors error = Errors.forCode(partition.errorCode());
OffsetSpec offsetRequestSpec = topicPartitionOffsets.get(tp);
if (offsetRequestSpec == null) {
log.warn("Server response mentioned unknown topic partition {}", tp);
} else if (MetadataOperationContext.shouldRefreshMetadata(error)) {
retryTopicPartitionOffsets.put(tp, offsetRequestSpec);
} else if (error == Errors.NONE) {
Optional<Integer> leaderEpoch = (partition.leaderEpoch() == ListOffsetsResponse.UNKNOWN_EPOCH) ? Optional.empty() : Optional.of(partition.leaderEpoch());
future.complete(new ListOffsetsResultInfo(partition.offset(), partition.timestamp(), leaderEpoch));
} else {
future.completeExceptionally(error.exception());
}
}
}
if (retryTopicPartitionOffsets.isEmpty()) {
// The server should send back a response for every topic partition. But do a sanity check anyway.
for (ListOffsetsTopic topic : partitionsToQuery) {
for (ListOffsetsPartition partition : topic.partitions()) {
TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
ApiException error = new ApiException("The response from broker " + brokerId + " did not contain a result for topic partition " + tp);
futures.get(tp).completeExceptionally(error);
}
}
} else {
Set<String> retryTopics = retryTopicPartitionOffsets.keySet().stream().map(TopicPartition::topic).collect(Collectors.toSet());
MetadataOperationContext<ListOffsetsResultInfo, ListOffsetsOptions> retryContext = new MetadataOperationContext<>(retryTopics, context.options(), context.deadline(), futures);
rescheduleMetadataTask(retryContext, () -> getListOffsetsCalls(retryContext, retryTopicPartitionOffsets, futures));
}
}
@Override
void handleFailure(Throwable throwable) {
for (ListOffsetsTopic topic : entry.getValue().values()) {
for (ListOffsetsPartition partition : topic.partitions()) {
TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
future.completeExceptionally(throwable);
}
}
}
@Override
boolean handleUnsupportedVersionException(UnsupportedVersionException exception) {
if (supportsMaxTimestamp) {
supportsMaxTimestamp = false;
// fail any unsupported futures and remove partitions from the downgraded retry
Iterator<ListOffsetsTopic> topicIterator = partitionsToQuery.iterator();
while (topicIterator.hasNext()) {
ListOffsetsTopic topic = topicIterator.next();
Iterator<ListOffsetsPartition> partitionIterator = topic.partitions().iterator();
while (partitionIterator.hasNext()) {
ListOffsetsPartition partition = partitionIterator.next();
if (partition.timestamp() == ListOffsetsRequest.MAX_TIMESTAMP) {
futures.get(new TopicPartition(topic.name(), partition.partitionIndex())).completeExceptionally(new UnsupportedVersionException("Broker " + brokerId + " does not support MAX_TIMESTAMP offset spec"));
partitionIterator.remove();
}
}
if (topic.partitions().isEmpty()) {
topicIterator.remove();
}
}
return !partitionsToQuery.isEmpty();
}
return false;
}
});
}
return calls;
}
use of org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse in project kafka by apache.
the class ListOffsetsRequestTest method testGetErrorResponseV0.
@Test
public void testGetErrorResponseV0() {
List<ListOffsetsTopic> topics = Arrays.asList(new ListOffsetsTopic().setName("topic").setPartitions(Collections.singletonList(new ListOffsetsPartition().setPartitionIndex(0))));
ListOffsetsRequest request = ListOffsetsRequest.Builder.forConsumer(true, IsolationLevel.READ_UNCOMMITTED, false).setTargetTimes(topics).build((short) 0);
ListOffsetsResponse response = (ListOffsetsResponse) request.getErrorResponse(0, Errors.NOT_LEADER_OR_FOLLOWER.exception());
List<ListOffsetsTopicResponse> v = Collections.singletonList(new ListOffsetsTopicResponse().setName("topic").setPartitions(Collections.singletonList(new ListOffsetsPartitionResponse().setErrorCode(Errors.NOT_LEADER_OR_FOLLOWER.code()).setOldStyleOffsets(Collections.emptyList()).setPartitionIndex(0))));
ListOffsetsResponseData data = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(v);
ListOffsetsResponse expectedResponse = new ListOffsetsResponse(data);
assertEquals(expectedResponse.data().topics(), response.data().topics());
assertEquals(expectedResponse.throttleTimeMs(), response.throttleTimeMs());
}
use of org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse in project kafka by apache.
the class ListOffsetsRequestTest method testGetErrorResponse.
@Test
public void testGetErrorResponse() {
for (short version = 1; version <= ApiKeys.LIST_OFFSETS.latestVersion(); version++) {
List<ListOffsetsTopic> topics = Arrays.asList(new ListOffsetsTopic().setName("topic").setPartitions(Collections.singletonList(new ListOffsetsPartition().setPartitionIndex(0))));
ListOffsetsRequest request = ListOffsetsRequest.Builder.forConsumer(true, IsolationLevel.READ_COMMITTED, false).setTargetTimes(topics).build(version);
ListOffsetsResponse response = (ListOffsetsResponse) request.getErrorResponse(0, Errors.NOT_LEADER_OR_FOLLOWER.exception());
List<ListOffsetsTopicResponse> v = Collections.singletonList(new ListOffsetsTopicResponse().setName("topic").setPartitions(Collections.singletonList(new ListOffsetsPartitionResponse().setErrorCode(Errors.NOT_LEADER_OR_FOLLOWER.code()).setLeaderEpoch(ListOffsetsResponse.UNKNOWN_EPOCH).setOffset(ListOffsetsResponse.UNKNOWN_OFFSET).setPartitionIndex(0).setTimestamp(ListOffsetsResponse.UNKNOWN_TIMESTAMP))));
ListOffsetsResponseData data = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(v);
ListOffsetsResponse expectedResponse = new ListOffsetsResponse(data);
assertEquals(expectedResponse.data().topics(), response.data().topics());
assertEquals(expectedResponse.throttleTimeMs(), response.throttleTimeMs());
}
}
use of org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse in project kafka by apache.
the class KafkaConsumerTest method listOffsetsResponse.
private ListOffsetsResponse listOffsetsResponse(Map<TopicPartition, Long> partitionOffsets, Map<TopicPartition, Errors> partitionErrors) {
Map<String, ListOffsetsTopicResponse> responses = new HashMap<>();
for (Map.Entry<TopicPartition, Long> partitionOffset : partitionOffsets.entrySet()) {
TopicPartition tp = partitionOffset.getKey();
ListOffsetsTopicResponse topic = responses.computeIfAbsent(tp.topic(), k -> new ListOffsetsTopicResponse().setName(tp.topic()));
topic.partitions().add(new ListOffsetsPartitionResponse().setPartitionIndex(tp.partition()).setErrorCode(Errors.NONE.code()).setTimestamp(ListOffsetsResponse.UNKNOWN_TIMESTAMP).setOffset(partitionOffset.getValue()));
}
for (Map.Entry<TopicPartition, Errors> partitionError : partitionErrors.entrySet()) {
TopicPartition tp = partitionError.getKey();
ListOffsetsTopicResponse topic = responses.computeIfAbsent(tp.topic(), k -> new ListOffsetsTopicResponse().setName(tp.topic()));
topic.partitions().add(new ListOffsetsPartitionResponse().setPartitionIndex(tp.partition()).setErrorCode(partitionError.getValue().code()).setTimestamp(ListOffsetsResponse.UNKNOWN_TIMESTAMP).setOffset(ListOffsetsResponse.UNKNOWN_OFFSET));
}
ListOffsetsResponseData data = new ListOffsetsResponseData().setTopics(new ArrayList<>(responses.values()));
return new ListOffsetsResponse(data);
}
Aggregations