Examples with ListOffsetResponse org.apache.kafka.common.requests.ListOffsetResponse used on opensource projects

Example 1 with ListOffsetResponse

use of org.apache.kafka.common.requests.ListOffsetResponse in project kafka by apache.

the class Fetcher method handleListOffsetResponse.

/**
     * Callback for the response of the list offset call above.
     * @param timestampsToSearch The mapping from partitions to target timestamps
     * @param listOffsetResponse The response from the server.
     * @param future The future to be completed by the response.
     */
@SuppressWarnings("deprecation")
private void handleListOffsetResponse(Map<TopicPartition, Long> timestampsToSearch, ListOffsetResponse listOffsetResponse, RequestFuture<Map<TopicPartition, OffsetData>> future) {
    Map<TopicPartition, OffsetData> timestampOffsetMap = new HashMap<>();
    for (Map.Entry<TopicPartition, Long> entry : timestampsToSearch.entrySet()) {
        TopicPartition topicPartition = entry.getKey();
        ListOffsetResponse.PartitionData partitionData = listOffsetResponse.responseData().get(topicPartition);
        Errors error = partitionData.error;
        if (error == Errors.NONE) {
            if (partitionData.offsets != null) {
                // Handle v0 response
                long offset;
                if (partitionData.offsets.size() > 1) {
                    future.raise(new IllegalStateException("Unexpected partitionData response of length " + partitionData.offsets.size()));
                    return;
                } else if (partitionData.offsets.isEmpty()) {
                    offset = ListOffsetResponse.UNKNOWN_OFFSET;
                } else {
                    offset = partitionData.offsets.get(0);
                }
                log.debug("Handling v0 ListOffsetResponse response for {}. Fetched offset {}", topicPartition, offset);
                if (offset != ListOffsetResponse.UNKNOWN_OFFSET) {
                    OffsetData offsetData = new OffsetData(offset, null);
                    timestampOffsetMap.put(topicPartition, offsetData);
                }
            } else {
                // Handle v1 and later response
                log.debug("Handling ListOffsetResponse response for {}. Fetched offset {}, timestamp {}", topicPartition, partitionData.offset, partitionData.timestamp);
                if (partitionData.offset != ListOffsetResponse.UNKNOWN_OFFSET) {
                    OffsetData offsetData = new OffsetData(partitionData.offset, partitionData.timestamp);
                    timestampOffsetMap.put(topicPartition, offsetData);
                }
            }
        } else if (error == Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT) {
            // The message format on the broker side is before 0.10.0, we simply put null in the response.
            log.debug("Cannot search by timestamp for partition {} because the message format version " + "is before 0.10.0", topicPartition);
            timestampOffsetMap.put(topicPartition, null);
        } else if (error == Errors.NOT_LEADER_FOR_PARTITION) {
            log.debug("Attempt to fetch offsets for partition {} failed due to obsolete leadership information, retrying.", topicPartition);
            future.raise(error);
        } else if (error == Errors.UNKNOWN_TOPIC_OR_PARTITION) {
            log.warn("Received unknown topic or partition error in ListOffset request for partition {}. The topic/partition " + "may not exist or the user may not have Describe access to it", topicPartition);
            future.raise(error);
        } else {
            log.warn("Attempt to fetch offsets for partition {} failed due to: {}", topicPartition, error.message());
            future.raise(new StaleMetadataException());
        }
    }
    if (!future.isDone())
        future.complete(timestampOffsetMap);
}

Example 2 with ListOffsetResponse

use of org.apache.kafka.common.requests.ListOffsetResponse in project apache-kafka-on-k8s by banzaicloud.

the class Fetcher method handleListOffsetResponse.

/**
 * Callback for the response of the list offset call above.
 * @param timestampsToSearch The mapping from partitions to target timestamps
 * @param listOffsetResponse The response from the server.
 * @param future The future to be completed when the response returns. Note that any partition-level errors will
 *               generally fail the entire future result. The one exception is UNSUPPORTED_FOR_MESSAGE_FORMAT,
 *               which indicates that the broker does not support the v1 message format. Partitions with this
 *               particular error are simply left out of the future map. Note that the corresponding timestamp
 *               value of each partition may be null only for v0. In v1 and later the ListOffset API would not
 *               return a null timestamp (-1 is returned instead when necessary).
 */
@SuppressWarnings("deprecation")
private void handleListOffsetResponse(Map<TopicPartition, Long> timestampsToSearch, ListOffsetResponse listOffsetResponse, RequestFuture<ListOffsetResult> future) {
    Map<TopicPartition, OffsetData> fetchedOffsets = new HashMap<>();
    Set<TopicPartition> partitionsToRetry = new HashSet<>();
    Set<String> unauthorizedTopics = new HashSet<>();
    for (Map.Entry<TopicPartition, Long> entry : timestampsToSearch.entrySet()) {
        TopicPartition topicPartition = entry.getKey();
        ListOffsetResponse.PartitionData partitionData = listOffsetResponse.responseData().get(topicPartition);
        Errors error = partitionData.error;
        if (error == Errors.NONE) {
            if (partitionData.offsets != null) {
                // Handle v0 response
                long offset;
                if (partitionData.offsets.size() > 1) {
                    future.raise(new IllegalStateException("Unexpected partitionData response of length " + partitionData.offsets.size()));
                    return;
                } else if (partitionData.offsets.isEmpty()) {
                    offset = ListOffsetResponse.UNKNOWN_OFFSET;
                } else {
                    offset = partitionData.offsets.get(0);
                }
                log.debug("Handling v0 ListOffsetResponse response for {}. Fetched offset {}", topicPartition, offset);
                if (offset != ListOffsetResponse.UNKNOWN_OFFSET) {
                    OffsetData offsetData = new OffsetData(offset, null);
                    fetchedOffsets.put(topicPartition, offsetData);
                }
            } else {
                // Handle v1 and later response
                log.debug("Handling ListOffsetResponse response for {}. Fetched offset {}, timestamp {}", topicPartition, partitionData.offset, partitionData.timestamp);
                if (partitionData.offset != ListOffsetResponse.UNKNOWN_OFFSET) {
                    OffsetData offsetData = new OffsetData(partitionData.offset, partitionData.timestamp);
                    fetchedOffsets.put(topicPartition, offsetData);
                }
            }
        } else if (error == Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT) {
            // The message format on the broker side is before 0.10.0, which means it does not
            // support timestamps. We treat this case the same as if we weren't able to find an
            // offset corresponding to the requested timestamp and leave it out of the result.
            log.debug("Cannot search by timestamp for partition {} because the message format version " + "is before 0.10.0", topicPartition);
        } else if (error == Errors.NOT_LEADER_FOR_PARTITION) {
            log.debug("Attempt to fetch offsets for partition {} failed due to obsolete leadership information, retrying.", topicPartition);
            partitionsToRetry.add(topicPartition);
        } else if (error == Errors.UNKNOWN_TOPIC_OR_PARTITION) {
            log.warn("Received unknown topic or partition error in ListOffset request for partition {}", topicPartition);
            partitionsToRetry.add(topicPartition);
        } else if (error == Errors.TOPIC_AUTHORIZATION_FAILED) {
            unauthorizedTopics.add(topicPartition.topic());
        } else {
            log.warn("Attempt to fetch offsets for partition {} failed due to: {}", topicPartition, error.message());
            partitionsToRetry.add(topicPartition);
        }
    }
    if (!unauthorizedTopics.isEmpty())
        future.raise(new TopicAuthorizationException(unauthorizedTopics));
    else
        future.complete(new ListOffsetResult(fetchedOffsets, partitionsToRetry));
}

Example 3 with ListOffsetResponse

use of org.apache.kafka.common.requests.ListOffsetResponse in project apache-kafka-on-k8s by banzaicloud.

the class FetcherTest method testGetOffsetsForTimesWithUnknownOffset.

private void testGetOffsetsForTimesWithUnknownOffset() {
    client.reset();
    // Ensure metadata has both partition.
    Cluster cluster = TestUtils.clusterWith(1, topicName, 1);
    metadata.update(cluster, Collections.<String>emptySet(), time.milliseconds());
    Map<TopicPartition, ListOffsetResponse.PartitionData> partitionData = new HashMap<>();
    partitionData.put(tp0, new ListOffsetResponse.PartitionData(Errors.NONE, ListOffsetResponse.UNKNOWN_TIMESTAMP, ListOffsetResponse.UNKNOWN_OFFSET));
    client.prepareResponseFrom(new ListOffsetResponse(0, partitionData), cluster.leaderFor(tp0));
    Map<TopicPartition, Long> timestampToSearch = new HashMap<>();
    timestampToSearch.put(tp0, 0L);
    Map<TopicPartition, OffsetAndTimestamp> offsetAndTimestampMap = fetcher.offsetsByTimes(timestampToSearch, Long.MAX_VALUE);
    assertTrue(offsetAndTimestampMap.containsKey(tp0));
    assertNull(offsetAndTimestampMap.get(tp0));
}

Example 4 with ListOffsetResponse

use of org.apache.kafka.common.requests.ListOffsetResponse in project apache-kafka-on-k8s by banzaicloud.

the class FetcherTest method testBatchedListOffsetsMetadataErrors.

@Test(expected = TimeoutException.class)
public void testBatchedListOffsetsMetadataErrors() {
    Map<TopicPartition, ListOffsetResponse.PartitionData> partitionData = new HashMap<>();
    partitionData.put(tp0, new ListOffsetResponse.PartitionData(Errors.NOT_LEADER_FOR_PARTITION, ListOffsetResponse.UNKNOWN_TIMESTAMP, ListOffsetResponse.UNKNOWN_OFFSET));
    partitionData.put(tp1, new ListOffsetResponse.PartitionData(Errors.UNKNOWN_TOPIC_OR_PARTITION, ListOffsetResponse.UNKNOWN_TIMESTAMP, ListOffsetResponse.UNKNOWN_OFFSET));
    client.prepareResponse(new ListOffsetResponse(0, partitionData));
    Map<TopicPartition, Long> offsetsToSearch = new HashMap<>();
    offsetsToSearch.put(tp0, ListOffsetRequest.EARLIEST_TIMESTAMP);
    offsetsToSearch.put(tp1, ListOffsetRequest.EARLIEST_TIMESTAMP);
    fetcher.offsetsByTimes(offsetsToSearch, 0);
}