Examples with ListOffsetsRequest org.apache.kafka.common.requests.ListOffsetsRequest used on opensource projects

Example 6 with ListOffsetsRequest

use of org.apache.kafka.common.requests.ListOffsetsRequest in project kafka by apache.

the class FetcherTest method testGetOffsetByTimeWithPartitionsRetryCouldTriggerMetadataUpdate.

@Test
public void testGetOffsetByTimeWithPartitionsRetryCouldTriggerMetadataUpdate() {
    List<Errors> retriableErrors = Arrays.asList(Errors.NOT_LEADER_OR_FOLLOWER, Errors.REPLICA_NOT_AVAILABLE, Errors.KAFKA_STORAGE_ERROR, Errors.OFFSET_NOT_AVAILABLE, Errors.LEADER_NOT_AVAILABLE, Errors.FENCED_LEADER_EPOCH, Errors.UNKNOWN_LEADER_EPOCH);
    final int newLeaderEpoch = 3;
    MetadataResponse updatedMetadata = RequestTestUtils.metadataUpdateWithIds("dummy", 3, singletonMap(topicName, Errors.NONE), singletonMap(topicName, 4), tp -> newLeaderEpoch, topicIds);
    Node originalLeader = initialUpdateResponse.buildCluster().leaderFor(tp1);
    Node newLeader = updatedMetadata.buildCluster().leaderFor(tp1);
    assertNotEquals(originalLeader, newLeader);
    for (Errors retriableError : retriableErrors) {
        buildFetcher();
        subscriptions.assignFromUser(mkSet(tp0, tp1));
        client.updateMetadata(initialUpdateResponse);
        final long fetchTimestamp = 10L;
        ListOffsetsPartitionResponse tp0NoError = new ListOffsetsPartitionResponse().setPartitionIndex(tp0.partition()).setErrorCode(Errors.NONE.code()).setTimestamp(fetchTimestamp).setOffset(4L);
        List<ListOffsetsTopicResponse> topics = Collections.singletonList(new ListOffsetsTopicResponse().setName(tp0.topic()).setPartitions(Arrays.asList(tp0NoError, new ListOffsetsPartitionResponse().setPartitionIndex(tp1.partition()).setErrorCode(retriableError.code()).setTimestamp(ListOffsetsRequest.LATEST_TIMESTAMP).setOffset(-1L))));
        ListOffsetsResponseData data = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(topics);
        client.prepareResponseFrom(body -> {
            boolean isListOffsetRequest = body instanceof ListOffsetsRequest;
            if (isListOffsetRequest) {
                ListOffsetsRequest request = (ListOffsetsRequest) body;
                List<ListOffsetsTopic> expectedTopics = Collections.singletonList(new ListOffsetsTopic().setName(tp0.topic()).setPartitions(Arrays.asList(new ListOffsetsPartition().setPartitionIndex(tp1.partition()).setTimestamp(fetchTimestamp).setCurrentLeaderEpoch(ListOffsetsResponse.UNKNOWN_EPOCH), new ListOffsetsPartition().setPartitionIndex(tp0.partition()).setTimestamp(fetchTimestamp).setCurrentLeaderEpoch(ListOffsetsResponse.UNKNOWN_EPOCH))));
                return request.topics().equals(expectedTopics);
            } else {
                return false;
            }
        }, new ListOffsetsResponse(data), originalLeader);
        client.prepareMetadataUpdate(updatedMetadata);
        // If the metadata wasn't updated before retrying, the fetcher would consult the original leader and hit a NOT_LEADER exception.
        // We will count the answered future response in the end to verify if this is the case.
        List<ListOffsetsTopicResponse> topicsWithFatalError = Collections.singletonList(new ListOffsetsTopicResponse().setName(tp0.topic()).setPartitions(Arrays.asList(tp0NoError, new ListOffsetsPartitionResponse().setPartitionIndex(tp1.partition()).setErrorCode(Errors.NOT_LEADER_OR_FOLLOWER.code()).setTimestamp(ListOffsetsRequest.LATEST_TIMESTAMP).setOffset(-1L))));
        ListOffsetsResponseData dataWithFatalError = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(topicsWithFatalError);
        client.prepareResponseFrom(new ListOffsetsResponse(dataWithFatalError), originalLeader);
        // The request to new leader must only contain one partition tp1 with error.
        client.prepareResponseFrom(body -> {
            boolean isListOffsetRequest = body instanceof ListOffsetsRequest;
            if (isListOffsetRequest) {
                ListOffsetsRequest request = (ListOffsetsRequest) body;
                ListOffsetsTopic requestTopic = request.topics().get(0);
                ListOffsetsPartition expectedPartition = new ListOffsetsPartition().setPartitionIndex(tp1.partition()).setTimestamp(fetchTimestamp).setCurrentLeaderEpoch(newLeaderEpoch);
                return expectedPartition.equals(requestTopic.partitions().get(0));
            } else {
                return false;
            }
        }, listOffsetResponse(tp1, Errors.NONE, fetchTimestamp, 5L), newLeader);
        Map<TopicPartition, OffsetAndTimestamp> offsetAndTimestampMap = fetcher.offsetsForTimes(Utils.mkMap(Utils.mkEntry(tp0, fetchTimestamp), Utils.mkEntry(tp1, fetchTimestamp)), time.timer(Integer.MAX_VALUE));
        assertEquals(Utils.mkMap(Utils.mkEntry(tp0, new OffsetAndTimestamp(4L, fetchTimestamp)), Utils.mkEntry(tp1, new OffsetAndTimestamp(5L, fetchTimestamp))), offsetAndTimestampMap);
        // The NOT_LEADER exception future should not be cleared as we already refreshed the metadata before
        // first retry, thus never hitting.
        assertEquals(1, client.numAwaitingResponses());
        fetcher.close();
    }
}

Example 7 with ListOffsetsRequest

use of org.apache.kafka.common.requests.ListOffsetsRequest in project kafka by apache.

the class KafkaConsumerTest method testFetchProgressWithMissingPartitionPosition.

@Test
public void testFetchProgressWithMissingPartitionPosition() {
    // Verifies that we can make progress on one partition while we are awaiting
    // a reset on another partition.
    ConsumerMetadata metadata = createMetadata(subscription);
    MockClient client = new MockClient(time, metadata);
    initMetadata(client, Collections.singletonMap(topic, 2));
    Node node = metadata.fetch().nodes().get(0);
    KafkaConsumer<String, String> consumer = newConsumerNoAutoCommit(time, client, subscription, metadata);
    consumer.assign(Arrays.asList(tp0, tp1));
    consumer.seekToEnd(singleton(tp0));
    consumer.seekToBeginning(singleton(tp1));
    client.prepareResponseFrom(FindCoordinatorResponse.prepareResponse(Errors.NONE, groupId, node), node);
    client.prepareResponse(body -> {
        ListOffsetsRequest request = (ListOffsetsRequest) body;
        List<ListOffsetsPartition> partitions = request.topics().stream().flatMap(t -> {
            if (t.name().equals(topic))
                return Stream.of(t.partitions());
            else
                return Stream.empty();
        }).flatMap(List::stream).collect(Collectors.toList());
        ListOffsetsPartition expectedTp0 = new ListOffsetsPartition().setPartitionIndex(tp0.partition()).setTimestamp(ListOffsetsRequest.LATEST_TIMESTAMP);
        ListOffsetsPartition expectedTp1 = new ListOffsetsPartition().setPartitionIndex(tp1.partition()).setTimestamp(ListOffsetsRequest.EARLIEST_TIMESTAMP);
        return partitions.contains(expectedTp0) && partitions.contains(expectedTp1);
    }, listOffsetsResponse(Collections.singletonMap(tp0, 50L), Collections.singletonMap(tp1, Errors.NOT_LEADER_OR_FOLLOWER)));
    client.prepareResponse(body -> {
        FetchRequest request = (FetchRequest) body;
        Map<TopicIdPartition, FetchRequest.PartitionData> fetchData = request.fetchData(topicNames);
        TopicIdPartition tidp0 = new TopicIdPartition(topicIds.get(tp0.topic()), tp0);
        return fetchData.keySet().equals(singleton(tidp0)) && fetchData.get(tidp0).fetchOffset == 50L;
    }, fetchResponse(tp0, 50L, 5));
    ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1));
    assertEquals(5, records.count());
    assertEquals(singleton(tp0), records.partitions());
}

Example 8 with ListOffsetsRequest

use of org.apache.kafka.common.requests.ListOffsetsRequest in project kafka by apache.

the class KafkaConsumerTest method consumerForCheckingTimeoutException.

private KafkaConsumer<String, String> consumerForCheckingTimeoutException() {
    ConsumerMetadata metadata = createMetadata(subscription);
    MockClient client = new MockClient(time, metadata);
    initMetadata(client, singletonMap(topic, 1));
    ConsumerPartitionAssignor assignor = new RangeAssignor();
    final KafkaConsumer<String, String> consumer = newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId);
    for (int i = 0; i < 10; i++) {
        client.prepareResponse(request -> {
            time.sleep(defaultApiTimeoutMs / 10);
            return request instanceof ListOffsetsRequest;
        }, listOffsetsResponse(Collections.emptyMap(), Collections.singletonMap(tp0, Errors.UNKNOWN_TOPIC_OR_PARTITION)));
    }
    return consumer;
}

Example 9 with ListOffsetsRequest

use of org.apache.kafka.common.requests.ListOffsetsRequest in project kafka by apache.

the class FetcherTest method listOffsetMatchesExpectedReset.

private boolean listOffsetMatchesExpectedReset(TopicPartition tp, OffsetResetStrategy strategy, AbstractRequest request) {
    assertTrue(request instanceof ListOffsetsRequest);
    ListOffsetsRequest req = (ListOffsetsRequest) request;
    assertEquals(singleton(tp.topic()), req.data().topics().stream().map(ListOffsetsTopic::name).collect(Collectors.toSet()));
    ListOffsetsTopic listTopic = req.data().topics().get(0);
    assertEquals(singleton(tp.partition()), listTopic.partitions().stream().map(ListOffsetsPartition::partitionIndex).collect(Collectors.toSet()));
    ListOffsetsPartition listPartition = listTopic.partitions().get(0);
    if (strategy == OffsetResetStrategy.EARLIEST) {
        assertEquals(ListOffsetsRequest.EARLIEST_TIMESTAMP, listPartition.timestamp());
    } else if (strategy == OffsetResetStrategy.LATEST) {
        assertEquals(ListOffsetsRequest.LATEST_TIMESTAMP, listPartition.timestamp());
    }
    return true;
}

Example 10 with ListOffsetsRequest

use of org.apache.kafka.common.requests.ListOffsetsRequest in project kafka by apache.

the class FetcherTest method testListOffsetsSendsIsolationLevel.

private void testListOffsetsSendsIsolationLevel(IsolationLevel isolationLevel) {
    buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, isolationLevel);
    assignFromUser(singleton(tp0));
    subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST);
    client.prepareResponse(body -> {
        ListOffsetsRequest request = (ListOffsetsRequest) body;
        return request.isolationLevel() == isolationLevel;
    }, listOffsetResponse(Errors.NONE, 1L, 5L));
    fetcher.resetOffsetsIfNeeded();
    consumerClient.pollNoWakeup();
    assertFalse(subscriptions.isOffsetResetNeeded(tp0));
    assertTrue(subscriptions.isFetchable(tp0));
    assertEquals(5, subscriptions.position(tp0).offset);
}