Examples with ListOffsetsResponse org.apache.kafka.common.requests.ListOffsetsResponse used on opensource projects

Example 16 with ListOffsetsResponse

use of org.apache.kafka.common.requests.ListOffsetsResponse in project kafka by apache.

the class KafkaAdminClientTest method testListOffsets.

@Test
public void testListOffsets() throws Exception {
    // Happy path
    Node node0 = new Node(0, "localhost", 8120);
    List<PartitionInfo> pInfos = new ArrayList<>();
    pInfos.add(new PartitionInfo("foo", 0, node0, new Node[] { node0 }, new Node[] { node0 }));
    pInfos.add(new PartitionInfo("bar", 0, node0, new Node[] { node0 }, new Node[] { node0 }));
    pInfos.add(new PartitionInfo("baz", 0, node0, new Node[] { node0 }, new Node[] { node0 }));
    pInfos.add(new PartitionInfo("qux", 0, node0, new Node[] { node0 }, new Node[] { node0 }));
    final Cluster cluster = new Cluster("mockClusterId", Arrays.asList(node0), pInfos, Collections.<String>emptySet(), Collections.<String>emptySet(), node0);
    final TopicPartition tp0 = new TopicPartition("foo", 0);
    final TopicPartition tp1 = new TopicPartition("bar", 0);
    final TopicPartition tp2 = new TopicPartition("baz", 0);
    final TopicPartition tp3 = new TopicPartition("qux", 0);
    try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) {
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
        env.kafkaClient().prepareResponse(prepareMetadataResponse(cluster, Errors.NONE));
        ListOffsetsTopicResponse t0 = ListOffsetsResponse.singletonListOffsetsTopicResponse(tp0, Errors.NONE, -1L, 123L, 321);
        ListOffsetsTopicResponse t1 = ListOffsetsResponse.singletonListOffsetsTopicResponse(tp1, Errors.NONE, -1L, 234L, 432);
        ListOffsetsTopicResponse t2 = ListOffsetsResponse.singletonListOffsetsTopicResponse(tp2, Errors.NONE, 123456789L, 345L, 543);
        ListOffsetsTopicResponse t3 = ListOffsetsResponse.singletonListOffsetsTopicResponse(tp3, Errors.NONE, 234567890L, 456L, 654);
        ListOffsetsResponseData responseData = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(Arrays.asList(t0, t1, t2, t3));
        env.kafkaClient().prepareResponse(new ListOffsetsResponse(responseData));
        Map<TopicPartition, OffsetSpec> partitions = new HashMap<>();
        partitions.put(tp0, OffsetSpec.latest());
        partitions.put(tp1, OffsetSpec.earliest());
        partitions.put(tp2, OffsetSpec.forTimestamp(System.currentTimeMillis()));
        partitions.put(tp3, OffsetSpec.maxTimestamp());
        ListOffsetsResult result = env.adminClient().listOffsets(partitions);
        Map<TopicPartition, ListOffsetsResultInfo> offsets = result.all().get();
        assertFalse(offsets.isEmpty());
        assertEquals(123L, offsets.get(tp0).offset());
        assertEquals(321, offsets.get(tp0).leaderEpoch().get().intValue());
        assertEquals(-1L, offsets.get(tp0).timestamp());
        assertEquals(234L, offsets.get(tp1).offset());
        assertEquals(432, offsets.get(tp1).leaderEpoch().get().intValue());
        assertEquals(-1L, offsets.get(tp1).timestamp());
        assertEquals(345L, offsets.get(tp2).offset());
        assertEquals(543, offsets.get(tp2).leaderEpoch().get().intValue());
        assertEquals(123456789L, offsets.get(tp2).timestamp());
        assertEquals(456L, offsets.get(tp3).offset());
        assertEquals(654, offsets.get(tp3).leaderEpoch().get().intValue());
        assertEquals(234567890L, offsets.get(tp3).timestamp());
        assertEquals(offsets.get(tp0), result.partitionResult(tp0).get());
        assertEquals(offsets.get(tp1), result.partitionResult(tp1).get());
        assertEquals(offsets.get(tp2), result.partitionResult(tp2).get());
        assertEquals(offsets.get(tp3), result.partitionResult(tp3).get());
        try {
            result.partitionResult(new TopicPartition("unknown", 0)).get();
            fail("should have thrown IllegalArgumentException");
        } catch (IllegalArgumentException expected) {
        }
    }
}

Example 17 with ListOffsetsResponse

use of org.apache.kafka.common.requests.ListOffsetsResponse in project kafka by apache.

the class KafkaAdminClientTest method testListOffsetsNonMaxTimestampDowngradedImmediately.

@Test
public void testListOffsetsNonMaxTimestampDowngradedImmediately() throws Exception {
    Node node = new Node(0, "localhost", 8120);
    List<Node> nodes = Collections.singletonList(node);
    List<PartitionInfo> pInfos = new ArrayList<>();
    pInfos.add(new PartitionInfo("foo", 0, node, new Node[] { node }, new Node[] { node }));
    final Cluster cluster = new Cluster("mockClusterId", nodes, pInfos, Collections.emptySet(), Collections.emptySet(), node);
    final TopicPartition tp0 = new TopicPartition("foo", 0);
    try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster, AdminClientConfig.RETRIES_CONFIG, "2")) {
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create(ApiKeys.LIST_OFFSETS.id, (short) 0, (short) 6));
        env.kafkaClient().prepareResponse(prepareMetadataResponse(cluster, Errors.NONE));
        ListOffsetsTopicResponse t0 = ListOffsetsResponse.singletonListOffsetsTopicResponse(tp0, Errors.NONE, -1L, 123L, 321);
        ListOffsetsResponseData responseData = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(Arrays.asList(t0));
        // listoffsets response from broker 0
        env.kafkaClient().prepareResponse(request -> request instanceof ListOffsetsRequest, new ListOffsetsResponse(responseData));
        ListOffsetsResult result = env.adminClient().listOffsets(Collections.singletonMap(tp0, OffsetSpec.latest()));
        ListOffsetsResultInfo tp0Offset = result.partitionResult(tp0).get();
        assertEquals(123L, tp0Offset.offset());
        assertEquals(321, tp0Offset.leaderEpoch().get().intValue());
        assertEquals(-1L, tp0Offset.timestamp());
    }
}

Example 18 with ListOffsetsResponse

use of org.apache.kafka.common.requests.ListOffsetsResponse in project kafka by apache.

the class TopicAdminTest method listOffsetsResult.

/**
 * Create a ListOffsetResponse that exposes the supplied error and includes offsets for the supplied partitions.
 * @param error               the error; may be null if an unknown error should be used
 * @param offsetsByPartitions offset for each partition, where offset is null signals the error should be used
 * @return the response
 */
private ListOffsetsResponse listOffsetsResult(ApiError error, Map<TopicPartition, Long> offsetsByPartitions) {
    if (error == null)
        error = new ApiError(Errors.UNKNOWN_TOPIC_OR_PARTITION, "unknown topic");
    List<ListOffsetsTopicResponse> tpResponses = new ArrayList<>();
    for (TopicPartition partition : offsetsByPartitions.keySet()) {
        Long offset = offsetsByPartitions.get(partition);
        ListOffsetsTopicResponse topicResponse;
        if (offset == null) {
            topicResponse = ListOffsetsResponse.singletonListOffsetsTopicResponse(partition, error.error(), -1L, 0, 321);
        } else {
            topicResponse = ListOffsetsResponse.singletonListOffsetsTopicResponse(partition, Errors.NONE, -1L, offset, 321);
        }
        tpResponses.add(topicResponse);
    }
    ListOffsetsResponseData responseData = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(tpResponses);
    return new ListOffsetsResponse(responseData);
}

Example 19 with ListOffsetsResponse

use of org.apache.kafka.common.requests.ListOffsetsResponse in project kafka by apache.

the class FetcherTest method testGetOffsetByTimeWithPartitionsRetryCouldTriggerMetadataUpdate.

@Test
public void testGetOffsetByTimeWithPartitionsRetryCouldTriggerMetadataUpdate() {
    List<Errors> retriableErrors = Arrays.asList(Errors.NOT_LEADER_OR_FOLLOWER, Errors.REPLICA_NOT_AVAILABLE, Errors.KAFKA_STORAGE_ERROR, Errors.OFFSET_NOT_AVAILABLE, Errors.LEADER_NOT_AVAILABLE, Errors.FENCED_LEADER_EPOCH, Errors.UNKNOWN_LEADER_EPOCH);
    final int newLeaderEpoch = 3;
    MetadataResponse updatedMetadata = RequestTestUtils.metadataUpdateWithIds("dummy", 3, singletonMap(topicName, Errors.NONE), singletonMap(topicName, 4), tp -> newLeaderEpoch, topicIds);
    Node originalLeader = initialUpdateResponse.buildCluster().leaderFor(tp1);
    Node newLeader = updatedMetadata.buildCluster().leaderFor(tp1);
    assertNotEquals(originalLeader, newLeader);
    for (Errors retriableError : retriableErrors) {
        buildFetcher();
        subscriptions.assignFromUser(mkSet(tp0, tp1));
        client.updateMetadata(initialUpdateResponse);
        final long fetchTimestamp = 10L;
        ListOffsetsPartitionResponse tp0NoError = new ListOffsetsPartitionResponse().setPartitionIndex(tp0.partition()).setErrorCode(Errors.NONE.code()).setTimestamp(fetchTimestamp).setOffset(4L);
        List<ListOffsetsTopicResponse> topics = Collections.singletonList(new ListOffsetsTopicResponse().setName(tp0.topic()).setPartitions(Arrays.asList(tp0NoError, new ListOffsetsPartitionResponse().setPartitionIndex(tp1.partition()).setErrorCode(retriableError.code()).setTimestamp(ListOffsetsRequest.LATEST_TIMESTAMP).setOffset(-1L))));
        ListOffsetsResponseData data = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(topics);
        client.prepareResponseFrom(body -> {
            boolean isListOffsetRequest = body instanceof ListOffsetsRequest;
            if (isListOffsetRequest) {
                ListOffsetsRequest request = (ListOffsetsRequest) body;
                List<ListOffsetsTopic> expectedTopics = Collections.singletonList(new ListOffsetsTopic().setName(tp0.topic()).setPartitions(Arrays.asList(new ListOffsetsPartition().setPartitionIndex(tp1.partition()).setTimestamp(fetchTimestamp).setCurrentLeaderEpoch(ListOffsetsResponse.UNKNOWN_EPOCH), new ListOffsetsPartition().setPartitionIndex(tp0.partition()).setTimestamp(fetchTimestamp).setCurrentLeaderEpoch(ListOffsetsResponse.UNKNOWN_EPOCH))));
                return request.topics().equals(expectedTopics);
            } else {
                return false;
            }
        }, new ListOffsetsResponse(data), originalLeader);
        client.prepareMetadataUpdate(updatedMetadata);
        // If the metadata wasn't updated before retrying, the fetcher would consult the original leader and hit a NOT_LEADER exception.
        // We will count the answered future response in the end to verify if this is the case.
        List<ListOffsetsTopicResponse> topicsWithFatalError = Collections.singletonList(new ListOffsetsTopicResponse().setName(tp0.topic()).setPartitions(Arrays.asList(tp0NoError, new ListOffsetsPartitionResponse().setPartitionIndex(tp1.partition()).setErrorCode(Errors.NOT_LEADER_OR_FOLLOWER.code()).setTimestamp(ListOffsetsRequest.LATEST_TIMESTAMP).setOffset(-1L))));
        ListOffsetsResponseData dataWithFatalError = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(topicsWithFatalError);
        client.prepareResponseFrom(new ListOffsetsResponse(dataWithFatalError), originalLeader);
        // The request to new leader must only contain one partition tp1 with error.
        client.prepareResponseFrom(body -> {
            boolean isListOffsetRequest = body instanceof ListOffsetsRequest;
            if (isListOffsetRequest) {
                ListOffsetsRequest request = (ListOffsetsRequest) body;
                ListOffsetsTopic requestTopic = request.topics().get(0);
                ListOffsetsPartition expectedPartition = new ListOffsetsPartition().setPartitionIndex(tp1.partition()).setTimestamp(fetchTimestamp).setCurrentLeaderEpoch(newLeaderEpoch);
                return expectedPartition.equals(requestTopic.partitions().get(0));
            } else {
                return false;
            }
        }, listOffsetResponse(tp1, Errors.NONE, fetchTimestamp, 5L), newLeader);
        Map<TopicPartition, OffsetAndTimestamp> offsetAndTimestampMap = fetcher.offsetsForTimes(Utils.mkMap(Utils.mkEntry(tp0, fetchTimestamp), Utils.mkEntry(tp1, fetchTimestamp)), time.timer(Integer.MAX_VALUE));
        assertEquals(Utils.mkMap(Utils.mkEntry(tp0, new OffsetAndTimestamp(4L, fetchTimestamp)), Utils.mkEntry(tp1, new OffsetAndTimestamp(5L, fetchTimestamp))), offsetAndTimestampMap);
        // The NOT_LEADER exception future should not be cleared as we already refreshed the metadata before
        // first retry, thus never hitting.
        assertEquals(1, client.numAwaitingResponses());
        fetcher.close();
    }
}