Examples with PartitionInfo org.apache.kafka.common.PartitionInfo used on opensource projects

Example 76 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class KafkaAdminClientTest method testListOffsetsMaxTimestampUnsupportedMultipleOffsetSpec.

@Test
public void testListOffsetsMaxTimestampUnsupportedMultipleOffsetSpec() throws Exception {
    Node node = new Node(0, "localhost", 8120);
    List<Node> nodes = Collections.singletonList(node);
    List<PartitionInfo> pInfos = new ArrayList<>();
    pInfos.add(new PartitionInfo("foo", 0, node, new Node[] { node }, new Node[] { node }));
    pInfos.add(new PartitionInfo("foo", 1, node, new Node[] { node }, new Node[] { node }));
    final Cluster cluster = new Cluster("mockClusterId", nodes, pInfos, Collections.emptySet(), Collections.emptySet(), node);
    final TopicPartition tp0 = new TopicPartition("foo", 0);
    final TopicPartition tp1 = new TopicPartition("foo", 1);
    try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster, AdminClientConfig.RETRIES_CONFIG, "2")) {
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create(ApiKeys.LIST_OFFSETS.id, (short) 0, (short) 6));
        env.kafkaClient().prepareResponse(prepareMetadataResponse(cluster, Errors.NONE));
        // listoffsets response from broker 0
        env.kafkaClient().prepareUnsupportedVersionResponse(request -> request instanceof ListOffsetsRequest);
        ListOffsetsTopicResponse topicResponse = ListOffsetsResponse.singletonListOffsetsTopicResponse(tp1, Errors.NONE, -1L, 345L, 543);
        ListOffsetsResponseData responseData = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(Arrays.asList(topicResponse));
        env.kafkaClient().prepareResponseFrom(// ensure that no max timestamp requests are retried
        request -> request instanceof ListOffsetsRequest && ((ListOffsetsRequest) request).topics().stream().flatMap(t -> t.partitions().stream()).noneMatch(p -> p.timestamp() == ListOffsetsRequest.MAX_TIMESTAMP), new ListOffsetsResponse(responseData), node);
        ListOffsetsResult result = env.adminClient().listOffsets(new HashMap<TopicPartition, OffsetSpec>() {

            {
                put(tp0, OffsetSpec.maxTimestamp());
                put(tp1, OffsetSpec.latest());
            }
        });
        TestUtils.assertFutureThrows(result.partitionResult(tp0), UnsupportedVersionException.class);
        ListOffsetsResultInfo tp1Offset = result.partitionResult(tp1).get();
        assertEquals(345L, tp1Offset.offset());
        assertEquals(543, tp1Offset.leaderEpoch().get().intValue());
        assertEquals(-1L, tp1Offset.timestamp());
    }
}

Example 77 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class KafkaAdminClientTest method testListOffsetsMetadataRetriableErrors.

@Test
public void testListOffsetsMetadataRetriableErrors() throws Exception {
    Node node0 = new Node(0, "localhost", 8120);
    Node node1 = new Node(1, "localhost", 8121);
    List<Node> nodes = Arrays.asList(node0, node1);
    List<PartitionInfo> pInfos = new ArrayList<>();
    pInfos.add(new PartitionInfo("foo", 0, node0, new Node[] { node0 }, new Node[] { node0 }));
    pInfos.add(new PartitionInfo("foo", 1, node1, new Node[] { node1 }, new Node[] { node1 }));
    final Cluster cluster = new Cluster("mockClusterId", nodes, pInfos, Collections.<String>emptySet(), Collections.<String>emptySet(), node0);
    final TopicPartition tp0 = new TopicPartition("foo", 0);
    final TopicPartition tp1 = new TopicPartition("foo", 1);
    try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) {
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
        env.kafkaClient().prepareResponse(prepareMetadataResponse(cluster, Errors.LEADER_NOT_AVAILABLE));
        env.kafkaClient().prepareResponse(prepareMetadataResponse(cluster, Errors.UNKNOWN_TOPIC_OR_PARTITION));
        env.kafkaClient().prepareResponse(prepareMetadataResponse(cluster, Errors.NONE));
        // listoffsets response from broker 0
        ListOffsetsTopicResponse t0 = ListOffsetsResponse.singletonListOffsetsTopicResponse(tp0, Errors.NONE, -1L, 345L, 543);
        ListOffsetsResponseData responseData = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(Arrays.asList(t0));
        env.kafkaClient().prepareResponseFrom(new ListOffsetsResponse(responseData), node0);
        // listoffsets response from broker 1
        ListOffsetsTopicResponse t1 = ListOffsetsResponse.singletonListOffsetsTopicResponse(tp1, Errors.NONE, -1L, 789L, 987);
        responseData = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(Arrays.asList(t1));
        env.kafkaClient().prepareResponseFrom(new ListOffsetsResponse(responseData), node1);
        Map<TopicPartition, OffsetSpec> partitions = new HashMap<>();
        partitions.put(tp0, OffsetSpec.latest());
        partitions.put(tp1, OffsetSpec.latest());
        ListOffsetsResult result = env.adminClient().listOffsets(partitions);
        Map<TopicPartition, ListOffsetsResultInfo> offsets = result.all().get();
        assertFalse(offsets.isEmpty());
        assertEquals(345L, offsets.get(tp0).offset());
        assertEquals(543, offsets.get(tp0).leaderEpoch().get().intValue());
        assertEquals(-1L, offsets.get(tp0).timestamp());
        assertEquals(789L, offsets.get(tp1).offset());
        assertEquals(987, offsets.get(tp1).leaderEpoch().get().intValue());
        assertEquals(-1L, offsets.get(tp1).timestamp());
    }
}

Example 78 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class KafkaAdminClientTest method testListOffsetsRetriableErrorOnMetadata.

@Test
public void testListOffsetsRetriableErrorOnMetadata() throws Exception {
    Node node = new Node(0, "localhost", 8120);
    List<Node> nodes = Collections.singletonList(node);
    final Cluster cluster = new Cluster("mockClusterId", nodes, Collections.singleton(new PartitionInfo("foo", 0, node, new Node[] { node }, new Node[] { node })), Collections.emptySet(), Collections.emptySet(), node);
    final TopicPartition tp0 = new TopicPartition("foo", 0);
    try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) {
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
        env.kafkaClient().prepareResponse(prepareMetadataResponse(cluster, Errors.UNKNOWN_TOPIC_OR_PARTITION, Errors.NONE));
        // metadata refresh because of UNKNOWN_TOPIC_OR_PARTITION
        env.kafkaClient().prepareResponse(prepareMetadataResponse(cluster, Errors.NONE));
        // listoffsets response from broker 0
        ListOffsetsResponseData responseData = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(Collections.singletonList(ListOffsetsResponse.singletonListOffsetsTopicResponse(tp0, Errors.NONE, -1L, 123L, 321)));
        env.kafkaClient().prepareResponseFrom(new ListOffsetsResponse(responseData), node);
        ListOffsetsResult result = env.adminClient().listOffsets(Collections.singletonMap(tp0, OffsetSpec.latest()));
        Map<TopicPartition, ListOffsetsResultInfo> offsets = result.all().get(3, TimeUnit.SECONDS);
        assertEquals(1, offsets.size());
        assertEquals(123L, offsets.get(tp0).offset());
        assertEquals(321, offsets.get(tp0).leaderEpoch().get().intValue());
        assertEquals(-1L, offsets.get(tp0).timestamp());
    }
}

Example 79 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class KafkaAdminClientTest method testSuccessfulRetryAfterRequestTimeout.

@Test
public void testSuccessfulRetryAfterRequestTimeout() throws Exception {
    HashMap<Integer, Node> nodes = new HashMap<>();
    MockTime time = new MockTime();
    Node node0 = new Node(0, "localhost", 8121);
    nodes.put(0, node0);
    Cluster cluster = new Cluster("mockClusterId", nodes.values(), Arrays.asList(new PartitionInfo("foo", 0, node0, new Node[] { node0 }, new Node[] { node0 })), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), nodes.get(0));
    final int requestTimeoutMs = 1000;
    final int retryBackoffMs = 100;
    final int apiTimeoutMs = 3000;
    try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(time, cluster, AdminClientConfig.RETRY_BACKOFF_MS_CONFIG, String.valueOf(retryBackoffMs), AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, String.valueOf(requestTimeoutMs))) {
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
        final ListTopicsResult result = env.adminClient().listTopics(new ListTopicsOptions().timeoutMs(apiTimeoutMs));
        // Wait until the first attempt has been sent, then advance the time
        TestUtils.waitForCondition(() -> env.kafkaClient().hasInFlightRequests(), "Timed out waiting for Metadata request to be sent");
        time.sleep(requestTimeoutMs + 1);
        // Wait for the request to be timed out before backing off
        TestUtils.waitForCondition(() -> !env.kafkaClient().hasInFlightRequests(), "Timed out waiting for inFlightRequests to be timed out");
        time.sleep(retryBackoffMs);
        // Since api timeout bound is not hit, AdminClient should retry
        TestUtils.waitForCondition(() -> env.kafkaClient().hasInFlightRequests(), "Failed to retry Metadata request");
        env.kafkaClient().respond(prepareMetadataResponse(cluster, Errors.NONE));
        assertEquals(1, result.listings().get().size());
        assertEquals("foo", result.listings().get().iterator().next().name());
    }
}

Example 80 with PartitionInfo

use of org.apache.kafka.common.PartitionInfo in project kafka by apache.

the class KafkaAdminClientTest method testRequestTimeoutExceedingDefaultApiTimeout.

@Test
public void testRequestTimeoutExceedingDefaultApiTimeout() throws Exception {
    HashMap<Integer, Node> nodes = new HashMap<>();
    MockTime time = new MockTime();
    Node node0 = new Node(0, "localhost", 8121);
    nodes.put(0, node0);
    Cluster cluster = new Cluster("mockClusterId", nodes.values(), Arrays.asList(new PartitionInfo("foo", 0, node0, new Node[] { node0 }, new Node[] { node0 })), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), nodes.get(0));
    // This test assumes the default api timeout value of 60000. When the request timeout
    // is set to something larger, we should adjust the api timeout accordingly for compatibility.
    final int retryBackoffMs = 100;
    final int requestTimeoutMs = 120000;
    try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(time, cluster, AdminClientConfig.RETRY_BACKOFF_MS_CONFIG, String.valueOf(retryBackoffMs), AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, String.valueOf(requestTimeoutMs))) {
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
        ListTopicsOptions options = new ListTopicsOptions();
        final ListTopicsResult result = env.adminClient().listTopics(options);
        // Wait until the first attempt has been sent, then advance the time by the default api timeout
        TestUtils.waitForCondition(() -> env.kafkaClient().hasInFlightRequests(), "Timed out waiting for Metadata request to be sent");
        time.sleep(60001);
        // The in-flight request should not be cancelled
        assertTrue(env.kafkaClient().hasInFlightRequests());
        // Now sleep the remaining time for the request timeout to expire
        time.sleep(60000);
        TestUtils.assertFutureThrows(result.future, TimeoutException.class);
    }
}