Examples with MetadataResponseData org.apache.kafka.common.message.MetadataResponseData used on opensource projects

Example 6 with MetadataResponseData

use of org.apache.kafka.common.message.MetadataResponseData in project kafka by apache.

the class GetListOffsetsCallsBenchmark method setup.

@Setup(Level.Trial)
public void setup() {
    MetadataResponseData data = new MetadataResponseData();
    List<MetadataResponseTopic> mrTopicList = new ArrayList<>();
    Set<String> topics = new HashSet<>();
    for (int topicIndex = 0; topicIndex < topicCount; topicIndex++) {
        Uuid topicId = Uuid.randomUuid();
        String topicName = "topic-" + topicIndex;
        MetadataResponseTopic mrTopic = new MetadataResponseTopic().setTopicId(topicId).setName(topicName).setErrorCode((short) 0).setIsInternal(false);
        List<MetadataResponsePartition> mrPartitionList = new ArrayList<>();
        for (int partition = 0; partition < partitionCount; partition++) {
            TopicPartition tp = new TopicPartition(topicName, partition);
            topics.add(tp.topic());
            futures.put(tp, new KafkaFutureImpl<>());
            topicPartitionOffsets.put(tp, OffsetSpec.latest());
            MetadataResponsePartition mrPartition = new MetadataResponsePartition().setLeaderId(partition % numNodes).setPartitionIndex(partition).setIsrNodes(Arrays.asList(0, 1, 2)).setReplicaNodes(Arrays.asList(0, 1, 2)).setOfflineReplicas(Collections.emptyList()).setErrorCode((short) 0);
            mrPartitionList.add(mrPartition);
        }
        mrTopic.setPartitions(mrPartitionList);
        mrTopicList.add(mrTopic);
    }
    data.setTopics(new MetadataResponseData.MetadataResponseTopicCollection(mrTopicList.listIterator()));
    long deadline = 0L;
    short version = 0;
    context = new MetadataOperationContext<>(topics, new ListOffsetsOptions(), deadline, futures);
    context.setResponse(Optional.of(new MetadataResponse(data, version)));
    AdminClientUnitTestEnv adminEnv = new AdminClientUnitTestEnv(mockCluster());
    admin = (KafkaAdminClient) adminEnv.adminClient();
}

Example 7 with MetadataResponseData

use of org.apache.kafka.common.message.MetadataResponseData in project kafka by apache.

the class MetadataTest method testIgnoreLeaderEpochInOlderMetadataResponse.

/**
 * Prior to Kafka version 2.4 (which coincides with Metadata version 9), the broker does not propagate leader epoch
 * information accurately while a reassignment is in progress, so we cannot rely on it. This is explained in more
 * detail in MetadataResponse's constructor.
 */
@Test
public void testIgnoreLeaderEpochInOlderMetadataResponse() {
    TopicPartition tp = new TopicPartition("topic", 0);
    MetadataResponsePartition partitionMetadata = new MetadataResponsePartition().setPartitionIndex(tp.partition()).setLeaderId(5).setLeaderEpoch(10).setReplicaNodes(Arrays.asList(1, 2, 3)).setIsrNodes(Arrays.asList(1, 2, 3)).setOfflineReplicas(Collections.emptyList()).setErrorCode(Errors.NONE.code());
    MetadataResponseTopic topicMetadata = new MetadataResponseTopic().setName(tp.topic()).setErrorCode(Errors.NONE.code()).setPartitions(Collections.singletonList(partitionMetadata)).setIsInternal(false);
    MetadataResponseTopicCollection topics = new MetadataResponseTopicCollection();
    topics.add(topicMetadata);
    MetadataResponseData data = new MetadataResponseData().setClusterId("clusterId").setControllerId(0).setTopics(topics).setBrokers(new MetadataResponseBrokerCollection());
    for (short version = ApiKeys.METADATA.oldestVersion(); version < 9; version++) {
        ByteBuffer buffer = MessageUtil.toByteBuffer(data, version);
        MetadataResponse response = MetadataResponse.parse(buffer, version);
        assertFalse(response.hasReliableLeaderEpochs());
        metadata.updateWithCurrentRequestVersion(response, false, 100);
        assertTrue(metadata.partitionMetadataIfCurrent(tp).isPresent());
        MetadataResponse.PartitionMetadata responseMetadata = this.metadata.partitionMetadataIfCurrent(tp).get();
        assertEquals(Optional.empty(), responseMetadata.leaderEpoch);
    }
    for (short version = 9; version <= ApiKeys.METADATA.latestVersion(); version++) {
        ByteBuffer buffer = MessageUtil.toByteBuffer(data, version);
        MetadataResponse response = MetadataResponse.parse(buffer, version);
        assertTrue(response.hasReliableLeaderEpochs());
        metadata.updateWithCurrentRequestVersion(response, false, 100);
        assertTrue(metadata.partitionMetadataIfCurrent(tp).isPresent());
        MetadataResponse.PartitionMetadata responseMetadata = metadata.partitionMetadataIfCurrent(tp).get();
        assertEquals(Optional.of(10), responseMetadata.leaderEpoch);
    }
}

Example 8 with MetadataResponseData

use of org.apache.kafka.common.message.MetadataResponseData in project kafka by apache.

the class MetadataTest method testStaleMetadata.

@Test
public void testStaleMetadata() {
    TopicPartition tp = new TopicPartition("topic", 0);
    MetadataResponsePartition partitionMetadata = new MetadataResponsePartition().setPartitionIndex(tp.partition()).setLeaderId(1).setLeaderEpoch(10).setReplicaNodes(Arrays.asList(1, 2, 3)).setIsrNodes(Arrays.asList(1, 2, 3)).setOfflineReplicas(Collections.emptyList()).setErrorCode(Errors.NONE.code());
    MetadataResponseTopic topicMetadata = new MetadataResponseTopic().setName(tp.topic()).setErrorCode(Errors.NONE.code()).setPartitions(Collections.singletonList(partitionMetadata)).setIsInternal(false);
    MetadataResponseTopicCollection topics = new MetadataResponseTopicCollection();
    topics.add(topicMetadata);
    MetadataResponseData data = new MetadataResponseData().setClusterId("clusterId").setControllerId(0).setTopics(topics).setBrokers(new MetadataResponseBrokerCollection());
    metadata.updateWithCurrentRequestVersion(new MetadataResponse(data, ApiKeys.METADATA.latestVersion()), false, 100);
    // Older epoch with changed ISR should be ignored
    partitionMetadata.setPartitionIndex(tp.partition()).setLeaderId(1).setLeaderEpoch(9).setReplicaNodes(Arrays.asList(1, 2, 3)).setIsrNodes(Arrays.asList(1, 2)).setOfflineReplicas(Collections.emptyList()).setErrorCode(Errors.NONE.code());
    metadata.updateWithCurrentRequestVersion(new MetadataResponse(data, ApiKeys.METADATA.latestVersion()), false, 101);
    assertEquals(Optional.of(10), metadata.lastSeenLeaderEpoch(tp));
    assertTrue(metadata.partitionMetadataIfCurrent(tp).isPresent());
    MetadataResponse.PartitionMetadata responseMetadata = this.metadata.partitionMetadataIfCurrent(tp).get();
    assertEquals(Arrays.asList(1, 2, 3), responseMetadata.inSyncReplicaIds);
    assertEquals(Optional.of(10), responseMetadata.leaderEpoch);
}

Example 9 with MetadataResponseData

use of org.apache.kafka.common.message.MetadataResponseData in project kafka by apache.

the class MetadataTest method testLeaderMetadataInconsistentWithBrokerMetadata.

@Test
public void testLeaderMetadataInconsistentWithBrokerMetadata() {
    // Tests a reordering scenario which can lead to inconsistent leader state.
    // A partition initially has one broker offline. That broker comes online and
    // is elected leader. The client sees these two events in the opposite order.
    TopicPartition tp = new TopicPartition("topic", 0);
    Node node0 = new Node(0, "localhost", 9092);
    Node node1 = new Node(1, "localhost", 9093);
    Node node2 = new Node(2, "localhost", 9094);
    // The first metadata received by broker (epoch=10)
    MetadataResponsePartition firstPartitionMetadata = new MetadataResponsePartition().setPartitionIndex(tp.partition()).setErrorCode(Errors.NONE.code()).setLeaderEpoch(10).setLeaderId(0).setReplicaNodes(Arrays.asList(0, 1, 2)).setIsrNodes(Arrays.asList(0, 1, 2)).setOfflineReplicas(Collections.emptyList());
    // The second metadata received has stale metadata (epoch=8)
    MetadataResponsePartition secondPartitionMetadata = new MetadataResponsePartition().setPartitionIndex(tp.partition()).setErrorCode(Errors.NONE.code()).setLeaderEpoch(8).setLeaderId(1).setReplicaNodes(Arrays.asList(0, 1, 2)).setIsrNodes(Arrays.asList(1, 2)).setOfflineReplicas(Collections.singletonList(0));
    metadata.updateWithCurrentRequestVersion(new MetadataResponse(new MetadataResponseData().setTopics(buildTopicCollection(tp.topic(), firstPartitionMetadata)).setBrokers(buildBrokerCollection(Arrays.asList(node0, node1, node2))), ApiKeys.METADATA.latestVersion()), false, 10L);
    metadata.updateWithCurrentRequestVersion(new MetadataResponse(new MetadataResponseData().setTopics(buildTopicCollection(tp.topic(), secondPartitionMetadata)).setBrokers(buildBrokerCollection(Arrays.asList(node1, node2))), ApiKeys.METADATA.latestVersion()), false, 20L);
    assertNull(metadata.fetch().leaderFor(tp));
    assertEquals(Optional.of(10), metadata.lastSeenLeaderEpoch(tp));
    assertFalse(metadata.currentLeader(tp).leader.isPresent());
}

Example 10 with MetadataResponseData

use of org.apache.kafka.common.message.MetadataResponseData in project kafka by apache.

the class KafkaAdminClientTest method testListTransactions.

@Test
public void testListTransactions() throws Exception {
    try (AdminClientUnitTestEnv env = mockClientEnv()) {
        MetadataResponseData.MetadataResponseBrokerCollection brokers = new MetadataResponseData.MetadataResponseBrokerCollection();
        env.cluster().nodes().forEach(node -> {
            brokers.add(new MetadataResponseData.MetadataResponseBroker().setHost(node.host()).setNodeId(node.id()).setPort(node.port()).setRack(node.rack()));
        });
        env.kafkaClient().prepareResponse(request -> request instanceof MetadataRequest, new MetadataResponse(new MetadataResponseData().setBrokers(brokers), MetadataResponseData.HIGHEST_SUPPORTED_VERSION));
        List<TransactionListing> expected = Arrays.asList(new TransactionListing("foo", 12345L, TransactionState.ONGOING), new TransactionListing("bar", 98765L, TransactionState.PREPARE_ABORT), new TransactionListing("baz", 13579L, TransactionState.COMPLETE_COMMIT));
        assertEquals(Utils.mkSet(0, 1, 2), env.cluster().nodes().stream().map(Node::id).collect(Collectors.toSet()));
        env.cluster().nodes().forEach(node -> {
            ListTransactionsResponseData response = new ListTransactionsResponseData().setErrorCode(Errors.NONE.code());
            TransactionListing listing = expected.get(node.id());
            response.transactionStates().add(new ListTransactionsResponseData.TransactionState().setTransactionalId(listing.transactionalId()).setProducerId(listing.producerId()).setTransactionState(listing.state().toString()));
            env.kafkaClient().prepareResponseFrom(request -> request instanceof ListTransactionsRequest, new ListTransactionsResponse(response), node);
        });
        ListTransactionsResult result = env.adminClient().listTransactions();
        assertEquals(new HashSet<>(expected), new HashSet<>(result.all().get()));
    }
}