Examples with MetadataResponse org.apache.kafka.common.requests.MetadataResponse used on opensource projects

Example 86 with MetadataResponse

use of org.apache.kafka.common.requests.MetadataResponse in project kafka by apache.

the class SenderTest method testInitProducerIdWithMaxInFlightOne.

/**
 * Verifies that InitProducerId of transactional producer succeeds even if metadata requests
 * are pending with only one bootstrap node available and maxInFlight=1, where multiple
 * polls are necessary to send requests.
 */
@Test
public void testInitProducerIdWithMaxInFlightOne() throws Exception {
    final long producerId = 123456L;
    createMockClientWithMaxFlightOneMetadataPending();
    // Initialize transaction manager. InitProducerId will be queued up until metadata response
    // is processed and FindCoordinator can be sent to `leastLoadedNode`.
    TransactionManager transactionManager = new TransactionManager(new LogContext(), "testInitProducerIdWithPendingMetadataRequest", 60000, 100L, new ApiVersions());
    setupWithTransactionState(transactionManager, false, null, false);
    ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId, (short) 0);
    transactionManager.initializeTransactions();
    sender.runOnce();
    // Process metadata response, prepare FindCoordinator and InitProducerId responses.
    // Verify producerId after the sender is run to process responses.
    MetadataResponse metadataUpdate = RequestTestUtils.metadataUpdateWith(1, Collections.emptyMap());
    client.respond(metadataUpdate);
    prepareFindCoordinatorResponse(Errors.NONE, "testInitProducerIdWithPendingMetadataRequest");
    prepareInitProducerResponse(Errors.NONE, producerIdAndEpoch.producerId, producerIdAndEpoch.epoch);
    waitForProducerId(transactionManager, producerIdAndEpoch);
}

Example 87 with MetadataResponse

use of org.apache.kafka.common.requests.MetadataResponse in project kafka by apache.

the class SenderTest method testIdempotentInitProducerIdWithMaxInFlightOne.

/**
 * Verifies that InitProducerId of idempotent producer succeeds even if metadata requests
 * are pending with only one bootstrap node available and maxInFlight=1, where multiple
 * polls are necessary to send requests.
 */
@Test
public void testIdempotentInitProducerIdWithMaxInFlightOne() throws Exception {
    final long producerId = 123456L;
    createMockClientWithMaxFlightOneMetadataPending();
    // Initialize transaction manager. InitProducerId will be queued up until metadata response
    // is processed.
    TransactionManager transactionManager = createTransactionManager();
    setupWithTransactionState(transactionManager, false, null, false);
    ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId, (short) 0);
    // Process metadata and InitProducerId responses.
    // Verify producerId after the sender is run to process responses.
    MetadataResponse metadataUpdate = RequestTestUtils.metadataUpdateWith(1, Collections.emptyMap());
    client.respond(metadataUpdate);
    sender.runOnce();
    sender.runOnce();
    client.respond(initProducerIdResponse(producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, Errors.NONE));
    waitForProducerId(transactionManager, producerIdAndEpoch);
}

Example 88 with MetadataResponse

use of org.apache.kafka.common.requests.MetadataResponse in project kafka by apache.

the class FetcherTest method testSubscriptionPositionUpdatedWithEpoch.

@Test
public void testSubscriptionPositionUpdatedWithEpoch() {
    // Create some records that include a leader epoch (1)
    MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, TimestampType.CREATE_TIME, 0L, RecordBatch.NO_TIMESTAMP, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, 1);
    builder.appendWithOffset(0L, 0L, "key".getBytes(), "value-1".getBytes());
    builder.appendWithOffset(1L, 0L, "key".getBytes(), "value-2".getBytes());
    builder.appendWithOffset(2L, 0L, "key".getBytes(), "value-3".getBytes());
    MemoryRecords records = builder.build();
    buildFetcher();
    assignFromUser(singleton(tp0));
    // Initialize the epoch=1
    Map<String, Integer> partitionCounts = new HashMap<>();
    partitionCounts.put(tp0.topic(), 4);
    MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWithIds("dummy", 1, Collections.emptyMap(), partitionCounts, tp -> 1, topicIds);
    metadata.updateWithCurrentRequestVersion(metadataResponse, false, 0L);
    // Seek
    subscriptions.seek(tp0, 0);
    // Do a normal fetch
    assertEquals(1, fetcher.sendFetches());
    assertFalse(fetcher.hasCompletedFetches());
    client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
    consumerClient.pollNoWakeup();
    assertTrue(fetcher.hasCompletedFetches());
    Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
    assertTrue(partitionRecords.containsKey(tp0));
    assertEquals(subscriptions.position(tp0).offset, 3L);
    assertOptional(subscriptions.position(tp0).offsetEpoch, value -> assertEquals(value.intValue(), 1));
}

Example 89 with MetadataResponse

use of org.apache.kafka.common.requests.MetadataResponse in project kafka by apache.

the class FetcherTest method testTruncationDetected.

@Test
public void testTruncationDetected() {
    // Create some records that include a leader epoch (1)
    MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, TimestampType.CREATE_TIME, 0L, RecordBatch.NO_TIMESTAMP, RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, // record epoch is earlier than the leader epoch on the client
    1);
    builder.appendWithOffset(0L, 0L, "key".getBytes(), "value-1".getBytes());
    builder.appendWithOffset(1L, 0L, "key".getBytes(), "value-2".getBytes());
    builder.appendWithOffset(2L, 0L, "key".getBytes(), "value-3".getBytes());
    MemoryRecords records = builder.build();
    buildFetcher();
    assignFromUser(singleton(tp0));
    // Initialize the epoch=2
    Map<String, Integer> partitionCounts = new HashMap<>();
    partitionCounts.put(tp0.topic(), 4);
    MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWithIds("dummy", 1, Collections.emptyMap(), partitionCounts, tp -> 2, topicIds);
    metadata.updateWithCurrentRequestVersion(metadataResponse, false, 0L);
    // Offset validation requires OffsetForLeaderEpoch request v3 or higher
    Node node = metadata.fetch().nodes().get(0);
    apiVersions.update(node.idString(), NodeApiVersions.create());
    // Seek
    Metadata.LeaderAndEpoch leaderAndEpoch = new Metadata.LeaderAndEpoch(metadata.currentLeader(tp0).leader, Optional.of(1));
    subscriptions.seekValidated(tp0, new SubscriptionState.FetchPosition(0, Optional.of(1), leaderAndEpoch));
    // Check for truncation, this should cause tp0 to go into validation
    fetcher.validateOffsetsIfNeeded();
    // No fetches sent since we entered validation
    assertEquals(0, fetcher.sendFetches());
    assertFalse(fetcher.hasCompletedFetches());
    assertTrue(subscriptions.awaitingValidation(tp0));
    // Prepare OffsetForEpoch response then check that we update the subscription position correctly.
    client.prepareResponse(prepareOffsetsForLeaderEpochResponse(tp0, Errors.NONE, 1, 10L));
    consumerClient.pollNoWakeup();
    assertFalse(subscriptions.awaitingValidation(tp0));
    // Fetch again, now it works
    assertEquals(1, fetcher.sendFetches());
    assertFalse(fetcher.hasCompletedFetches());
    client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
    consumerClient.pollNoWakeup();
    assertTrue(fetcher.hasCompletedFetches());
    Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
    assertTrue(partitionRecords.containsKey(tp0));
    assertEquals(subscriptions.position(tp0).offset, 3L);
    assertOptional(subscriptions.position(tp0).offsetEpoch, value -> assertEquals(value.intValue(), 1));
}

Example 90 with MetadataResponse

use of org.apache.kafka.common.requests.MetadataResponse in project kafka by apache.

the class KafkaProducerTest method testTransactionalMethodThrowsWhenSenderClosed.

@Test
public void testTransactionalMethodThrowsWhenSenderClosed() {
    Map<String, Object> configs = new HashMap<>();
    configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000");
    configs.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "this-is-a-transactional-id");
    Time time = new MockTime();
    MetadataResponse initialUpdateResponse = RequestTestUtils.metadataUpdateWith(1, emptyMap());
    ProducerMetadata metadata = newMetadata(0, Long.MAX_VALUE);
    metadata.updateWithCurrentRequestVersion(initialUpdateResponse, false, time.milliseconds());
    MockClient client = new MockClient(time, metadata);
    Producer<String, String> producer = kafkaProducer(configs, new StringSerializer(), new StringSerializer(), metadata, client, null, time);
    producer.close();
    assertThrows(IllegalStateException.class, producer::initTransactions);
}