Examples with ProducerMetadata org.apache.kafka.clients.producer.internals.ProducerMetadata used on opensource projects

Example 31 with ProducerMetadata

use of org.apache.kafka.clients.producer.internals.ProducerMetadata in project kafka by apache.

the class KafkaProducerTest method testMetadataWithPartitionOutOfRange.

@ParameterizedTest
@ValueSource(booleans = { true, false })
public void testMetadataWithPartitionOutOfRange(boolean isIdempotenceEnabled) throws Exception {
    Map<String, Object> configs = new HashMap<>();
    configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999");
    configs.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 60000);
    configs.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, isIdempotenceEnabled);
    // Create a record with a partition higher than the initial (outdated) partition range
    ProducerRecord<String, String> record = new ProducerRecord<>(topic, 2, null, "value");
    ProducerMetadata metadata = mock(ProducerMetadata.class);
    MockTime mockTime = new MockTime();
    when(metadata.fetch()).thenReturn(onePartitionCluster, onePartitionCluster, threePartitionCluster);
    KafkaProducer<String, String> producer = producerWithOverrideNewSender(configs, metadata, mockTime);
    // One request update if metadata is available but outdated for the given record
    producer.send(record);
    verify(metadata, times(2)).requestUpdateForTopic(topic);
    verify(metadata, times(2)).awaitUpdate(anyInt(), anyLong());
    verify(metadata, times(3)).fetch();
    producer.close(Duration.ofMillis(0));
}

Example 32 with ProducerMetadata

use of org.apache.kafka.clients.producer.internals.ProducerMetadata in project kafka by apache.

the class KafkaProducerTest method testSendToInvalidTopic.

@Test
public void testSendToInvalidTopic() throws Exception {
    Map<String, Object> configs = new HashMap<>();
    configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000");
    configs.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "15000");
    Time time = new MockTime();
    MetadataResponse initialUpdateResponse = RequestTestUtils.metadataUpdateWith(1, emptyMap());
    ProducerMetadata metadata = newMetadata(0, Long.MAX_VALUE);
    metadata.updateWithCurrentRequestVersion(initialUpdateResponse, false, time.milliseconds());
    MockClient client = new MockClient(time, metadata);
    Producer<String, String> producer = kafkaProducer(configs, new StringSerializer(), new StringSerializer(), metadata, client, null, time);
    // Invalid topic name due to space
    String invalidTopicName = "topic abc";
    ProducerRecord<String, String> record = new ProducerRecord<>(invalidTopicName, "HelloKafka");
    List<MetadataResponse.TopicMetadata> topicMetadata = new ArrayList<>();
    topicMetadata.add(new MetadataResponse.TopicMetadata(Errors.INVALID_TOPIC_EXCEPTION, invalidTopicName, false, Collections.emptyList()));
    MetadataResponse updateResponse = RequestTestUtils.metadataResponse(new ArrayList<>(initialUpdateResponse.brokers()), initialUpdateResponse.clusterId(), initialUpdateResponse.controller().id(), topicMetadata);
    client.prepareMetadataUpdate(updateResponse);
    Future<RecordMetadata> future = producer.send(record);
    assertEquals(Collections.singleton(invalidTopicName), metadata.fetch().invalidTopics(), "Cluster has incorrect invalid topic list.");
    TestUtils.assertFutureError(future, InvalidTopicException.class);
    producer.close(Duration.ofMillis(0));
}

Example 33 with ProducerMetadata

use of org.apache.kafka.clients.producer.internals.ProducerMetadata in project kafka by apache.

the class KafkaProducerTest method testCloseWhenWaitingForMetadataUpdate.

@Test
public void testCloseWhenWaitingForMetadataUpdate() throws InterruptedException {
    Map<String, Object> configs = new HashMap<>();
    configs.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, Long.MAX_VALUE);
    configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000");
    // Simulate a case where metadata for a particular topic is not available. This will cause KafkaProducer#send to
    // block in Metadata#awaitUpdate for the configured max.block.ms. When close() is invoked, KafkaProducer#send should
    // return with a KafkaException.
    String topicName = "test";
    Time time = Time.SYSTEM;
    MetadataResponse initialUpdateResponse = RequestTestUtils.metadataUpdateWith(1, emptyMap());
    ProducerMetadata metadata = new ProducerMetadata(0, Long.MAX_VALUE, Long.MAX_VALUE, new LogContext(), new ClusterResourceListeners(), time);
    metadata.updateWithCurrentRequestVersion(initialUpdateResponse, false, time.milliseconds());
    MockClient client = new MockClient(time, metadata);
    Producer<String, String> producer = kafkaProducer(configs, new StringSerializer(), new StringSerializer(), metadata, client, null, time);
    ExecutorService executor = Executors.newSingleThreadExecutor();
    final AtomicReference<Exception> sendException = new AtomicReference<>();
    try {
        executor.submit(() -> {
            try {
                // Metadata for topic "test" will not be available which will cause us to block indefinitely until
                // KafkaProducer#close is invoked.
                producer.send(new ProducerRecord<>(topicName, "key", "value"));
                fail();
            } catch (Exception e) {
                sendException.set(e);
            }
        });
        // Wait until metadata update for the topic has been requested
        TestUtils.waitForCondition(() -> metadata.containsTopic(topicName), "Timeout when waiting for topic to be added to metadata");
        producer.close(Duration.ofMillis(0));
        TestUtils.waitForCondition(() -> sendException.get() != null, "No producer exception within timeout");
        assertEquals(KafkaException.class, sendException.get().getClass());
    } finally {
        executor.shutdownNow();
    }
}

Example 34 with ProducerMetadata

use of org.apache.kafka.clients.producer.internals.ProducerMetadata in project kafka by apache.

the class KafkaProducerTest method testMetadataTimeoutWithMissingTopic.

@ParameterizedTest
@ValueSource(booleans = { true, false })
public void testMetadataTimeoutWithMissingTopic(boolean isIdempotenceEnabled) throws Exception {
    Map<String, Object> configs = new HashMap<>();
    configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999");
    configs.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 60000);
    configs.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, isIdempotenceEnabled);
    // Create a record for a not-yet-created topic
    ProducerRecord<String, String> record = new ProducerRecord<>(topic, 2, null, "value");
    ProducerMetadata metadata = mock(ProducerMetadata.class);
    MockTime mockTime = new MockTime();
    AtomicInteger invocationCount = new AtomicInteger(0);
    when(metadata.fetch()).then(invocation -> {
        invocationCount.incrementAndGet();
        if (invocationCount.get() == 5) {
            mockTime.setCurrentTimeMs(mockTime.milliseconds() + 70000);
        }
        return emptyCluster;
    });
    KafkaProducer<String, String> producer = producerWithOverrideNewSender(configs, metadata, mockTime);
    // Four request updates where the topic isn't present, at which point the timeout expires and a
    // TimeoutException is thrown
    // For idempotence enabled case, the first metadata.fetch will be called in Sender#maybeSendAndPollTransactionalRequest
    Future<RecordMetadata> future = producer.send(record);
    verify(metadata, times(4)).requestUpdateForTopic(topic);
    verify(metadata, times(4)).awaitUpdate(anyInt(), anyLong());
    verify(metadata, times(5)).fetch();
    try {
        future.get();
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof TimeoutException);
    } finally {
        producer.close(Duration.ofMillis(0));
    }
}