Examples with MetadataResponse org.apache.kafka.common.requests.MetadataResponse used on opensource projects

Example 16 with MetadataResponse

use of org.apache.kafka.common.requests.MetadataResponse in project kafka by apache.

the class SenderTest method testSplitBatchAndSend.

@SuppressWarnings("deprecation")
private void testSplitBatchAndSend(TransactionManager txnManager, ProducerIdAndEpoch producerIdAndEpoch, TopicPartition tp) throws Exception {
    int maxRetries = 1;
    String topic = tp.topic();
    int deliveryTimeoutMs = 3000;
    long totalSize = 1024 * 1024;
    String metricGrpName = "producer-metrics";
    // Set a good compression ratio.
    CompressionRatioEstimator.setEstimation(topic, CompressionType.GZIP, 0.2f);
    try (Metrics m = new Metrics()) {
        accumulator = new RecordAccumulator(logContext, batchSize, CompressionType.GZIP, 0, 0L, deliveryTimeoutMs, m, metricGrpName, time, new ApiVersions(), txnManager, new BufferPool(totalSize, batchSize, metrics, time, "producer-internal-metrics"));
        SenderMetricsRegistry senderMetrics = new SenderMetricsRegistry(m);
        Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries, senderMetrics, time, REQUEST_TIMEOUT, 1000L, txnManager, new ApiVersions());
        // Create a two broker cluster, with partition 0 on broker 0 and partition 1 on broker 1
        MetadataResponse metadataUpdate1 = RequestTestUtils.metadataUpdateWith(2, Collections.singletonMap(topic, 2));
        client.prepareMetadataUpdate(metadataUpdate1);
        // Send the first message.
        long nowMs = time.milliseconds();
        Future<RecordMetadata> f1 = accumulator.append(tp, 0L, "key1".getBytes(), new byte[batchSize / 2], null, null, MAX_BLOCK_TIMEOUT, false, nowMs).future;
        Future<RecordMetadata> f2 = accumulator.append(tp, 0L, "key2".getBytes(), new byte[batchSize / 2], null, null, MAX_BLOCK_TIMEOUT, false, nowMs).future;
        // connect
        sender.runOnce();
        // send produce request
        sender.runOnce();
        assertEquals(2, txnManager.sequenceNumber(tp).longValue(), "The next sequence should be 2");
        String id = client.requests().peek().destination();
        assertEquals(ApiKeys.PRODUCE, client.requests().peek().requestBuilder().apiKey());
        Node node = new Node(Integer.valueOf(id), "localhost", 0);
        assertEquals(1, client.inFlightRequestCount());
        assertTrue(client.isReady(node, time.milliseconds()), "Client ready status should be true");
        Map<TopicPartition, ProduceResponse.PartitionResponse> responseMap = new HashMap<>();
        responseMap.put(tp, new ProduceResponse.PartitionResponse(Errors.MESSAGE_TOO_LARGE));
        client.respond(new ProduceResponse(responseMap));
        // split and reenqueue
        sender.runOnce();
        assertEquals(2, txnManager.sequenceNumber(tp).longValue(), "The next sequence should be 2");
        // The compression ratio should have been improved once.
        assertEquals(CompressionType.GZIP.rate - CompressionRatioEstimator.COMPRESSION_RATIO_IMPROVING_STEP, CompressionRatioEstimator.estimation(topic, CompressionType.GZIP), 0.01);
        // send the first produce request
        sender.runOnce();
        assertEquals(2, txnManager.sequenceNumber(tp).longValue(), "The next sequence number should be 2");
        assertFalse(f1.isDone(), "The future shouldn't have been done.");
        assertFalse(f2.isDone(), "The future shouldn't have been done.");
        id = client.requests().peek().destination();
        assertEquals(ApiKeys.PRODUCE, client.requests().peek().requestBuilder().apiKey());
        node = new Node(Integer.valueOf(id), "localhost", 0);
        assertEquals(1, client.inFlightRequestCount());
        assertTrue(client.isReady(node, time.milliseconds()), "Client ready status should be true");
        responseMap.put(tp, new ProduceResponse.PartitionResponse(Errors.NONE, 0L, 0L, 0L));
        client.respond(produceRequestMatcher(tp, producerIdAndEpoch, 0, txnManager.isTransactional()), new ProduceResponse(responseMap));
        // receive
        sender.runOnce();
        assertTrue(f1.isDone(), "The future should have been done.");
        assertEquals(2, txnManager.sequenceNumber(tp).longValue(), "The next sequence number should still be 2");
        assertEquals(OptionalInt.of(0), txnManager.lastAckedSequence(tp), "The last ack'd sequence number should be 0");
        assertFalse(f2.isDone(), "The future shouldn't have been done.");
        assertEquals(0L, f1.get().offset(), "Offset of the first message should be 0");
        // send the seconcd produce request
        sender.runOnce();
        id = client.requests().peek().destination();
        assertEquals(ApiKeys.PRODUCE, client.requests().peek().requestBuilder().apiKey());
        node = new Node(Integer.valueOf(id), "localhost", 0);
        assertEquals(1, client.inFlightRequestCount());
        assertTrue(client.isReady(node, time.milliseconds()), "Client ready status should be true");
        responseMap.put(tp, new ProduceResponse.PartitionResponse(Errors.NONE, 1L, 0L, 0L));
        client.respond(produceRequestMatcher(tp, producerIdAndEpoch, 1, txnManager.isTransactional()), new ProduceResponse(responseMap));
        // receive
        sender.runOnce();
        assertTrue(f2.isDone(), "The future should have been done.");
        assertEquals(2, txnManager.sequenceNumber(tp).longValue(), "The next sequence number should be 2");
        assertEquals(OptionalInt.of(1), txnManager.lastAckedSequence(tp), "The last ack'd sequence number should be 1");
        assertEquals(1L, f2.get().offset(), "Offset of the first message should be 1");
        assertTrue(accumulator.batches().get(tp).isEmpty(), "There should be no batch in the accumulator");
        assertTrue((Double) (m.metrics().get(senderMetrics.batchSplitRate).metricValue()) > 0, "There should be a split");
    }
}

Example 17 with MetadataResponse

use of org.apache.kafka.common.requests.MetadataResponse in project kafka by apache.

the class KafkaProducerTest method testInitTransactionWhileThrottled.

@Test
public void testInitTransactionWhileThrottled() {
    Map<String, Object> configs = new HashMap<>();
    configs.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "some.id");
    configs.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 10000);
    configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000");
    Time time = new MockTime(1);
    MetadataResponse initialUpdateResponse = RequestTestUtils.metadataUpdateWith(1, singletonMap("topic", 1));
    ProducerMetadata metadata = newMetadata(0, Long.MAX_VALUE);
    MockClient client = new MockClient(time, metadata);
    client.updateMetadata(initialUpdateResponse);
    Node node = metadata.fetch().nodes().get(0);
    client.throttle(node, 5000);
    client.prepareResponse(FindCoordinatorResponse.prepareResponse(Errors.NONE, "some.id", NODE));
    client.prepareResponse(initProducerIdResponse(1L, (short) 5, Errors.NONE));
    try (Producer<String, String> producer = kafkaProducer(configs, new StringSerializer(), new StringSerializer(), metadata, client, null, time)) {
        producer.initTransactions();
    }
}

Example 18 with MetadataResponse

use of org.apache.kafka.common.requests.MetadataResponse in project kafka by apache.

the class KafkaProducerTest method testCloseIsForcedOnPendingFindCoordinator.

@Test
public void testCloseIsForcedOnPendingFindCoordinator() throws InterruptedException {
    Map<String, Object> configs = new HashMap<>();
    configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000");
    configs.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "this-is-a-transactional-id");
    Time time = new MockTime();
    MetadataResponse initialUpdateResponse = RequestTestUtils.metadataUpdateWith(1, singletonMap("testTopic", 1));
    ProducerMetadata metadata = newMetadata(0, Long.MAX_VALUE);
    metadata.updateWithCurrentRequestVersion(initialUpdateResponse, false, time.milliseconds());
    MockClient client = new MockClient(time, metadata);
    Producer<String, String> producer = kafkaProducer(configs, new StringSerializer(), new StringSerializer(), metadata, client, null, time);
    ExecutorService executorService = Executors.newSingleThreadExecutor();
    CountDownLatch assertionDoneLatch = new CountDownLatch(1);
    executorService.submit(() -> {
        assertThrows(KafkaException.class, producer::initTransactions);
        assertionDoneLatch.countDown();
    });
    client.waitForRequests(1, 2000);
    producer.close(Duration.ofMillis(1000));
    assertionDoneLatch.await(5000, TimeUnit.MILLISECONDS);
}

Example 19 with MetadataResponse

use of org.apache.kafka.common.requests.MetadataResponse in project kafka by apache.

the class KafkaProducerTest method testTopicExpiryInMetadata.

@Test
public void testTopicExpiryInMetadata() throws InterruptedException {
    Map<String, Object> configs = new HashMap<>();
    configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999");
    configs.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "30000");
    long refreshBackoffMs = 500L;
    long metadataExpireMs = 60000L;
    long metadataIdleMs = 60000L;
    final Time time = new MockTime();
    final ProducerMetadata metadata = new ProducerMetadata(refreshBackoffMs, metadataExpireMs, metadataIdleMs, new LogContext(), new ClusterResourceListeners(), time);
    final String topic = "topic";
    try (KafkaProducer<String, String> producer = kafkaProducer(configs, new StringSerializer(), new StringSerializer(), metadata, new MockClient(time, metadata), null, time)) {
        Exchanger<Void> exchanger = new Exchanger<>();
        Thread t = new Thread(() -> {
            try {
                // 1
                exchanger.exchange(null);
                while (!metadata.updateRequested()) Thread.sleep(100);
                MetadataResponse updateResponse = RequestTestUtils.metadataUpdateWith(1, singletonMap(topic, 1));
                metadata.updateWithCurrentRequestVersion(updateResponse, false, time.milliseconds());
                // 2
                exchanger.exchange(null);
                time.sleep(120 * 1000L);
                // Update the metadata again, but it should be expired at this point.
                updateResponse = RequestTestUtils.metadataUpdateWith(1, singletonMap(topic, 1));
                metadata.updateWithCurrentRequestVersion(updateResponse, false, time.milliseconds());
                // 3
                exchanger.exchange(null);
                while (!metadata.updateRequested()) Thread.sleep(100);
                time.sleep(30 * 1000L);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        t.start();
        // 1
        exchanger.exchange(null);
        assertNotNull(producer.partitionsFor(topic));
        // 2
        exchanger.exchange(null);
        // 3
        exchanger.exchange(null);
        assertThrows(TimeoutException.class, () -> producer.partitionsFor(topic));
        t.join();
    }
}

Example 20 with MetadataResponse

use of org.apache.kafka.common.requests.MetadataResponse in project kafka by apache.

the class KafkaProducerTest method testFlushMeasureLatency.

@Test
public void testFlushMeasureLatency() {
    Map<String, Object> configs = new HashMap<>();
    configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000");
    Time time = new MockTime(1);
    MetadataResponse initialUpdateResponse = RequestTestUtils.metadataUpdateWith(1, singletonMap("topic", 1));
    ProducerMetadata metadata = newMetadata(0, Long.MAX_VALUE);
    MockClient client = new MockClient(time, metadata);
    client.updateMetadata(initialUpdateResponse);
    try (KafkaProducer<String, String> producer = kafkaProducer(configs, new StringSerializer(), new StringSerializer(), metadata, client, null, time)) {
        producer.flush();
        double first = getMetricValue(producer, "flush-time-ns-total");
        assertTrue(first > 0);
        producer.flush();
        assertTrue(getMetricValue(producer, "flush-time-ns-total") > first);
    }
}