Examples with ProduceResponse org.apache.kafka.common.requests.ProduceResponse used on opensource projects

Example 6 with ProduceResponse

use of org.apache.kafka.common.requests.ProduceResponse in project kafka by apache.

the class SenderTest method testExpiredBatchesInMultiplePartitions.

@SuppressWarnings("deprecation")
@Test
public void testExpiredBatchesInMultiplePartitions() throws Exception {
    long deliveryTimeoutMs = 1500L;
    setupWithTransactionState(null, true, null);
    // Send multiple ProduceRequest across multiple partitions.
    Future<RecordMetadata> request1 = appendToAccumulator(tp0, time.milliseconds(), "k1", "v1");
    Future<RecordMetadata> request2 = appendToAccumulator(tp1, time.milliseconds(), "k2", "v2");
    // Send request.
    sender.runOnce();
    assertEquals(1, client.inFlightRequestCount());
    assertEquals(1, sender.inFlightBatches(tp0).size(), "Expect one in-flight batch in accumulator");
    Map<TopicPartition, ProduceResponse.PartitionResponse> responseMap = new HashMap<>();
    responseMap.put(tp0, new ProduceResponse.PartitionResponse(Errors.NONE, 0L, 0L, 0L));
    client.respond(new ProduceResponse(responseMap));
    // Successfully expire both batches.
    time.sleep(deliveryTimeoutMs);
    sender.runOnce();
    assertEquals(0, sender.inFlightBatches(tp0).size(), "Expect zero in-flight batch in accumulator");
    ExecutionException e = assertThrows(ExecutionException.class, request1::get);
    assertTrue(e.getCause() instanceof TimeoutException);
    e = assertThrows(ExecutionException.class, request2::get);
    assertTrue(e.getCause() instanceof TimeoutException);
}

Example 7 with ProduceResponse

use of org.apache.kafka.common.requests.ProduceResponse in project kafka by apache.

the class SenderTest method produceResponse.

private ProduceResponse produceResponse(Map<TopicPartition, OffsetAndError> responses) {
    ProduceResponseData data = new ProduceResponseData();
    for (Map.Entry<TopicPartition, OffsetAndError> entry : responses.entrySet()) {
        TopicPartition topicPartition = entry.getKey();
        ProduceResponseData.TopicProduceResponse topicData = data.responses().find(topicPartition.topic());
        if (topicData == null) {
            topicData = new ProduceResponseData.TopicProduceResponse().setName(topicPartition.topic());
            data.responses().add(topicData);
        }
        OffsetAndError offsetAndError = entry.getValue();
        ProduceResponseData.PartitionProduceResponse partitionData = new ProduceResponseData.PartitionProduceResponse().setIndex(topicPartition.partition()).setBaseOffset(offsetAndError.offset).setErrorCode(offsetAndError.error.code()).setRecordErrors(offsetAndError.recordErrors);
        topicData.partitionResponses().add(partitionData);
    }
    return new ProduceResponse(data);
}

Example 8 with ProduceResponse

use of org.apache.kafka.common.requests.ProduceResponse in project kafka by apache.

the class SenderTest method testInflightBatchesExpireOnDeliveryTimeout.

@SuppressWarnings("deprecation")
@Test
public void testInflightBatchesExpireOnDeliveryTimeout() throws InterruptedException {
    long deliveryTimeoutMs = 1500L;
    setupWithTransactionState(null, true, null);
    // Send first ProduceRequest
    Future<RecordMetadata> request = appendToAccumulator(tp0);
    // send request
    sender.runOnce();
    assertEquals(1, client.inFlightRequestCount());
    assertEquals(1, sender.inFlightBatches(tp0).size(), "Expect one in-flight batch in accumulator");
    Map<TopicPartition, ProduceResponse.PartitionResponse> responseMap = new HashMap<>();
    responseMap.put(tp0, new ProduceResponse.PartitionResponse(Errors.NONE, 0L, 0L, 0L));
    client.respond(new ProduceResponse(responseMap));
    time.sleep(deliveryTimeoutMs);
    // receive first response
    sender.runOnce();
    assertEquals(0, sender.inFlightBatches(tp0).size(), "Expect zero in-flight batch in accumulator");
    try {
        request.get();
        fail("The expired batch should throw a TimeoutException");
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof TimeoutException);
    }
}

Example 9 with ProduceResponse

use of org.apache.kafka.common.requests.ProduceResponse in project kafka by apache.

the class SenderTest method testSplitBatchAndSend.

@SuppressWarnings("deprecation")
private void testSplitBatchAndSend(TransactionManager txnManager, ProducerIdAndEpoch producerIdAndEpoch, TopicPartition tp) throws Exception {
    int maxRetries = 1;
    String topic = tp.topic();
    int deliveryTimeoutMs = 3000;
    long totalSize = 1024 * 1024;
    String metricGrpName = "producer-metrics";
    // Set a good compression ratio.
    CompressionRatioEstimator.setEstimation(topic, CompressionType.GZIP, 0.2f);
    try (Metrics m = new Metrics()) {
        accumulator = new RecordAccumulator(logContext, batchSize, CompressionType.GZIP, 0, 0L, deliveryTimeoutMs, m, metricGrpName, time, new ApiVersions(), txnManager, new BufferPool(totalSize, batchSize, metrics, time, "producer-internal-metrics"));
        SenderMetricsRegistry senderMetrics = new SenderMetricsRegistry(m);
        Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries, senderMetrics, time, REQUEST_TIMEOUT, 1000L, txnManager, new ApiVersions());
        // Create a two broker cluster, with partition 0 on broker 0 and partition 1 on broker 1
        MetadataResponse metadataUpdate1 = RequestTestUtils.metadataUpdateWith(2, Collections.singletonMap(topic, 2));
        client.prepareMetadataUpdate(metadataUpdate1);
        // Send the first message.
        long nowMs = time.milliseconds();
        Future<RecordMetadata> f1 = accumulator.append(tp, 0L, "key1".getBytes(), new byte[batchSize / 2], null, null, MAX_BLOCK_TIMEOUT, false, nowMs).future;
        Future<RecordMetadata> f2 = accumulator.append(tp, 0L, "key2".getBytes(), new byte[batchSize / 2], null, null, MAX_BLOCK_TIMEOUT, false, nowMs).future;
        // connect
        sender.runOnce();
        // send produce request
        sender.runOnce();
        assertEquals(2, txnManager.sequenceNumber(tp).longValue(), "The next sequence should be 2");
        String id = client.requests().peek().destination();
        assertEquals(ApiKeys.PRODUCE, client.requests().peek().requestBuilder().apiKey());
        Node node = new Node(Integer.valueOf(id), "localhost", 0);
        assertEquals(1, client.inFlightRequestCount());
        assertTrue(client.isReady(node, time.milliseconds()), "Client ready status should be true");
        Map<TopicPartition, ProduceResponse.PartitionResponse> responseMap = new HashMap<>();
        responseMap.put(tp, new ProduceResponse.PartitionResponse(Errors.MESSAGE_TOO_LARGE));
        client.respond(new ProduceResponse(responseMap));
        // split and reenqueue
        sender.runOnce();
        assertEquals(2, txnManager.sequenceNumber(tp).longValue(), "The next sequence should be 2");
        // The compression ratio should have been improved once.
        assertEquals(CompressionType.GZIP.rate - CompressionRatioEstimator.COMPRESSION_RATIO_IMPROVING_STEP, CompressionRatioEstimator.estimation(topic, CompressionType.GZIP), 0.01);
        // send the first produce request
        sender.runOnce();
        assertEquals(2, txnManager.sequenceNumber(tp).longValue(), "The next sequence number should be 2");
        assertFalse(f1.isDone(), "The future shouldn't have been done.");
        assertFalse(f2.isDone(), "The future shouldn't have been done.");
        id = client.requests().peek().destination();
        assertEquals(ApiKeys.PRODUCE, client.requests().peek().requestBuilder().apiKey());
        node = new Node(Integer.valueOf(id), "localhost", 0);
        assertEquals(1, client.inFlightRequestCount());
        assertTrue(client.isReady(node, time.milliseconds()), "Client ready status should be true");
        responseMap.put(tp, new ProduceResponse.PartitionResponse(Errors.NONE, 0L, 0L, 0L));
        client.respond(produceRequestMatcher(tp, producerIdAndEpoch, 0, txnManager.isTransactional()), new ProduceResponse(responseMap));
        // receive
        sender.runOnce();
        assertTrue(f1.isDone(), "The future should have been done.");
        assertEquals(2, txnManager.sequenceNumber(tp).longValue(), "The next sequence number should still be 2");
        assertEquals(OptionalInt.of(0), txnManager.lastAckedSequence(tp), "The last ack'd sequence number should be 0");
        assertFalse(f2.isDone(), "The future shouldn't have been done.");
        assertEquals(0L, f1.get().offset(), "Offset of the first message should be 0");
        // send the seconcd produce request
        sender.runOnce();
        id = client.requests().peek().destination();
        assertEquals(ApiKeys.PRODUCE, client.requests().peek().requestBuilder().apiKey());
        node = new Node(Integer.valueOf(id), "localhost", 0);
        assertEquals(1, client.inFlightRequestCount());
        assertTrue(client.isReady(node, time.milliseconds()), "Client ready status should be true");
        responseMap.put(tp, new ProduceResponse.PartitionResponse(Errors.NONE, 1L, 0L, 0L));
        client.respond(produceRequestMatcher(tp, producerIdAndEpoch, 1, txnManager.isTransactional()), new ProduceResponse(responseMap));
        // receive
        sender.runOnce();
        assertTrue(f2.isDone(), "The future should have been done.");
        assertEquals(2, txnManager.sequenceNumber(tp).longValue(), "The next sequence number should be 2");
        assertEquals(OptionalInt.of(1), txnManager.lastAckedSequence(tp), "The last ack'd sequence number should be 1");
        assertEquals(1L, f2.get().offset(), "Offset of the first message should be 1");
        assertTrue(accumulator.batches().get(tp).isEmpty(), "There should be no batch in the accumulator");
        assertTrue((Double) (m.metrics().get(senderMetrics.batchSplitRate).metricValue()) > 0, "There should be a split");
    }
}

Example 10 with ProduceResponse

use of org.apache.kafka.common.requests.ProduceResponse in project apache-kafka-on-k8s by banzaicloud.

the class SenderTest method testQuotaMetrics.

/*
     * Send multiple requests. Verify that the client side quota metrics have the right values
     */
@Test
@SuppressWarnings("deprecation")
public void testQuotaMetrics() throws Exception {
    MockSelector selector = new MockSelector(time);
    Sensor throttleTimeSensor = Sender.throttleTimeSensor(this.senderMetricsRegistry);
    Cluster cluster = TestUtils.singletonCluster("test", 1);
    Node node = cluster.nodes().get(0);
    NetworkClient client = new NetworkClient(selector, metadata, "mock", Integer.MAX_VALUE, 1000, 1000, 64 * 1024, 64 * 1024, 1000, time, true, new ApiVersions(), throttleTimeSensor, logContext);
    short apiVersionsResponseVersion = ApiKeys.API_VERSIONS.latestVersion();
    ByteBuffer buffer = ApiVersionsResponse.createApiVersionsResponse(400, RecordBatch.CURRENT_MAGIC_VALUE).serialize(apiVersionsResponseVersion, new ResponseHeader(0));
    selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer)));
    while (!client.ready(node, time.milliseconds())) client.poll(1, time.milliseconds());
    selector.clear();
    for (int i = 1; i <= 3; i++) {
        int throttleTimeMs = 100 * i;
        ProduceRequest.Builder builder = ProduceRequest.Builder.forCurrentMagic((short) 1, 1000, Collections.<TopicPartition, MemoryRecords>emptyMap());
        ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true, null);
        client.send(request, time.milliseconds());
        client.poll(1, time.milliseconds());
        ProduceResponse response = produceResponse(tp0, i, Errors.NONE, throttleTimeMs);
        buffer = response.serialize(ApiKeys.PRODUCE.latestVersion(), new ResponseHeader(request.correlationId()));
        selector.completeReceive(new NetworkReceive(node.idString(), buffer));
        client.poll(1, time.milliseconds());
        selector.clear();
    }
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric avgMetric = allMetrics.get(this.senderMetricsRegistry.produceThrottleTimeAvg);
    KafkaMetric maxMetric = allMetrics.get(this.senderMetricsRegistry.produceThrottleTimeMax);
    // Throttle times are ApiVersions=400, Produce=(100, 200, 300)
    assertEquals(250, avgMetric.value(), EPS);
    assertEquals(400, maxMetric.value(), EPS);
    client.close();
}