Examples with ProduceResponse org.apache.kafka.common.requests.ProduceResponse used on opensource projects

Example 1 with ProduceResponse

use of org.apache.kafka.common.requests.ProduceResponse in project apache-kafka-on-k8s by banzaicloud.

the class SenderTest method testSplitBatchAndSend.

@SuppressWarnings("deprecation")
private void testSplitBatchAndSend(TransactionManager txnManager, ProducerIdAndEpoch producerIdAndEpoch, TopicPartition tp) throws Exception {
    int maxRetries = 1;
    String topic = tp.topic();
    // Set a good compression ratio.
    CompressionRatioEstimator.setEstimation(topic, CompressionType.GZIP, 0.2f);
    try (Metrics m = new Metrics()) {
        accumulator = new RecordAccumulator(logContext, batchSize, 1024 * 1024, CompressionType.GZIP, 0L, 0L, m, time, new ApiVersions(), txnManager);
        SenderMetricsRegistry senderMetrics = new SenderMetricsRegistry(m);
        Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries, senderMetrics, time, REQUEST_TIMEOUT, 1000L, txnManager, new ApiVersions());
        // Create a two broker cluster, with partition 0 on broker 0 and partition 1 on broker 1
        Cluster cluster1 = TestUtils.clusterWith(2, topic, 2);
        metadata.update(cluster1, Collections.<String>emptySet(), time.milliseconds());
        // Send the first message.
        Future<RecordMetadata> f1 = accumulator.append(tp, 0L, "key1".getBytes(), new byte[batchSize / 2], null, null, MAX_BLOCK_TIMEOUT).future;
        Future<RecordMetadata> f2 = accumulator.append(tp, 0L, "key2".getBytes(), new byte[batchSize / 2], null, null, MAX_BLOCK_TIMEOUT).future;
        // connect
        sender.run(time.milliseconds());
        // send produce request
        sender.run(time.milliseconds());
        assertEquals("The next sequence should be 2", 2, txnManager.sequenceNumber(tp).longValue());
        String id = client.requests().peek().destination();
        assertEquals(ApiKeys.PRODUCE, client.requests().peek().requestBuilder().apiKey());
        Node node = new Node(Integer.valueOf(id), "localhost", 0);
        assertEquals(1, client.inFlightRequestCount());
        assertTrue("Client ready status should be true", client.isReady(node, 0L));
        Map<TopicPartition, ProduceResponse.PartitionResponse> responseMap = new HashMap<>();
        responseMap.put(tp, new ProduceResponse.PartitionResponse(Errors.MESSAGE_TOO_LARGE));
        client.respond(new ProduceResponse(responseMap));
        // split and reenqueue
        sender.run(time.milliseconds());
        assertEquals("The next sequence should be 2", 2, txnManager.sequenceNumber(tp).longValue());
        // The compression ratio should have been improved once.
        assertEquals(CompressionType.GZIP.rate - CompressionRatioEstimator.COMPRESSION_RATIO_IMPROVING_STEP, CompressionRatioEstimator.estimation(topic, CompressionType.GZIP), 0.01);
        // send the first produce request
        sender.run(time.milliseconds());
        assertEquals("The next sequence number should be 2", 2, txnManager.sequenceNumber(tp).longValue());
        assertFalse("The future shouldn't have been done.", f1.isDone());
        assertFalse("The future shouldn't have been done.", f2.isDone());
        id = client.requests().peek().destination();
        assertEquals(ApiKeys.PRODUCE, client.requests().peek().requestBuilder().apiKey());
        node = new Node(Integer.valueOf(id), "localhost", 0);
        assertEquals(1, client.inFlightRequestCount());
        assertTrue("Client ready status should be true", client.isReady(node, 0L));
        responseMap.put(tp, new ProduceResponse.PartitionResponse(Errors.NONE, 0L, 0L, 0L));
        client.respond(produceRequestMatcher(tp, producerIdAndEpoch, 0, txnManager.isTransactional()), new ProduceResponse(responseMap));
        // receive
        sender.run(time.milliseconds());
        assertTrue("The future should have been done.", f1.isDone());
        assertEquals("The next sequence number should still be 2", 2, txnManager.sequenceNumber(tp).longValue());
        assertEquals("The last ack'd sequence number should be 0", 0, txnManager.lastAckedSequence(tp));
        assertFalse("The future shouldn't have been done.", f2.isDone());
        assertEquals("Offset of the first message should be 0", 0L, f1.get().offset());
        // send the seconcd produce request
        sender.run(time.milliseconds());
        id = client.requests().peek().destination();
        assertEquals(ApiKeys.PRODUCE, client.requests().peek().requestBuilder().apiKey());
        node = new Node(Integer.valueOf(id), "localhost", 0);
        assertEquals(1, client.inFlightRequestCount());
        assertTrue("Client ready status should be true", client.isReady(node, 0L));
        responseMap.put(tp, new ProduceResponse.PartitionResponse(Errors.NONE, 1L, 0L, 0L));
        client.respond(produceRequestMatcher(tp, producerIdAndEpoch, 1, txnManager.isTransactional()), new ProduceResponse(responseMap));
        // receive
        sender.run(time.milliseconds());
        assertTrue("The future should have been done.", f2.isDone());
        assertEquals("The next sequence number should be 2", 2, txnManager.sequenceNumber(tp).longValue());
        assertEquals("The last ack'd sequence number should be 1", 1, txnManager.lastAckedSequence(tp));
        assertEquals("Offset of the first message should be 1", 1L, f2.get().offset());
        assertTrue("There should be no batch in the accumulator", accumulator.batches().get(tp).isEmpty());
        assertTrue("There should be a split", m.metrics().get(senderMetrics.batchSplitRate).value() > 0);
    }
}

Example 2 with ProduceResponse

use of org.apache.kafka.common.requests.ProduceResponse in project apache-kafka-on-k8s by banzaicloud.

the class Sender method handleProduceResponse.

/**
 * Handle a produce response
 */
private void handleProduceResponse(ClientResponse response, Map<TopicPartition, ProducerBatch> batches, long now) {
    RequestHeader requestHeader = response.requestHeader();
    int correlationId = requestHeader.correlationId();
    if (response.wasDisconnected()) {
        log.trace("Cancelled request with header {} due to node {} being disconnected", requestHeader, response.destination());
        for (ProducerBatch batch : batches.values()) completeBatch(batch, new ProduceResponse.PartitionResponse(Errors.NETWORK_EXCEPTION), correlationId, now);
    } else if (response.versionMismatch() != null) {
        log.warn("Cancelled request {} due to a version mismatch with node {}", response, response.destination(), response.versionMismatch());
        for (ProducerBatch batch : batches.values()) completeBatch(batch, new ProduceResponse.PartitionResponse(Errors.UNSUPPORTED_VERSION), correlationId, now);
    } else {
        log.trace("Received produce response from node {} with correlation id {}", response.destination(), correlationId);
        // if we have a response, parse it
        if (response.hasResponse()) {
            ProduceResponse produceResponse = (ProduceResponse) response.responseBody();
            for (Map.Entry<TopicPartition, ProduceResponse.PartitionResponse> entry : produceResponse.responses().entrySet()) {
                TopicPartition tp = entry.getKey();
                ProduceResponse.PartitionResponse partResp = entry.getValue();
                ProducerBatch batch = batches.get(tp);
                completeBatch(batch, partResp, correlationId, now);
            }
            this.sensors.recordLatency(response.destination(), response.requestLatencyMs());
        } else {
            // this is the acks = 0 case, just complete all requests
            for (ProducerBatch batch : batches.values()) {
                completeBatch(batch, new ProduceResponse.PartitionResponse(Errors.NONE), correlationId, now);
            }
        }
    }
}

Example 3 with ProduceResponse

use of org.apache.kafka.common.requests.ProduceResponse in project kafka by apache.

the class NetworkClientTest method checkSimpleRequestResponse.

private void checkSimpleRequestResponse(NetworkClient networkClient) {
    // has to be before creating any request, as it may send ApiVersionsRequest and its response is mocked with correlation id 0
    awaitReady(networkClient, node);
    short requestVersion = PRODUCE.latestVersion();
    ProduceRequest.Builder builder = new ProduceRequest.Builder(requestVersion, requestVersion, new ProduceRequestData().setAcks((short) 1).setTimeoutMs(1000));
    TestCallbackHandler handler = new TestCallbackHandler();
    ClientRequest request = networkClient.newClientRequest(node.idString(), builder, time.milliseconds(), true, defaultRequestTimeoutMs, handler);
    networkClient.send(request, time.milliseconds());
    networkClient.poll(1, time.milliseconds());
    assertEquals(1, networkClient.inFlightRequestCount());
    ProduceResponse produceResponse = new ProduceResponse(new ProduceResponseData());
    ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(produceResponse, requestVersion, request.correlationId());
    selector.completeReceive(new NetworkReceive(node.idString(), buffer));
    List<ClientResponse> responses = networkClient.poll(1, time.milliseconds());
    assertEquals(1, responses.size());
    assertTrue(handler.executed, "The handler should have executed.");
    assertTrue(handler.response.hasResponse(), "Should have a response body.");
    assertEquals(request.correlationId(), handler.response.requestHeader().correlationId(), "Should be correlated to the original request");
}

Example 4 with ProduceResponse

use of org.apache.kafka.common.requests.ProduceResponse in project kafka by apache.

the class NetworkClientTest method testConnectionThrottling.

@Test
public void testConnectionThrottling() {
    // Instrument the test to return a response with a 100ms throttle delay.
    awaitReady(client, node);
    short requestVersion = PRODUCE.latestVersion();
    ProduceRequest.Builder builder = new ProduceRequest.Builder(requestVersion, requestVersion, new ProduceRequestData().setAcks((short) 1).setTimeoutMs(1000));
    TestCallbackHandler handler = new TestCallbackHandler();
    ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true, defaultRequestTimeoutMs, handler);
    client.send(request, time.milliseconds());
    client.poll(1, time.milliseconds());
    int throttleTime = 100;
    ProduceResponse produceResponse = new ProduceResponse(new ProduceResponseData().setThrottleTimeMs(throttleTime));
    ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(produceResponse, requestVersion, request.correlationId());
    selector.completeReceive(new NetworkReceive(node.idString(), buffer));
    client.poll(1, time.milliseconds());
    // The connection is not ready due to throttling.
    assertFalse(client.ready(node, time.milliseconds()));
    assertEquals(100, client.throttleDelayMs(node, time.milliseconds()));
    // After 50ms, the connection is not ready yet.
    time.sleep(50);
    assertFalse(client.ready(node, time.milliseconds()));
    assertEquals(50, client.throttleDelayMs(node, time.milliseconds()));
    // After another 50ms, the throttling is done and the connection becomes ready again.
    time.sleep(50);
    assertTrue(client.ready(node, time.milliseconds()));
    assertEquals(0, client.throttleDelayMs(node, time.milliseconds()));
}

Example 5 with ProduceResponse

use of org.apache.kafka.common.requests.ProduceResponse in project kafka by apache.

the class Sender method handleProduceResponse.

/**
 * Handle a produce response
 */
private void handleProduceResponse(ClientResponse response, Map<TopicPartition, ProducerBatch> batches, long now) {
    RequestHeader requestHeader = response.requestHeader();
    int correlationId = requestHeader.correlationId();
    if (response.wasDisconnected()) {
        log.trace("Cancelled request with header {} due to node {} being disconnected", requestHeader, response.destination());
        for (ProducerBatch batch : batches.values()) completeBatch(batch, new ProduceResponse.PartitionResponse(Errors.NETWORK_EXCEPTION, String.format("Disconnected from node %s", response.destination())), correlationId, now);
    } else if (response.versionMismatch() != null) {
        log.warn("Cancelled request {} due to a version mismatch with node {}", response, response.destination(), response.versionMismatch());
        for (ProducerBatch batch : batches.values()) completeBatch(batch, new ProduceResponse.PartitionResponse(Errors.UNSUPPORTED_VERSION), correlationId, now);
    } else {
        log.trace("Received produce response from node {} with correlation id {}", response.destination(), correlationId);
        // if we have a response, parse it
        if (response.hasResponse()) {
            // Sender should exercise PartitionProduceResponse rather than ProduceResponse.PartitionResponse
            // https://issues.apache.org/jira/browse/KAFKA-10696
            ProduceResponse produceResponse = (ProduceResponse) response.responseBody();
            produceResponse.data().responses().forEach(r -> r.partitionResponses().forEach(p -> {
                TopicPartition tp = new TopicPartition(r.name(), p.index());
                ProduceResponse.PartitionResponse partResp = new ProduceResponse.PartitionResponse(Errors.forCode(p.errorCode()), p.baseOffset(), p.logAppendTimeMs(), p.logStartOffset(), p.recordErrors().stream().map(e -> new ProduceResponse.RecordError(e.batchIndex(), e.batchIndexErrorMessage())).collect(Collectors.toList()), p.errorMessage());
                ProducerBatch batch = batches.get(tp);
                completeBatch(batch, partResp, correlationId, now);
            }));
            this.sensors.recordLatency(response.destination(), response.requestLatencyMs());
        } else {
            // this is the acks = 0 case, just complete all requests
            for (ProducerBatch batch : batches.values()) {
                completeBatch(batch, new ProduceResponse.PartitionResponse(Errors.NONE), correlationId, now);
            }
        }
    }
}