Examples with ProduceRequest org.apache.kafka.common.requests.ProduceRequest used on opensource projects

Example 11 with ProduceRequest

use of org.apache.kafka.common.requests.ProduceRequest in project apache-kafka-on-k8s by banzaicloud.

the class SenderTest method testDownConversionForMismatchedMagicValues.

@Test
public void testDownConversionForMismatchedMagicValues() throws Exception {
    // it can happen that we construct a record set with mismatching magic values (perhaps
    // because the partition leader changed after the record set was initially constructed)
    // in this case, we down-convert record sets with newer magic values to match the oldest
    // created record set
    long offset = 0;
    // start off support produce request v3
    apiVersions.update("0", NodeApiVersions.create());
    Future<RecordMetadata> future1 = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
    // now the partition leader supports only v2
    apiVersions.update("0", NodeApiVersions.create(Collections.singleton(new ApiVersionsResponse.ApiVersion(ApiKeys.PRODUCE.id, (short) 0, (short) 2))));
    Future<RecordMetadata> future2 = accumulator.append(tp1, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
    // start off support produce request v3
    apiVersions.update("0", NodeApiVersions.create());
    ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse(Errors.NONE, offset, RecordBatch.NO_TIMESTAMP, 100);
    Map<TopicPartition, ProduceResponse.PartitionResponse> partResp = new HashMap<>();
    partResp.put(tp0, resp);
    partResp.put(tp1, resp);
    ProduceResponse produceResponse = new ProduceResponse(partResp, 0);
    client.prepareResponse(new MockClient.RequestMatcher() {

        @Override
        public boolean matches(AbstractRequest body) {
            ProduceRequest request = (ProduceRequest) body;
            if (request.version() != 2)
                return false;
            Map<TopicPartition, MemoryRecords> recordsMap = request.partitionRecordsOrFail();
            if (recordsMap.size() != 2)
                return false;
            for (MemoryRecords records : recordsMap.values()) {
                if (records == null || records.sizeInBytes() == 0 || !records.hasMatchingMagic(RecordBatch.MAGIC_VALUE_V1))
                    return false;
            }
            return true;
        }
    }, produceResponse);
    // connect
    sender.run(time.milliseconds());
    // send produce request
    sender.run(time.milliseconds());
    assertTrue("Request should be completed", future1.isDone());
    assertTrue("Request should be completed", future2.isDone());
}

Example 12 with ProduceRequest

use of org.apache.kafka.common.requests.ProduceRequest in project apache-kafka-on-k8s by banzaicloud.

the class SenderTest method testClusterAuthorizationExceptionInProduceRequest.

@Test
public void testClusterAuthorizationExceptionInProduceRequest() throws Exception {
    final long producerId = 343434L;
    TransactionManager transactionManager = new TransactionManager();
    setupWithTransactionState(transactionManager);
    client.setNode(new Node(1, "localhost", 33343));
    prepareAndReceiveInitProducerId(producerId, Errors.NONE);
    assertTrue(transactionManager.hasProducerId());
    // cluster authorization is a fatal error for the producer
    Future<RecordMetadata> future = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
    client.prepareResponse(new MockClient.RequestMatcher() {

        @Override
        public boolean matches(AbstractRequest body) {
            return body instanceof ProduceRequest && ((ProduceRequest) body).isIdempotent();
        }
    }, produceResponse(tp0, -1, Errors.CLUSTER_AUTHORIZATION_FAILED, 0));
    sender.run(time.milliseconds());
    assertFutureFailure(future, ClusterAuthorizationException.class);
    // cluster authorization errors are fatal, so we should continue seeing it on future sends
    assertTrue(transactionManager.hasFatalError());
    assertSendFailure(ClusterAuthorizationException.class);
}

Example 13 with ProduceRequest

use of org.apache.kafka.common.requests.ProduceRequest in project kafka by apache.

the class SenderTest method testDownConversionForMismatchedMagicValues.

@SuppressWarnings("deprecation")
@Test
public void testDownConversionForMismatchedMagicValues() throws Exception {
    // it can happen that we construct a record set with mismatching magic values (perhaps
    // because the partition leader changed after the record set was initially constructed)
    // in this case, we down-convert record sets with newer magic values to match the oldest
    // created record set
    long offset = 0;
    // start off support produce request v3
    apiVersions.update("0", NodeApiVersions.create());
    Future<RecordMetadata> future1 = appendToAccumulator(tp0, 0L, "key", "value");
    // now the partition leader supports only v2
    apiVersions.update("0", NodeApiVersions.create(ApiKeys.PRODUCE.id, (short) 0, (short) 2));
    Future<RecordMetadata> future2 = appendToAccumulator(tp1, 0L, "key", "value");
    // start off support produce request v3
    apiVersions.update("0", NodeApiVersions.create());
    ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse(Errors.NONE, offset, RecordBatch.NO_TIMESTAMP, 100);
    Map<TopicPartition, ProduceResponse.PartitionResponse> partResp = new HashMap<>();
    partResp.put(tp0, resp);
    partResp.put(tp1, resp);
    ProduceResponse produceResponse = new ProduceResponse(partResp, 0);
    client.prepareResponse(body -> {
        ProduceRequest request = (ProduceRequest) body;
        if (request.version() != 2)
            return false;
        Map<TopicPartition, MemoryRecords> recordsMap = partitionRecords(request);
        if (recordsMap.size() != 2)
            return false;
        for (MemoryRecords records : recordsMap.values()) {
            if (records == null || records.sizeInBytes() == 0 || !records.hasMatchingMagic(RecordBatch.MAGIC_VALUE_V1))
                return false;
        }
        return true;
    }, produceResponse);
    // connect
    sender.runOnce();
    // send produce request
    sender.runOnce();
    assertTrue(future1.isDone(), "Request should be completed");
    assertTrue(future2.isDone(), "Request should be completed");
}

Example 14 with ProduceRequest

use of org.apache.kafka.common.requests.ProduceRequest in project kafka by apache.

the class SenderTest method partitionRecords.

private static Map<TopicPartition, MemoryRecords> partitionRecords(ProduceRequest request) {
    Map<TopicPartition, MemoryRecords> partitionRecords = new HashMap<>();
    request.data().topicData().forEach(tpData -> tpData.partitionData().forEach(p -> {
        TopicPartition tp = new TopicPartition(tpData.name(), p.index());
        partitionRecords.put(tp, (MemoryRecords) p.records());
    }));
    return Collections.unmodifiableMap(partitionRecords);
}

Example 15 with ProduceRequest

use of org.apache.kafka.common.requests.ProduceRequest in project kafka by apache.

the class SenderTest method sendIdempotentProducerResponse.

void sendIdempotentProducerResponse(int expectedEpoch, int expectedSequence, TopicPartition tp, Errors responseError, long responseOffset, long logStartOffset) {
    client.respond(body -> {
        ProduceRequest produceRequest = (ProduceRequest) body;
        assertTrue(RequestTestUtils.hasIdempotentRecords(produceRequest));
        MemoryRecords records = partitionRecords(produceRequest).get(tp);
        Iterator<MutableRecordBatch> batchIterator = records.batches().iterator();
        RecordBatch firstBatch = batchIterator.next();
        assertFalse(batchIterator.hasNext());
        if (expectedEpoch > -1)
            assertEquals((short) expectedEpoch, firstBatch.producerEpoch());
        assertEquals(expectedSequence, firstBatch.baseSequence());
        return true;
    }, produceResponse(tp, responseOffset, responseError, 0, logStartOffset, null));
}