Search in sources :

Example 1 with MutableRecordBatch

use of org.apache.kafka.common.record.MutableRecordBatch in project apache-kafka-on-k8s by banzaicloud.

the class RecordAccumulatorTest method testAppendLarge.

private void testAppendLarge(CompressionType compressionType) throws Exception {
    int batchSize = 512;
    byte[] value = new byte[2 * batchSize];
    RecordAccumulator accum = createTestRecordAccumulator(batchSize + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 10 * 1024, compressionType, 0L);
    accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
    assertEquals("Our partition's leader should be ready", Collections.singleton(node1), accum.ready(cluster, time.milliseconds()).readyNodes);
    Deque<ProducerBatch> batches = accum.batches().get(tp1);
    assertEquals(1, batches.size());
    ProducerBatch producerBatch = batches.peek();
    List<MutableRecordBatch> recordBatches = TestUtils.toList(producerBatch.records().batches());
    assertEquals(1, recordBatches.size());
    MutableRecordBatch recordBatch = recordBatches.get(0);
    assertEquals(0L, recordBatch.baseOffset());
    List<Record> records = TestUtils.toList(recordBatch);
    assertEquals(1, records.size());
    Record record = records.get(0);
    assertEquals(0L, record.offset());
    assertEquals(ByteBuffer.wrap(key), record.key());
    assertEquals(ByteBuffer.wrap(value), record.value());
    assertEquals(0L, record.timestamp());
}
Also used : MutableRecordBatch(org.apache.kafka.common.record.MutableRecordBatch) DefaultRecord(org.apache.kafka.common.record.DefaultRecord) Record(org.apache.kafka.common.record.Record)

Example 2 with MutableRecordBatch

use of org.apache.kafka.common.record.MutableRecordBatch in project apache-kafka-on-k8s by banzaicloud.

the class RecordAccumulatorTest method testAppendLargeOldMessageFormat.

private void testAppendLargeOldMessageFormat(CompressionType compressionType) throws Exception {
    int batchSize = 512;
    byte[] value = new byte[2 * batchSize];
    ApiVersions apiVersions = new ApiVersions();
    apiVersions.update(node1.idString(), NodeApiVersions.create(Collections.singleton(new ApiVersionsResponse.ApiVersion(ApiKeys.PRODUCE.id, (short) 0, (short) 2))));
    RecordAccumulator accum = createTestRecordAccumulator(batchSize + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 10 * 1024, compressionType, 0L);
    accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
    assertEquals("Our partition's leader should be ready", Collections.singleton(node1), accum.ready(cluster, time.milliseconds()).readyNodes);
    Deque<ProducerBatch> batches = accum.batches().get(tp1);
    assertEquals(1, batches.size());
    ProducerBatch producerBatch = batches.peek();
    List<MutableRecordBatch> recordBatches = TestUtils.toList(producerBatch.records().batches());
    assertEquals(1, recordBatches.size());
    MutableRecordBatch recordBatch = recordBatches.get(0);
    assertEquals(0L, recordBatch.baseOffset());
    List<Record> records = TestUtils.toList(recordBatch);
    assertEquals(1, records.size());
    Record record = records.get(0);
    assertEquals(0L, record.offset());
    assertEquals(ByteBuffer.wrap(key), record.key());
    assertEquals(ByteBuffer.wrap(value), record.value());
    assertEquals(0L, record.timestamp());
}
Also used : ApiVersionsResponse(org.apache.kafka.common.requests.ApiVersionsResponse) MutableRecordBatch(org.apache.kafka.common.record.MutableRecordBatch) ApiVersions(org.apache.kafka.clients.ApiVersions) NodeApiVersions(org.apache.kafka.clients.NodeApiVersions) DefaultRecord(org.apache.kafka.common.record.DefaultRecord) Record(org.apache.kafka.common.record.Record)

Example 3 with MutableRecordBatch

use of org.apache.kafka.common.record.MutableRecordBatch in project apache-kafka-on-k8s by banzaicloud.

the class SenderTest method testSequenceNumberIncrement.

@Test
public void testSequenceNumberIncrement() throws InterruptedException {
    final long producerId = 343434L;
    TransactionManager transactionManager = new TransactionManager();
    transactionManager.setProducerIdAndEpoch(new ProducerIdAndEpoch(producerId, (short) 0));
    setupWithTransactionState(transactionManager);
    client.setNode(new Node(1, "localhost", 33343));
    int maxRetries = 10;
    Metrics m = new Metrics();
    SenderMetricsRegistry senderMetrics = new SenderMetricsRegistry(m);
    Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries, senderMetrics, time, REQUEST_TIMEOUT, 50, transactionManager, apiVersions);
    Future<RecordMetadata> responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
    client.prepareResponse(new MockClient.RequestMatcher() {

        @Override
        public boolean matches(AbstractRequest body) {
            if (body instanceof ProduceRequest) {
                ProduceRequest request = (ProduceRequest) body;
                MemoryRecords records = request.partitionRecordsOrFail().get(tp0);
                Iterator<MutableRecordBatch> batchIterator = records.batches().iterator();
                assertTrue(batchIterator.hasNext());
                RecordBatch batch = batchIterator.next();
                assertFalse(batchIterator.hasNext());
                assertEquals(0, batch.baseSequence());
                assertEquals(producerId, batch.producerId());
                assertEquals(0, batch.producerEpoch());
                return true;
            }
            return false;
        }
    }, produceResponse(tp0, 0, Errors.NONE, 0));
    // connect.
    sender.run(time.milliseconds());
    // send.
    sender.run(time.milliseconds());
    // receive response
    sender.run(time.milliseconds());
    assertTrue(responseFuture.isDone());
    assertEquals(0L, (long) transactionManager.lastAckedSequence(tp0));
    assertEquals(1L, (long) transactionManager.sequenceNumber(tp0));
}
Also used : ProduceRequest(org.apache.kafka.common.requests.ProduceRequest) RecordBatch(org.apache.kafka.common.record.RecordBatch) MutableRecordBatch(org.apache.kafka.common.record.MutableRecordBatch) Node(org.apache.kafka.common.Node) AbstractRequest(org.apache.kafka.common.requests.AbstractRequest) RecordMetadata(org.apache.kafka.clients.producer.RecordMetadata) Metrics(org.apache.kafka.common.metrics.Metrics) Iterator(java.util.Iterator) MockClient(org.apache.kafka.clients.MockClient) MemoryRecords(org.apache.kafka.common.record.MemoryRecords) Test(org.junit.Test)

Example 4 with MutableRecordBatch

use of org.apache.kafka.common.record.MutableRecordBatch in project apache-kafka-on-k8s by banzaicloud.

the class SenderTest method produceRequestMatcher.

private MockClient.RequestMatcher produceRequestMatcher(final TopicPartition tp, final ProducerIdAndEpoch producerIdAndEpoch, final int sequence, final boolean isTransactional) {
    return new MockClient.RequestMatcher() {

        @Override
        public boolean matches(AbstractRequest body) {
            if (!(body instanceof ProduceRequest))
                return false;
            ProduceRequest request = (ProduceRequest) body;
            Map<TopicPartition, MemoryRecords> recordsMap = request.partitionRecordsOrFail();
            MemoryRecords records = recordsMap.get(tp);
            if (records == null)
                return false;
            List<MutableRecordBatch> batches = TestUtils.toList(records.batches());
            if (batches.isEmpty() || batches.size() > 1)
                return false;
            MutableRecordBatch batch = batches.get(0);
            return batch.baseOffset() == 0L && batch.baseSequence() == sequence && batch.producerId() == producerIdAndEpoch.producerId && batch.producerEpoch() == producerIdAndEpoch.epoch && batch.isTransactional() == isTransactional;
        }
    };
}
Also used : MutableRecordBatch(org.apache.kafka.common.record.MutableRecordBatch) ProduceRequest(org.apache.kafka.common.requests.ProduceRequest) TopicPartition(org.apache.kafka.common.TopicPartition) AbstractRequest(org.apache.kafka.common.requests.AbstractRequest) MemoryRecords(org.apache.kafka.common.record.MemoryRecords)

Example 5 with MutableRecordBatch

use of org.apache.kafka.common.record.MutableRecordBatch in project apache-kafka-on-k8s by banzaicloud.

the class TransactionManagerTest method produceRequestMatcher.

private MockClient.RequestMatcher produceRequestMatcher(final long pid, final short epoch) {
    return new MockClient.RequestMatcher() {

        @Override
        public boolean matches(AbstractRequest body) {
            ProduceRequest produceRequest = (ProduceRequest) body;
            MemoryRecords records = produceRequest.partitionRecordsOrFail().get(tp0);
            assertNotNull(records);
            Iterator<MutableRecordBatch> batchIterator = records.batches().iterator();
            assertTrue(batchIterator.hasNext());
            MutableRecordBatch batch = batchIterator.next();
            assertFalse(batchIterator.hasNext());
            assertTrue(batch.isTransactional());
            assertEquals(pid, batch.producerId());
            assertEquals(epoch, batch.producerEpoch());
            assertEquals(transactionalId, produceRequest.transactionalId());
            return true;
        }
    };
}
Also used : MutableRecordBatch(org.apache.kafka.common.record.MutableRecordBatch) ProduceRequest(org.apache.kafka.common.requests.ProduceRequest) AbstractRequest(org.apache.kafka.common.requests.AbstractRequest) MemoryRecords(org.apache.kafka.common.record.MemoryRecords)

Aggregations

MutableRecordBatch (org.apache.kafka.common.record.MutableRecordBatch)15 MemoryRecords (org.apache.kafka.common.record.MemoryRecords)9 Record (org.apache.kafka.common.record.Record)7 RecordBatch (org.apache.kafka.common.record.RecordBatch)5 ProduceRequest (org.apache.kafka.common.requests.ProduceRequest)5 DefaultRecord (org.apache.kafka.common.record.DefaultRecord)4 AbstractRequest (org.apache.kafka.common.requests.AbstractRequest)3 ArrayDeque (java.util.ArrayDeque)2 ApiVersions (org.apache.kafka.clients.ApiVersions)2 NodeApiVersions (org.apache.kafka.clients.NodeApiVersions)2 RecordMetadata (org.apache.kafka.clients.producer.RecordMetadata)2 RecordBatchTooLargeException (org.apache.kafka.common.errors.RecordBatchTooLargeException)2 Metrics (org.apache.kafka.common.metrics.Metrics)2 Test (org.junit.jupiter.api.Test)2 ByteBuffer (java.nio.ByteBuffer)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Iterator (java.util.Iterator)1 List (java.util.List)1 Collectors (java.util.stream.Collectors)1