use of org.apache.kafka.common.record.MutableRecordBatch in project apache-kafka-on-k8s by banzaicloud.
the class RecordAccumulatorTest method testAppendLarge.
private void testAppendLarge(CompressionType compressionType) throws Exception {
int batchSize = 512;
byte[] value = new byte[2 * batchSize];
RecordAccumulator accum = createTestRecordAccumulator(batchSize + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 10 * 1024, compressionType, 0L);
accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
assertEquals("Our partition's leader should be ready", Collections.singleton(node1), accum.ready(cluster, time.milliseconds()).readyNodes);
Deque<ProducerBatch> batches = accum.batches().get(tp1);
assertEquals(1, batches.size());
ProducerBatch producerBatch = batches.peek();
List<MutableRecordBatch> recordBatches = TestUtils.toList(producerBatch.records().batches());
assertEquals(1, recordBatches.size());
MutableRecordBatch recordBatch = recordBatches.get(0);
assertEquals(0L, recordBatch.baseOffset());
List<Record> records = TestUtils.toList(recordBatch);
assertEquals(1, records.size());
Record record = records.get(0);
assertEquals(0L, record.offset());
assertEquals(ByteBuffer.wrap(key), record.key());
assertEquals(ByteBuffer.wrap(value), record.value());
assertEquals(0L, record.timestamp());
}
use of org.apache.kafka.common.record.MutableRecordBatch in project apache-kafka-on-k8s by banzaicloud.
the class RecordAccumulatorTest method testAppendLargeOldMessageFormat.
private void testAppendLargeOldMessageFormat(CompressionType compressionType) throws Exception {
int batchSize = 512;
byte[] value = new byte[2 * batchSize];
ApiVersions apiVersions = new ApiVersions();
apiVersions.update(node1.idString(), NodeApiVersions.create(Collections.singleton(new ApiVersionsResponse.ApiVersion(ApiKeys.PRODUCE.id, (short) 0, (short) 2))));
RecordAccumulator accum = createTestRecordAccumulator(batchSize + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 10 * 1024, compressionType, 0L);
accum.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs);
assertEquals("Our partition's leader should be ready", Collections.singleton(node1), accum.ready(cluster, time.milliseconds()).readyNodes);
Deque<ProducerBatch> batches = accum.batches().get(tp1);
assertEquals(1, batches.size());
ProducerBatch producerBatch = batches.peek();
List<MutableRecordBatch> recordBatches = TestUtils.toList(producerBatch.records().batches());
assertEquals(1, recordBatches.size());
MutableRecordBatch recordBatch = recordBatches.get(0);
assertEquals(0L, recordBatch.baseOffset());
List<Record> records = TestUtils.toList(recordBatch);
assertEquals(1, records.size());
Record record = records.get(0);
assertEquals(0L, record.offset());
assertEquals(ByteBuffer.wrap(key), record.key());
assertEquals(ByteBuffer.wrap(value), record.value());
assertEquals(0L, record.timestamp());
}
use of org.apache.kafka.common.record.MutableRecordBatch in project apache-kafka-on-k8s by banzaicloud.
the class SenderTest method testSequenceNumberIncrement.
@Test
public void testSequenceNumberIncrement() throws InterruptedException {
final long producerId = 343434L;
TransactionManager transactionManager = new TransactionManager();
transactionManager.setProducerIdAndEpoch(new ProducerIdAndEpoch(producerId, (short) 0));
setupWithTransactionState(transactionManager);
client.setNode(new Node(1, "localhost", 33343));
int maxRetries = 10;
Metrics m = new Metrics();
SenderMetricsRegistry senderMetrics = new SenderMetricsRegistry(m);
Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries, senderMetrics, time, REQUEST_TIMEOUT, 50, transactionManager, apiVersions);
Future<RecordMetadata> responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
client.prepareResponse(new MockClient.RequestMatcher() {
@Override
public boolean matches(AbstractRequest body) {
if (body instanceof ProduceRequest) {
ProduceRequest request = (ProduceRequest) body;
MemoryRecords records = request.partitionRecordsOrFail().get(tp0);
Iterator<MutableRecordBatch> batchIterator = records.batches().iterator();
assertTrue(batchIterator.hasNext());
RecordBatch batch = batchIterator.next();
assertFalse(batchIterator.hasNext());
assertEquals(0, batch.baseSequence());
assertEquals(producerId, batch.producerId());
assertEquals(0, batch.producerEpoch());
return true;
}
return false;
}
}, produceResponse(tp0, 0, Errors.NONE, 0));
// connect.
sender.run(time.milliseconds());
// send.
sender.run(time.milliseconds());
// receive response
sender.run(time.milliseconds());
assertTrue(responseFuture.isDone());
assertEquals(0L, (long) transactionManager.lastAckedSequence(tp0));
assertEquals(1L, (long) transactionManager.sequenceNumber(tp0));
}
use of org.apache.kafka.common.record.MutableRecordBatch in project apache-kafka-on-k8s by banzaicloud.
the class SenderTest method produceRequestMatcher.
private MockClient.RequestMatcher produceRequestMatcher(final TopicPartition tp, final ProducerIdAndEpoch producerIdAndEpoch, final int sequence, final boolean isTransactional) {
return new MockClient.RequestMatcher() {
@Override
public boolean matches(AbstractRequest body) {
if (!(body instanceof ProduceRequest))
return false;
ProduceRequest request = (ProduceRequest) body;
Map<TopicPartition, MemoryRecords> recordsMap = request.partitionRecordsOrFail();
MemoryRecords records = recordsMap.get(tp);
if (records == null)
return false;
List<MutableRecordBatch> batches = TestUtils.toList(records.batches());
if (batches.isEmpty() || batches.size() > 1)
return false;
MutableRecordBatch batch = batches.get(0);
return batch.baseOffset() == 0L && batch.baseSequence() == sequence && batch.producerId() == producerIdAndEpoch.producerId && batch.producerEpoch() == producerIdAndEpoch.epoch && batch.isTransactional() == isTransactional;
}
};
}
use of org.apache.kafka.common.record.MutableRecordBatch in project apache-kafka-on-k8s by banzaicloud.
the class TransactionManagerTest method produceRequestMatcher.
private MockClient.RequestMatcher produceRequestMatcher(final long pid, final short epoch) {
return new MockClient.RequestMatcher() {
@Override
public boolean matches(AbstractRequest body) {
ProduceRequest produceRequest = (ProduceRequest) body;
MemoryRecords records = produceRequest.partitionRecordsOrFail().get(tp0);
assertNotNull(records);
Iterator<MutableRecordBatch> batchIterator = records.batches().iterator();
assertTrue(batchIterator.hasNext());
MutableRecordBatch batch = batchIterator.next();
assertFalse(batchIterator.hasNext());
assertTrue(batch.isTransactional());
assertEquals(pid, batch.producerId());
assertEquals(epoch, batch.producerEpoch());
assertEquals(transactionalId, produceRequest.transactionalId());
return true;
}
};
}
Aggregations