Search in sources :

Example 6 with RecordBatch

use of org.apache.kafka.common.record.RecordBatch in project apache-kafka-on-k8s by banzaicloud.

the class FetcherTest method testUpdatePositionWithLastRecordMissingFromBatch.

@Test
public void testUpdatePositionWithLastRecordMissingFromBatch() {
    MemoryRecords records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("0".getBytes(), "v".getBytes()), new SimpleRecord("1".getBytes(), "v".getBytes()), new SimpleRecord("2".getBytes(), "v".getBytes()), new SimpleRecord(null, "value".getBytes()));
    // Remove the last record to simulate compaction
    MemoryRecords.FilterResult result = records.filterTo(tp0, new MemoryRecords.RecordFilter() {

        @Override
        protected BatchRetention checkBatchRetention(RecordBatch batch) {
            return BatchRetention.DELETE_EMPTY;
        }

        @Override
        protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) {
            return record.key() != null;
        }
    }, ByteBuffer.allocate(1024), Integer.MAX_VALUE, BufferSupplier.NO_CACHING);
    result.output.flip();
    MemoryRecords compactedRecords = MemoryRecords.readableRecords(result.output);
    subscriptions.assignFromUser(singleton(tp0));
    subscriptions.seek(tp0, 0);
    assertEquals(1, fetcher.sendFetches());
    client.prepareResponse(fullFetchResponse(tp0, compactedRecords, Errors.NONE, 100L, 0));
    consumerClient.poll(0);
    assertTrue(fetcher.hasCompletedFetches());
    Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> allFetchedRecords = fetcher.fetchedRecords();
    assertTrue(allFetchedRecords.containsKey(tp0));
    List<ConsumerRecord<byte[], byte[]>> fetchedRecords = allFetchedRecords.get(tp0);
    assertEquals(3, fetchedRecords.size());
    for (int i = 0; i < 3; i++) {
        assertEquals(Integer.toString(i), new String(fetchedRecords.get(i).key()));
    }
    // The next offset should point to the next batch
    assertEquals(4L, subscriptions.position(tp0).longValue());
}
Also used : DefaultRecordBatch(org.apache.kafka.common.record.DefaultRecordBatch) RecordBatch(org.apache.kafka.common.record.RecordBatch) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) TopicPartition(org.apache.kafka.common.TopicPartition) SimpleRecord(org.apache.kafka.common.record.SimpleRecord) Record(org.apache.kafka.common.record.Record) LegacyRecord(org.apache.kafka.common.record.LegacyRecord) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) SimpleRecord(org.apache.kafka.common.record.SimpleRecord) List(java.util.List) ArrayList(java.util.ArrayList) MemoryRecords(org.apache.kafka.common.record.MemoryRecords) Test(org.junit.Test)

Aggregations

RecordBatch (org.apache.kafka.common.record.RecordBatch)6 TopicPartition (org.apache.kafka.common.TopicPartition)4 Test (org.junit.Test)4 MemoryRecords (org.apache.kafka.common.record.MemoryRecords)3 Record (org.apache.kafka.common.record.Record)3 Iterator (java.util.Iterator)2 CompressionType (org.apache.kafka.common.record.CompressionType)2 LegacyRecord (org.apache.kafka.common.record.LegacyRecord)2 MemoryRecordsBuilder (org.apache.kafka.common.record.MemoryRecordsBuilder)2 MutableRecordBatch (org.apache.kafka.common.record.MutableRecordBatch)2 ArrayDeque (java.util.ArrayDeque)1 ArrayList (java.util.ArrayList)1 HashMap (java.util.HashMap)1 LinkedHashMap (java.util.LinkedHashMap)1 List (java.util.List)1 Map (java.util.Map)1 MockClient (org.apache.kafka.clients.MockClient)1 ConsumerRecord (org.apache.kafka.clients.consumer.ConsumerRecord)1 OffsetOutOfRangeException (org.apache.kafka.clients.consumer.OffsetOutOfRangeException)1 RecordMetadata (org.apache.kafka.clients.producer.RecordMetadata)1