Examples with BatchRetention org.apache.kafka.common.record.MemoryRecords.RecordFilter.BatchRetention used on opensource projects

Example 1 with BatchRetention

use of org.apache.kafka.common.record.MemoryRecords.RecordFilter.BatchRetention in project apache-kafka-on-k8s by banzaicloud.

the class MemoryRecords method filterTo.

private static FilterResult filterTo(TopicPartition partition, Iterable<MutableRecordBatch> batches, RecordFilter filter, ByteBuffer destinationBuffer, int maxRecordBatchSize, BufferSupplier decompressionBufferSupplier) {
    long maxTimestamp = RecordBatch.NO_TIMESTAMP;
    long maxOffset = -1L;
    long shallowOffsetOfMaxTimestamp = -1L;
    int messagesRead = 0;
    int bytesRead = 0;
    int messagesRetained = 0;
    int bytesRetained = 0;
    ByteBufferOutputStream bufferOutputStream = new ByteBufferOutputStream(destinationBuffer);
    for (MutableRecordBatch batch : batches) {
        bytesRead += batch.sizeInBytes();
        BatchRetention batchRetention = filter.checkBatchRetention(batch);
        if (batchRetention == BatchRetention.DELETE)
            continue;
        // We use the absolute offset to decide whether to retain the message or not. Due to KAFKA-4298, we have to
        // allow for the possibility that a previous version corrupted the log by writing a compressed record batch
        // with a magic value not matching the magic of the records (magic < 2). This will be fixed as we
        // recopy the messages to the destination buffer.
        byte batchMagic = batch.magic();
        boolean writeOriginalBatch = true;
        List<Record> retainedRecords = new ArrayList<>();
        try (final CloseableIterator<Record> iterator = batch.streamingIterator(decompressionBufferSupplier)) {
            while (iterator.hasNext()) {
                Record record = iterator.next();
                messagesRead += 1;
                if (filter.shouldRetainRecord(batch, record)) {
                    // the corrupted batch with correct data.
                    if (!record.hasMagic(batchMagic))
                        writeOriginalBatch = false;
                    if (record.offset() > maxOffset)
                        maxOffset = record.offset();
                    retainedRecords.add(record);
                } else {
                    writeOriginalBatch = false;
                }
            }
        }
        if (!retainedRecords.isEmpty()) {
            if (writeOriginalBatch) {
                batch.writeTo(bufferOutputStream);
                messagesRetained += retainedRecords.size();
                bytesRetained += batch.sizeInBytes();
                if (batch.maxTimestamp() > maxTimestamp) {
                    maxTimestamp = batch.maxTimestamp();
                    shallowOffsetOfMaxTimestamp = batch.lastOffset();
                }
            } else {
                MemoryRecordsBuilder builder = buildRetainedRecordsInto(batch, retainedRecords, bufferOutputStream);
                MemoryRecords records = builder.build();
                int filteredBatchSize = records.sizeInBytes();
                messagesRetained += retainedRecords.size();
                bytesRetained += filteredBatchSize;
                if (filteredBatchSize > batch.sizeInBytes() && filteredBatchSize > maxRecordBatchSize)
                    log.warn("Record batch from {} with last offset {} exceeded max record batch size {} after cleaning " + "(new size is {}). Consumers with version earlier than 0.10.1.0 may need to " + "increase their fetch sizes.", partition, batch.lastOffset(), maxRecordBatchSize, filteredBatchSize);
                MemoryRecordsBuilder.RecordsInfo info = builder.info();
                if (info.maxTimestamp > maxTimestamp) {
                    maxTimestamp = info.maxTimestamp;
                    shallowOffsetOfMaxTimestamp = info.shallowOffsetOfMaxTimestamp;
                }
            }
        } else if (batchRetention == BatchRetention.RETAIN_EMPTY) {
            if (batchMagic < RecordBatch.MAGIC_VALUE_V2)
                throw new IllegalStateException("Empty batches are only supported for magic v2 and above");
            bufferOutputStream.ensureRemaining(DefaultRecordBatch.RECORD_BATCH_OVERHEAD);
            DefaultRecordBatch.writeEmptyHeader(bufferOutputStream.buffer(), batchMagic, batch.producerId(), batch.producerEpoch(), batch.baseSequence(), batch.baseOffset(), batch.lastOffset(), batch.partitionLeaderEpoch(), batch.timestampType(), batch.maxTimestamp(), batch.isTransactional(), batch.isControlBatch());
        }
        // If we had to allocate a new buffer to fit the filtered output (see KAFKA-5316), return early to
        // avoid the need for additional allocations.
        ByteBuffer outputBuffer = bufferOutputStream.buffer();
        if (outputBuffer != destinationBuffer)
            return new FilterResult(outputBuffer, messagesRead, bytesRead, messagesRetained, bytesRetained, maxOffset, maxTimestamp, shallowOffsetOfMaxTimestamp);
    }
    return new FilterResult(destinationBuffer, messagesRead, bytesRead, messagesRetained, bytesRetained, maxOffset, maxTimestamp, shallowOffsetOfMaxTimestamp);
}

Example 2 with BatchRetention

use of org.apache.kafka.common.record.MemoryRecords.RecordFilter.BatchRetention in project kafka by apache.

the class MemoryRecordsTest method testEmptyBatchDeletion.

@Test
public void testEmptyBatchDeletion() {
    for (final BatchRetention deleteRetention : Arrays.asList(BatchRetention.DELETE, BatchRetention.DELETE_EMPTY)) {
        ByteBuffer buffer = ByteBuffer.allocate(DefaultRecordBatch.RECORD_BATCH_OVERHEAD);
        long producerId = 23L;
        short producerEpoch = 5;
        long baseOffset = 3L;
        int baseSequence = 10;
        int partitionLeaderEpoch = 293;
        long timestamp = System.currentTimeMillis();
        DefaultRecordBatch.writeEmptyHeader(buffer, RecordBatch.MAGIC_VALUE_V2, producerId, producerEpoch, baseSequence, baseOffset, baseOffset, partitionLeaderEpoch, TimestampType.CREATE_TIME, timestamp, false, false);
        buffer.flip();
        ByteBuffer filtered = ByteBuffer.allocate(2048);
        MemoryRecords records = MemoryRecords.readableRecords(buffer);
        MemoryRecords.FilterResult filterResult = records.filterTo(new TopicPartition("foo", 0), new MemoryRecords.RecordFilter(0, 0) {

            @Override
            protected BatchRetentionResult checkBatchRetention(RecordBatch batch) {
                return new BatchRetentionResult(deleteRetention, false);
            }

            @Override
            protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) {
                return false;
            }
        }, filtered, Integer.MAX_VALUE, BufferSupplier.NO_CACHING);
        // Verify filter result
        assertEquals(0, filterResult.outputBuffer().position());
        // Verify filtered records
        filtered.flip();
        MemoryRecords filteredRecords = MemoryRecords.readableRecords(filtered);
        assertEquals(0, filteredRecords.sizeInBytes());
    }
}

Example 3 with BatchRetention

use of org.apache.kafka.common.record.MemoryRecords.RecordFilter.BatchRetention in project apache-kafka-on-k8s by banzaicloud.

the class MemoryRecordsTest method testFilterToBatchDiscard.

@Test
public void testFilterToBatchDiscard() {
    if (compression != CompressionType.NONE || magic >= RecordBatch.MAGIC_VALUE_V2) {
        ByteBuffer buffer = ByteBuffer.allocate(2048);
        MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, magic, compression, TimestampType.CREATE_TIME, 0L);
        builder.append(10L, "1".getBytes(), "a".getBytes());
        builder.close();
        builder = MemoryRecords.builder(buffer, magic, compression, TimestampType.CREATE_TIME, 1L);
        builder.append(11L, "2".getBytes(), "b".getBytes());
        builder.append(12L, "3".getBytes(), "c".getBytes());
        builder.close();
        builder = MemoryRecords.builder(buffer, magic, compression, TimestampType.CREATE_TIME, 3L);
        builder.append(13L, "4".getBytes(), "d".getBytes());
        builder.append(20L, "5".getBytes(), "e".getBytes());
        builder.append(15L, "6".getBytes(), "f".getBytes());
        builder.close();
        builder = MemoryRecords.builder(buffer, magic, compression, TimestampType.CREATE_TIME, 6L);
        builder.append(16L, "7".getBytes(), "g".getBytes());
        builder.close();
        buffer.flip();
        ByteBuffer filtered = ByteBuffer.allocate(2048);
        MemoryRecords.readableRecords(buffer).filterTo(new TopicPartition("foo", 0), new MemoryRecords.RecordFilter() {

            @Override
            protected BatchRetention checkBatchRetention(RecordBatch batch) {
                // discard the second and fourth batches
                if (batch.lastOffset() == 2L || batch.lastOffset() == 6L)
                    return BatchRetention.DELETE;
                return BatchRetention.DELETE_EMPTY;
            }

            @Override
            protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) {
                return true;
            }
        }, filtered, Integer.MAX_VALUE, BufferSupplier.NO_CACHING);
        filtered.flip();
        MemoryRecords filteredRecords = MemoryRecords.readableRecords(filtered);
        List<MutableRecordBatch> batches = TestUtils.toList(filteredRecords.batches());
        assertEquals(2, batches.size());
        assertEquals(0L, batches.get(0).lastOffset());
        assertEquals(5L, batches.get(1).lastOffset());
    }
}

Example 4 with BatchRetention

use of org.apache.kafka.common.record.MemoryRecords.RecordFilter.BatchRetention in project apache-kafka-on-k8s by banzaicloud.

the class MemoryRecordsTest method testFilterToEmptyBatchRetention.

@Test
public void testFilterToEmptyBatchRetention() {
    if (magic >= RecordBatch.MAGIC_VALUE_V2) {
        for (boolean isTransactional : Arrays.asList(true, false)) {
            ByteBuffer buffer = ByteBuffer.allocate(2048);
            long producerId = 23L;
            short producerEpoch = 5;
            long baseOffset = 3L;
            int baseSequence = 10;
            int partitionLeaderEpoch = 293;
            MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, magic, compression, TimestampType.CREATE_TIME, baseOffset, RecordBatch.NO_TIMESTAMP, producerId, producerEpoch, baseSequence, isTransactional, partitionLeaderEpoch);
            builder.append(11L, "2".getBytes(), "b".getBytes());
            builder.append(12L, "3".getBytes(), "c".getBytes());
            builder.close();
            ByteBuffer filtered = ByteBuffer.allocate(2048);
            builder.build().filterTo(new TopicPartition("foo", 0), new MemoryRecords.RecordFilter() {

                @Override
                protected BatchRetention checkBatchRetention(RecordBatch batch) {
                    // retain all batches
                    return BatchRetention.RETAIN_EMPTY;
                }

                @Override
                protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) {
                    // delete the records
                    return false;
                }
            }, filtered, Integer.MAX_VALUE, BufferSupplier.NO_CACHING);
            filtered.flip();
            MemoryRecords filteredRecords = MemoryRecords.readableRecords(filtered);
            List<MutableRecordBatch> batches = TestUtils.toList(filteredRecords.batches());
            assertEquals(1, batches.size());
            MutableRecordBatch batch = batches.get(0);
            assertEquals(0, batch.countOrNull().intValue());
            assertEquals(12L, batch.maxTimestamp());
            assertEquals(TimestampType.CREATE_TIME, batch.timestampType());
            assertEquals(baseOffset, batch.baseOffset());
            assertEquals(baseOffset + 1, batch.lastOffset());
            assertEquals(baseSequence, batch.baseSequence());
            assertEquals(baseSequence + 1, batch.lastSequence());
            assertEquals(isTransactional, batch.isTransactional());
        }
    }
}

Example 5 with BatchRetention

use of org.apache.kafka.common.record.MemoryRecords.RecordFilter.BatchRetention in project apache-kafka-on-k8s by banzaicloud.

the class MemoryRecordsTest method testEmptyBatchDeletion.

@Test
public void testEmptyBatchDeletion() {
    if (magic >= RecordBatch.MAGIC_VALUE_V2) {
        for (final BatchRetention deleteRetention : Arrays.asList(BatchRetention.DELETE, BatchRetention.DELETE_EMPTY)) {
            ByteBuffer buffer = ByteBuffer.allocate(DefaultRecordBatch.RECORD_BATCH_OVERHEAD);
            long producerId = 23L;
            short producerEpoch = 5;
            long baseOffset = 3L;
            int baseSequence = 10;
            int partitionLeaderEpoch = 293;
            DefaultRecordBatch.writeEmptyHeader(buffer, RecordBatch.MAGIC_VALUE_V2, producerId, producerEpoch, baseSequence, baseOffset, baseOffset, partitionLeaderEpoch, TimestampType.CREATE_TIME, System.currentTimeMillis(), false, false);
            buffer.flip();
            ByteBuffer filtered = ByteBuffer.allocate(2048);
            MemoryRecords.readableRecords(buffer).filterTo(new TopicPartition("foo", 0), new MemoryRecords.RecordFilter() {

                @Override
                protected BatchRetention checkBatchRetention(RecordBatch batch) {
                    return deleteRetention;
                }

                @Override
                protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) {
                    return false;
                }
            }, filtered, Integer.MAX_VALUE, BufferSupplier.NO_CACHING);
            filtered.flip();
            MemoryRecords filteredRecords = MemoryRecords.readableRecords(filtered);
            assertEquals(0, filteredRecords.sizeInBytes());
        }
    }
}