Examples with BatchRetentionResult org.apache.kafka.common.record.MemoryRecords.RecordFilter.BatchRetentionResult used on opensource projects

Example 1 with BatchRetentionResult

use of org.apache.kafka.common.record.MemoryRecords.RecordFilter.BatchRetentionResult in project kafka by apache.

the class MemoryRecords method filterTo.

/**
 * Note: This method is also used to convert the first timestamp of the batch (which is usually the timestamp of the first record)
 * to the delete horizon of the tombstones or txn markers which are present in the batch.
 */
private static FilterResult filterTo(TopicPartition partition, Iterable<MutableRecordBatch> batches, RecordFilter filter, ByteBuffer destinationBuffer, int maxRecordBatchSize, BufferSupplier decompressionBufferSupplier) {
    FilterResult filterResult = new FilterResult(destinationBuffer);
    ByteBufferOutputStream bufferOutputStream = new ByteBufferOutputStream(destinationBuffer);
    for (MutableRecordBatch batch : batches) {
        final BatchRetentionResult batchRetentionResult = filter.checkBatchRetention(batch);
        final boolean containsMarkerForEmptyTxn = batchRetentionResult.containsMarkerForEmptyTxn;
        final BatchRetention batchRetention = batchRetentionResult.batchRetention;
        filterResult.bytesRead += batch.sizeInBytes();
        if (batchRetention == BatchRetention.DELETE)
            continue;
        // We use the absolute offset to decide whether to retain the message or not. Due to KAFKA-4298, we have to
        // allow for the possibility that a previous version corrupted the log by writing a compressed record batch
        // with a magic value not matching the magic of the records (magic < 2). This will be fixed as we
        // recopy the messages to the destination buffer.
        byte batchMagic = batch.magic();
        List<Record> retainedRecords = new ArrayList<>();
        final BatchFilterResult iterationResult = filterBatch(batch, decompressionBufferSupplier, filterResult, filter, batchMagic, true, retainedRecords);
        boolean containsTombstones = iterationResult.containsTombstones;
        boolean writeOriginalBatch = iterationResult.writeOriginalBatch;
        long maxOffset = iterationResult.maxOffset;
        if (!retainedRecords.isEmpty()) {
            // we check if the delete horizon should be set to a new value
            // in which case, we need to reset the base timestamp and overwrite the timestamp deltas
            // if the batch does not contain tombstones, then we don't need to overwrite batch
            boolean needToSetDeleteHorizon = batch.magic() >= RecordBatch.MAGIC_VALUE_V2 && (containsTombstones || containsMarkerForEmptyTxn) && !batch.deleteHorizonMs().isPresent();
            if (writeOriginalBatch && !needToSetDeleteHorizon) {
                batch.writeTo(bufferOutputStream);
                filterResult.updateRetainedBatchMetadata(batch, retainedRecords.size(), false);
            } else {
                final MemoryRecordsBuilder builder;
                long deleteHorizonMs;
                if (needToSetDeleteHorizon)
                    deleteHorizonMs = filter.currentTime + filter.deleteRetentionMs;
                else
                    deleteHorizonMs = batch.deleteHorizonMs().orElse(RecordBatch.NO_TIMESTAMP);
                builder = buildRetainedRecordsInto(batch, retainedRecords, bufferOutputStream, deleteHorizonMs);
                MemoryRecords records = builder.build();
                int filteredBatchSize = records.sizeInBytes();
                if (filteredBatchSize > batch.sizeInBytes() && filteredBatchSize > maxRecordBatchSize)
                    log.warn("Record batch from {} with last offset {} exceeded max record batch size {} after cleaning " + "(new size is {}). Consumers with version earlier than 0.10.1.0 may need to " + "increase their fetch sizes.", partition, batch.lastOffset(), maxRecordBatchSize, filteredBatchSize);
                MemoryRecordsBuilder.RecordsInfo info = builder.info();
                filterResult.updateRetainedBatchMetadata(info.maxTimestamp, info.shallowOffsetOfMaxTimestamp, maxOffset, retainedRecords.size(), filteredBatchSize);
            }
        } else if (batchRetention == BatchRetention.RETAIN_EMPTY) {
            if (batchMagic < RecordBatch.MAGIC_VALUE_V2)
                throw new IllegalStateException("Empty batches are only supported for magic v2 and above");
            bufferOutputStream.ensureRemaining(DefaultRecordBatch.RECORD_BATCH_OVERHEAD);
            DefaultRecordBatch.writeEmptyHeader(bufferOutputStream.buffer(), batchMagic, batch.producerId(), batch.producerEpoch(), batch.baseSequence(), batch.baseOffset(), batch.lastOffset(), batch.partitionLeaderEpoch(), batch.timestampType(), batch.maxTimestamp(), batch.isTransactional(), batch.isControlBatch());
            filterResult.updateRetainedBatchMetadata(batch, 0, true);
        }
        // If we had to allocate a new buffer to fit the filtered buffer (see KAFKA-5316), return early to
        // avoid the need for additional allocations.
        ByteBuffer outputBuffer = bufferOutputStream.buffer();
        if (outputBuffer != destinationBuffer) {
            filterResult.outputBuffer = outputBuffer;
            return filterResult;
        }
    }
    return filterResult;
}