Examples with RecordBatch org.apache.kafka.common.record.RecordBatch used on opensource projects

Example 1 with RecordBatch

use of org.apache.kafka.common.record.RecordBatch in project apache-kafka-on-k8s by banzaicloud.

the class SenderTest method testSequenceNumberIncrement.

@Test
public void testSequenceNumberIncrement() throws InterruptedException {
    final long producerId = 343434L;
    TransactionManager transactionManager = new TransactionManager();
    transactionManager.setProducerIdAndEpoch(new ProducerIdAndEpoch(producerId, (short) 0));
    setupWithTransactionState(transactionManager);
    client.setNode(new Node(1, "localhost", 33343));
    int maxRetries = 10;
    Metrics m = new Metrics();
    SenderMetricsRegistry senderMetrics = new SenderMetricsRegistry(m);
    Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries, senderMetrics, time, REQUEST_TIMEOUT, 50, transactionManager, apiVersions);
    Future<RecordMetadata> responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
    client.prepareResponse(new MockClient.RequestMatcher() {

        @Override
        public boolean matches(AbstractRequest body) {
            if (body instanceof ProduceRequest) {
                ProduceRequest request = (ProduceRequest) body;
                MemoryRecords records = request.partitionRecordsOrFail().get(tp0);
                Iterator<MutableRecordBatch> batchIterator = records.batches().iterator();
                assertTrue(batchIterator.hasNext());
                RecordBatch batch = batchIterator.next();
                assertFalse(batchIterator.hasNext());
                assertEquals(0, batch.baseSequence());
                assertEquals(producerId, batch.producerId());
                assertEquals(0, batch.producerEpoch());
                return true;
            }
            return false;
        }
    }, produceResponse(tp0, 0, Errors.NONE, 0));
    // connect.
    sender.run(time.milliseconds());
    // send.
    sender.run(time.milliseconds());
    // receive response
    sender.run(time.milliseconds());
    assertTrue(responseFuture.isDone());
    assertEquals(0L, (long) transactionManager.lastAckedSequence(tp0));
    assertEquals(1L, (long) transactionManager.sequenceNumber(tp0));
}

Example 2 with RecordBatch

use of org.apache.kafka.common.record.RecordBatch in project apache-kafka-on-k8s by banzaicloud.

the class Fetcher method parseCompletedFetch.

/**
 * The callback for fetch completion
 */
private PartitionRecords parseCompletedFetch(CompletedFetch completedFetch) {
    TopicPartition tp = completedFetch.partition;
    FetchResponse.PartitionData partition = completedFetch.partitionData;
    long fetchOffset = completedFetch.fetchedOffset;
    PartitionRecords partitionRecords = null;
    Errors error = partition.error;
    try {
        if (!subscriptions.isFetchable(tp)) {
            // this can happen when a rebalance happened or a partition consumption paused
            // while fetch is still in-flight
            log.debug("Ignoring fetched records for partition {} since it is no longer fetchable", tp);
        } else if (error == Errors.NONE) {
            // we are interested in this fetch only if the beginning offset matches the
            // current consumed position
            Long position = subscriptions.position(tp);
            if (position == null || position != fetchOffset) {
                log.debug("Discarding stale fetch response for partition {} since its offset {} does not match " + "the expected offset {}", tp, fetchOffset, position);
                return null;
            }
            log.trace("Preparing to read {} bytes of data for partition {} with offset {}", partition.records.sizeInBytes(), tp, position);
            Iterator<? extends RecordBatch> batches = partition.records.batches().iterator();
            partitionRecords = new PartitionRecords(tp, completedFetch, batches);
            if (!batches.hasNext() && partition.records.sizeInBytes() > 0) {
                if (completedFetch.responseVersion < 3) {
                    // Implement the pre KIP-74 behavior of throwing a RecordTooLargeException.
                    Map<TopicPartition, Long> recordTooLargePartitions = Collections.singletonMap(tp, fetchOffset);
                    throw new RecordTooLargeException("There are some messages at [Partition=Offset]: " + recordTooLargePartitions + " whose size is larger than the fetch size " + this.fetchSize + " and hence cannot be returned. Please considering upgrading your broker to 0.10.1.0 or " + "newer to avoid this issue. Alternately, increase the fetch size on the client (using " + ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG + ")", recordTooLargePartitions);
                } else {
                    // This should not happen with brokers that support FetchRequest/Response V3 or higher (i.e. KIP-74)
                    throw new KafkaException("Failed to make progress reading messages at " + tp + "=" + fetchOffset + ". Received a non-empty fetch response from the server, but no " + "complete records were found.");
                }
            }
            if (partition.highWatermark >= 0) {
                log.trace("Updating high watermark for partition {} to {}", tp, partition.highWatermark);
                subscriptions.updateHighWatermark(tp, partition.highWatermark);
            }
            if (partition.logStartOffset >= 0) {
                log.trace("Updating log start offset for partition {} to {}", tp, partition.logStartOffset);
                subscriptions.updateLogStartOffset(tp, partition.logStartOffset);
            }
            if (partition.lastStableOffset >= 0) {
                log.trace("Updating last stable offset for partition {} to {}", tp, partition.lastStableOffset);
                subscriptions.updateLastStableOffset(tp, partition.lastStableOffset);
            }
        } else if (error == Errors.NOT_LEADER_FOR_PARTITION || error == Errors.KAFKA_STORAGE_ERROR) {
            log.debug("Error in fetch for partition {}: {}", tp, error.exceptionName());
            this.metadata.requestUpdate();
        } else if (error == Errors.UNKNOWN_TOPIC_OR_PARTITION) {
            log.warn("Received unknown topic or partition error in fetch for partition {}", tp);
            this.metadata.requestUpdate();
        } else if (error == Errors.OFFSET_OUT_OF_RANGE) {
            if (fetchOffset != subscriptions.position(tp)) {
                log.debug("Discarding stale fetch response for partition {} since the fetched offset {}" + "does not match the current offset {}", tp, fetchOffset, subscriptions.position(tp));
            } else if (subscriptions.hasDefaultOffsetResetPolicy()) {
                log.info("Fetch offset {} is out of range for partition {}, resetting offset", fetchOffset, tp);
                subscriptions.requestOffsetReset(tp);
            } else {
                throw new OffsetOutOfRangeException(Collections.singletonMap(tp, fetchOffset));
            }
        } else if (error == Errors.TOPIC_AUTHORIZATION_FAILED) {
            log.warn("Not authorized to read from topic {}.", tp.topic());
            throw new TopicAuthorizationException(Collections.singleton(tp.topic()));
        } else if (error == Errors.UNKNOWN_SERVER_ERROR) {
            log.warn("Unknown error fetching data for topic-partition {}", tp);
        } else {
            throw new IllegalStateException("Unexpected error code " + error.code() + " while fetching data");
        }
    } finally {
        if (partitionRecords == null)
            completedFetch.metricAggregator.record(tp, 0, 0);
        if (error != Errors.NONE)
            // we move the partition to the end if there was an error. This way, it's more likely that partitions for
            // the same topic can remain together (allowing for more efficient serialization).
            subscriptions.movePartitionToEnd(tp);
    }
    return partitionRecords;
}

Example 3 with RecordBatch

use of org.apache.kafka.common.record.RecordBatch in project apache-kafka-on-k8s by banzaicloud.

the class ProducerBatch method split.

public Deque<ProducerBatch> split(int splitBatchSize) {
    Deque<ProducerBatch> batches = new ArrayDeque<>();
    MemoryRecords memoryRecords = recordsBuilder.build();
    Iterator<MutableRecordBatch> recordBatchIter = memoryRecords.batches().iterator();
    if (!recordBatchIter.hasNext())
        throw new IllegalStateException("Cannot split an empty producer batch.");
    RecordBatch recordBatch = recordBatchIter.next();
    if (recordBatch.magic() < MAGIC_VALUE_V2 && !recordBatch.isCompressed())
        throw new IllegalArgumentException("Batch splitting cannot be used with non-compressed messages " + "with version v0 and v1");
    if (recordBatchIter.hasNext())
        throw new IllegalArgumentException("A producer batch should only have one record batch.");
    Iterator<Thunk> thunkIter = thunks.iterator();
    // We always allocate batch size because we are already splitting a big batch.
    // And we also Retain the create time of the original batch.
    ProducerBatch batch = null;
    for (Record record : recordBatch) {
        assert thunkIter.hasNext();
        Thunk thunk = thunkIter.next();
        if (batch == null)
            batch = createBatchOffAccumulatorForRecord(record, splitBatchSize);
        // A newly created batch can always host the first message.
        if (!batch.tryAppendForSplit(record.timestamp(), record.key(), record.value(), record.headers(), thunk)) {
            batches.add(batch);
            batch = createBatchOffAccumulatorForRecord(record, splitBatchSize);
            batch.tryAppendForSplit(record.timestamp(), record.key(), record.value(), record.headers(), thunk);
        }
    }
    // Close the last batch and add it to the batch list after split.
    if (batch != null)
        batches.add(batch);
    produceFuture.set(ProduceResponse.INVALID_OFFSET, NO_TIMESTAMP, new RecordBatchTooLargeException());
    produceFuture.done();
    if (hasSequence()) {
        int sequence = baseSequence();
        ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId(), producerEpoch());
        for (ProducerBatch newBatch : batches) {
            newBatch.setProducerState(producerIdAndEpoch, sequence, isTransactional());
            sequence += newBatch.recordCount;
        }
    }
    return batches;
}

Example 4 with RecordBatch

use of org.apache.kafka.common.record.RecordBatch in project apache-kafka-on-k8s by banzaicloud.

the class ProducerBatchTest method testSplitPreservesHeaders.

@Test
public void testSplitPreservesHeaders() {
    for (CompressionType compressionType : CompressionType.values()) {
        MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), MAGIC_VALUE_V2, compressionType, TimestampType.CREATE_TIME, 0L);
        ProducerBatch batch = new ProducerBatch(new TopicPartition("topic", 1), builder, now);
        Header header = new RecordHeader("header-key", "header-value".getBytes());
        while (true) {
            FutureRecordMetadata future = batch.tryAppend(now, "hi".getBytes(), "there".getBytes(), new Header[] { header }, null, now);
            if (future == null) {
                break;
            }
        }
        Deque<ProducerBatch> batches = batch.split(200);
        assertTrue("This batch should be split to multiple small batches.", batches.size() >= 2);
        for (ProducerBatch splitProducerBatch : batches) {
            for (RecordBatch splitBatch : splitProducerBatch.records().batches()) {
                for (Record record : splitBatch) {
                    assertTrue("Header size should be 1.", record.headers().length == 1);
                    assertTrue("Header key should be 'header-key'.", record.headers()[0].key().equals("header-key"));
                    assertTrue("Header value should be 'header-value'.", new String(record.headers()[0].value()).equals("header-value"));
                }
            }
        }
    }
}

Example 5 with RecordBatch

use of org.apache.kafka.common.record.RecordBatch in project apache-kafka-on-k8s by banzaicloud.

the class ProducerBatchTest method testSplitPreservesMagicAndCompressionType.

@Test
public void testSplitPreservesMagicAndCompressionType() {
    for (byte magic : Arrays.asList(MAGIC_VALUE_V0, MAGIC_VALUE_V1, MAGIC_VALUE_V2)) {
        for (CompressionType compressionType : CompressionType.values()) {
            if (compressionType == CompressionType.NONE && magic < MAGIC_VALUE_V2)
                continue;
            MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), magic, compressionType, TimestampType.CREATE_TIME, 0L);
            ProducerBatch batch = new ProducerBatch(new TopicPartition("topic", 1), builder, now);
            while (true) {
                FutureRecordMetadata future = batch.tryAppend(now, "hi".getBytes(), "there".getBytes(), Record.EMPTY_HEADERS, null, now);
                if (future == null)
                    break;
            }
            Deque<ProducerBatch> batches = batch.split(512);
            assertTrue(batches.size() >= 2);
            for (ProducerBatch splitProducerBatch : batches) {
                assertEquals(magic, splitProducerBatch.magic());
                assertTrue(splitProducerBatch.isSplitBatch());
                for (RecordBatch splitBatch : splitProducerBatch.records().batches()) {
                    assertEquals(magic, splitBatch.magic());
                    assertEquals(0L, splitBatch.baseOffset());
                    assertEquals(compressionType, splitBatch.compressionType());
                }
            }
        }
    }
}