Search in sources :

Example 1 with Records

use of org.apache.kafka.common.record.Records in project kafka by apache.

the class FetchResponse method addPartitionData.

private static void addPartitionData(String dest, List<Send> sends, Struct partitionData) {
    Struct header = partitionData.getStruct(PARTITION_HEADER_KEY_NAME);
    Records records = partitionData.getRecords(RECORD_SET_KEY_NAME);
    // include the partition header and the size of the record set
    ByteBuffer buffer = ByteBuffer.allocate(header.sizeOf() + 4);
    header.writeTo(buffer);
    buffer.putInt(records.sizeInBytes());
    buffer.rewind();
    sends.add(new ByteBufferSend(dest, buffer));
    // finally the send for the record set itself
    sends.add(new RecordsSend(dest, records));
}
Also used : ByteBufferSend(org.apache.kafka.common.network.ByteBufferSend) Records(org.apache.kafka.common.record.Records) ByteBuffer(java.nio.ByteBuffer) Struct(org.apache.kafka.common.protocol.types.Struct)

Example 2 with Records

use of org.apache.kafka.common.record.Records in project apache-kafka-on-k8s by banzaicloud.

the class FetchResponse method addPartitionData.

private static void addPartitionData(String dest, Queue<Send> sends, Struct partitionData) {
    Struct header = partitionData.getStruct(PARTITION_HEADER_KEY_NAME);
    Records records = partitionData.getRecords(RECORD_SET_KEY_NAME);
    // include the partition header and the size of the record set
    ByteBuffer buffer = ByteBuffer.allocate(header.sizeOf() + 4);
    header.writeTo(buffer);
    buffer.putInt(records.sizeInBytes());
    buffer.rewind();
    sends.add(new ByteBufferSend(dest, buffer));
    // finally the send for the record set itself
    sends.add(new RecordsSend(dest, records));
}
Also used : ByteBufferSend(org.apache.kafka.common.network.ByteBufferSend) Records(org.apache.kafka.common.record.Records) ByteBuffer(java.nio.ByteBuffer) Struct(org.apache.kafka.common.protocol.types.Struct)

Example 3 with Records

use of org.apache.kafka.common.record.Records in project kafka by apache.

the class FetcherTest method testFetchResponseMetrics.

@Test
public void testFetchResponseMetrics() {
    buildFetcher();
    String topic1 = "foo";
    String topic2 = "bar";
    TopicPartition tp1 = new TopicPartition(topic1, 0);
    TopicPartition tp2 = new TopicPartition(topic2, 0);
    subscriptions.assignFromUser(mkSet(tp1, tp2));
    Map<String, Integer> partitionCounts = new HashMap<>();
    partitionCounts.put(topic1, 1);
    partitionCounts.put(topic2, 1);
    topicIds.put(topic1, Uuid.randomUuid());
    topicIds.put(topic2, Uuid.randomUuid());
    TopicIdPartition tidp1 = new TopicIdPartition(topicIds.get(topic1), tp1);
    TopicIdPartition tidp2 = new TopicIdPartition(topicIds.get(topic2), tp2);
    client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, partitionCounts, tp -> validLeaderEpoch, topicIds));
    int expectedBytes = 0;
    LinkedHashMap<TopicIdPartition, FetchResponseData.PartitionData> fetchPartitionData = new LinkedHashMap<>();
    for (TopicIdPartition tp : mkSet(tidp1, tidp2)) {
        subscriptions.seek(tp.topicPartition(), 0);
        MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L);
        for (int v = 0; v < 3; v++) builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes());
        MemoryRecords records = builder.build();
        for (Record record : records.records()) expectedBytes += record.sizeInBytes();
        fetchPartitionData.put(tp, new FetchResponseData.PartitionData().setPartitionIndex(tp.topicPartition().partition()).setHighWatermark(15).setLogStartOffset(0).setRecords(records));
    }
    assertEquals(1, fetcher.sendFetches());
    client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, fetchPartitionData));
    consumerClient.poll(time.timer(0));
    Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
    assertEquals(3, fetchedRecords.get(tp1).size());
    assertEquals(3, fetchedRecords.get(tp2).size());
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg));
    KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg));
    assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON);
    assertEquals(6, (Double) recordsCountAverage.metricValue(), EPSILON);
}
Also used : BeforeEach(org.junit.jupiter.api.BeforeEach) MockTime(org.apache.kafka.common.utils.MockTime) Arrays(java.util.Arrays) ListOffsetsRequest(org.apache.kafka.common.requests.ListOffsetsRequest) SerializationException(org.apache.kafka.common.errors.SerializationException) KafkaException(org.apache.kafka.common.KafkaException) DefaultRecordBatch(org.apache.kafka.common.record.DefaultRecordBatch) OffsetResetStrategy(org.apache.kafka.clients.consumer.OffsetResetStrategy) Collections.singletonList(java.util.Collections.singletonList) ClientUtils(org.apache.kafka.clients.ClientUtils) Cluster(org.apache.kafka.common.Cluster) Future(java.util.concurrent.Future) StringDeserializer(org.apache.kafka.common.serialization.StringDeserializer) DataOutputStream(java.io.DataOutputStream) ApiVersionsResponse(org.apache.kafka.common.requests.ApiVersionsResponse) Arrays.asList(java.util.Arrays.asList) RecordBatch(org.apache.kafka.common.record.RecordBatch) ListOffsetsResponse(org.apache.kafka.common.requests.ListOffsetsResponse) LogContext(org.apache.kafka.common.utils.LogContext) Duration(java.time.Duration) Map(java.util.Map) FetchResponse(org.apache.kafka.common.requests.FetchResponse) TimestampType(org.apache.kafka.common.record.TimestampType) Sensor(org.apache.kafka.common.metrics.Sensor) CompressionType(org.apache.kafka.common.record.CompressionType) TestUtils(org.apache.kafka.test.TestUtils) Utils.mkSet(org.apache.kafka.common.utils.Utils.mkSet) InvalidTopicException(org.apache.kafka.common.errors.InvalidTopicException) Set(java.util.Set) PartitionInfo(org.apache.kafka.common.PartitionInfo) OffsetAndTimestamp(org.apache.kafka.clients.consumer.OffsetAndTimestamp) TopicIdPartition(org.apache.kafka.common.TopicIdPartition) StandardCharsets(java.nio.charset.StandardCharsets) Executors(java.util.concurrent.Executors) MemoryRecords(org.apache.kafka.common.record.MemoryRecords) Metrics(org.apache.kafka.common.metrics.Metrics) ApiMessageType(org.apache.kafka.common.message.ApiMessageType) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) ListOffsetsTopicResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse) Errors(org.apache.kafka.common.protocol.Errors) Node(org.apache.kafka.common.Node) FetchRequest(org.apache.kafka.common.requests.FetchRequest) MetadataResponse(org.apache.kafka.common.requests.MetadataResponse) NodeApiVersions(org.apache.kafka.clients.NodeApiVersions) Records(org.apache.kafka.common.record.Records) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) OffsetForLeaderPartition(org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderPartition) ClientDnsLookup(org.apache.kafka.clients.ClientDnsLookup) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) RequestTestUtils(org.apache.kafka.common.requests.RequestTestUtils) RecordHeader(org.apache.kafka.common.header.internals.RecordHeader) ListOffsetsPartitionResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse) OffsetForLeaderEpochRequestData(org.apache.kafka.common.message.OffsetForLeaderEpochRequestData) INVALID_SESSION_ID(org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) UNDEFINED_EPOCH(org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse.UNDEFINED_EPOCH) NetworkClient(org.apache.kafka.clients.NetworkClient) Deserializer(org.apache.kafka.common.serialization.Deserializer) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) TestUtils.assertOptional(org.apache.kafka.test.TestUtils.assertOptional) OffsetOutOfRangeException(org.apache.kafka.clients.consumer.OffsetOutOfRangeException) ByteArrayDeserializer(org.apache.kafka.common.serialization.ByteArrayDeserializer) MockSelector(org.apache.kafka.test.MockSelector) Field(java.lang.reflect.Field) ApiVersions(org.apache.kafka.clients.ApiVersions) MetricNameTemplate(org.apache.kafka.common.MetricNameTemplate) OffsetForLeaderEpochResponseData(org.apache.kafka.common.message.OffsetForLeaderEpochResponseData) Assertions.assertArrayEquals(org.junit.jupiter.api.Assertions.assertArrayEquals) AfterEach(org.junit.jupiter.api.AfterEach) NetworkReceive(org.apache.kafka.common.network.NetworkReceive) SimpleRecord(org.apache.kafka.common.record.SimpleRecord) PartitionData(org.apache.kafka.common.requests.FetchRequest.PartitionData) BytesDeserializer(org.apache.kafka.common.serialization.BytesDeserializer) ByteBufferOutputStream(org.apache.kafka.common.utils.ByteBufferOutputStream) LogTruncationException(org.apache.kafka.clients.consumer.LogTruncationException) Assertions.assertNotEquals(org.junit.jupiter.api.Assertions.assertNotEquals) AbstractRequest(org.apache.kafka.common.requests.AbstractRequest) ControlRecordType(org.apache.kafka.common.record.ControlRecordType) ByteBuffer(java.nio.ByteBuffer) ClientRequest(org.apache.kafka.clients.ClientRequest) FetchResponseData(org.apache.kafka.common.message.FetchResponseData) Record(org.apache.kafka.common.record.Record) Collections.singleton(java.util.Collections.singleton) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) BufferSupplier(org.apache.kafka.common.utils.BufferSupplier) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) MetricName(org.apache.kafka.common.MetricName) OffsetForLeaderTopicResult(org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.OffsetForLeaderTopicResult) ListOffsetsTopic(org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic) TopicPartition(org.apache.kafka.common.TopicPartition) MemoryRecordsBuilder(org.apache.kafka.common.record.MemoryRecordsBuilder) LegacyRecord(org.apache.kafka.common.record.LegacyRecord) Collections.emptyList(java.util.Collections.emptyList) MetricConfig(org.apache.kafka.common.metrics.MetricConfig) ClusterResourceListeners(org.apache.kafka.common.internals.ClusterResourceListeners) Collectors(java.util.stream.Collectors) ListOffsetsResponseData(org.apache.kafka.common.message.ListOffsetsResponseData) Test(org.junit.jupiter.api.Test) ConsumerRebalanceListener(org.apache.kafka.clients.consumer.ConsumerRebalanceListener) List(java.util.List) Header(org.apache.kafka.common.header.Header) OffsetAndMetadata(org.apache.kafka.clients.consumer.OffsetAndMetadata) RecordTooLargeException(org.apache.kafka.common.errors.RecordTooLargeException) Optional(java.util.Optional) KafkaMetric(org.apache.kafka.common.metrics.KafkaMetric) OffsetsForLeaderEpochResponse(org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse) Uuid(org.apache.kafka.common.Uuid) EpochEndOffset(org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset) Metadata(org.apache.kafka.clients.Metadata) EndTransactionMarker(org.apache.kafka.common.record.EndTransactionMarker) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) FetchSessionHandler(org.apache.kafka.clients.FetchSessionHandler) HashMap(java.util.HashMap) Function(java.util.function.Function) HashSet(java.util.HashSet) MetadataRequest(org.apache.kafka.common.requests.MetadataRequest) Collections.singletonMap(java.util.Collections.singletonMap) ExecutorService(java.util.concurrent.ExecutorService) Utils(org.apache.kafka.common.utils.Utils) UNDEFINED_EPOCH_OFFSET(org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse.UNDEFINED_EPOCH_OFFSET) Collections.emptyMap(java.util.Collections.emptyMap) TimeoutException(org.apache.kafka.common.errors.TimeoutException) MockClient(org.apache.kafka.clients.MockClient) ListOffsetsPartition(org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition) Iterator(java.util.Iterator) ApiKeys(org.apache.kafka.common.protocol.ApiKeys) TimeUnit(java.util.concurrent.TimeUnit) IsolationLevel(org.apache.kafka.common.IsolationLevel) DelayedReceive(org.apache.kafka.test.DelayedReceive) TopicAuthorizationException(org.apache.kafka.common.errors.TopicAuthorizationException) OffsetsForLeaderEpochRequest(org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest) Collections(java.util.Collections) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) TopicIdPartition(org.apache.kafka.common.TopicIdPartition) KafkaMetric(org.apache.kafka.common.metrics.KafkaMetric) LinkedHashMap(java.util.LinkedHashMap) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) MetricName(org.apache.kafka.common.MetricName) FetchResponseData(org.apache.kafka.common.message.FetchResponseData) PartitionData(org.apache.kafka.common.requests.FetchRequest.PartitionData) TopicPartition(org.apache.kafka.common.TopicPartition) MemoryRecordsBuilder(org.apache.kafka.common.record.MemoryRecordsBuilder) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) SimpleRecord(org.apache.kafka.common.record.SimpleRecord) Record(org.apache.kafka.common.record.Record) LegacyRecord(org.apache.kafka.common.record.LegacyRecord) Collections.singletonList(java.util.Collections.singletonList) Arrays.asList(java.util.Arrays.asList) ArrayList(java.util.ArrayList) Collections.emptyList(java.util.Collections.emptyList) List(java.util.List) MemoryRecords(org.apache.kafka.common.record.MemoryRecords) Test(org.junit.jupiter.api.Test)

Example 4 with Records

use of org.apache.kafka.common.record.Records in project kafka by apache.

the class MockLog method read.

@Override
public LogFetchInfo read(long startOffset, Isolation isolation) {
    OptionalLong maxOffsetOpt = isolation == Isolation.COMMITTED ? OptionalLong.of(highWatermark.offset) : OptionalLong.empty();
    verifyOffsetInRange(startOffset);
    long maxOffset = maxOffsetOpt.orElse(endOffset().offset);
    if (startOffset >= maxOffset) {
        return new LogFetchInfo(MemoryRecords.EMPTY, new LogOffsetMetadata(startOffset, metadataForOffset(startOffset)));
    }
    ByteBuffer buffer = ByteBuffer.allocate(512);
    int batchCount = 0;
    LogOffsetMetadata batchStartOffset = null;
    for (LogBatch batch : batches) {
        // batch returned in a fetch response.
        if (batch.lastOffset() >= startOffset && batch.lastOffset() < maxOffset && !batch.entries.isEmpty()) {
            buffer = batch.writeTo(buffer);
            if (batchStartOffset == null) {
                batchStartOffset = batch.entries.get(0).logOffsetMetadata();
            }
            // Read on the mock log should return at most 2 batches. This is a simple solution
            // for testing interesting partial read scenarios.
            batchCount += 1;
            if (batchCount >= 2) {
                break;
            }
        }
    }
    buffer.flip();
    Records records = MemoryRecords.readableRecords(buffer);
    if (batchStartOffset == null) {
        throw new RuntimeException("Expected to find at least one entry starting from offset " + startOffset + " but found none");
    }
    return new LogFetchInfo(records, batchStartOffset);
}
Also used : OptionalLong(java.util.OptionalLong) ByteBuffer(java.nio.ByteBuffer) Records(org.apache.kafka.common.record.Records) MemoryRecords(org.apache.kafka.common.record.MemoryRecords)

Example 5 with Records

use of org.apache.kafka.common.record.Records in project kafka by apache.

the class MockLogTest method testReadRecords.

@Test
public void testReadRecords() {
    int epoch = 2;
    ByteBuffer recordOneBuffer = ByteBuffer.allocate(4);
    recordOneBuffer.putInt(1);
    SimpleRecord recordOne = new SimpleRecord(recordOneBuffer);
    ByteBuffer recordTwoBuffer = ByteBuffer.allocate(4);
    recordTwoBuffer.putInt(2);
    SimpleRecord recordTwo = new SimpleRecord(recordTwoBuffer);
    appendAsLeader(Arrays.asList(recordOne, recordTwo), epoch);
    Records records = log.read(0, Isolation.UNCOMMITTED).records;
    List<ByteBuffer> extractRecords = new ArrayList<>();
    for (Record record : records.records()) {
        extractRecords.add(record.value());
    }
    assertEquals(Arrays.asList(recordOne.value(), recordTwo.value()), extractRecords);
}
Also used : SimpleRecord(org.apache.kafka.common.record.SimpleRecord) ArrayList(java.util.ArrayList) Record(org.apache.kafka.common.record.Record) SimpleRecord(org.apache.kafka.common.record.SimpleRecord) ByteBuffer(java.nio.ByteBuffer) Records(org.apache.kafka.common.record.Records) MemoryRecords(org.apache.kafka.common.record.MemoryRecords) Test(org.junit.jupiter.api.Test)

Aggregations

Records (org.apache.kafka.common.record.Records)19 MemoryRecords (org.apache.kafka.common.record.MemoryRecords)16 Test (org.junit.jupiter.api.Test)11 Record (org.apache.kafka.common.record.Record)9 ByteBuffer (java.nio.ByteBuffer)8 RecordBatch (org.apache.kafka.common.record.RecordBatch)7 SimpleRecord (org.apache.kafka.common.record.SimpleRecord)7 FetchResponseData (org.apache.kafka.common.message.FetchResponseData)6 ArrayList (java.util.ArrayList)5 KafkaException (org.apache.kafka.common.KafkaException)3 Errors (org.apache.kafka.common.protocol.Errors)3 DataOutputStream (java.io.DataOutputStream)2 Field (java.lang.reflect.Field)2 StandardCharsets (java.nio.charset.StandardCharsets)2 Duration (java.time.Duration)2 Arrays (java.util.Arrays)2 Arrays.asList (java.util.Arrays.asList)2 Collections (java.util.Collections)2 Collections.emptyList (java.util.Collections.emptyList)2 Collections.emptyMap (java.util.Collections.emptyMap)2