Examples with MemoryRecords org.apache.kafka.common.record.MemoryRecords used on opensource projects

Example 36 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class KafkaConsumerTest method fetchResponse.

private FetchResponse fetchResponse(Map<TopicPartition, FetchInfo> fetches) {
    LinkedHashMap<TopicIdPartition, FetchResponseData.PartitionData> tpResponses = new LinkedHashMap<>();
    for (Map.Entry<TopicPartition, FetchInfo> fetchEntry : fetches.entrySet()) {
        TopicPartition partition = fetchEntry.getKey();
        long fetchOffset = fetchEntry.getValue().offset;
        int fetchCount = fetchEntry.getValue().count;
        final long highWatermark = fetchEntry.getValue().logLastOffset + 1;
        final long logStartOffset = fetchEntry.getValue().logFirstOffset;
        final MemoryRecords records;
        if (fetchCount == 0) {
            records = MemoryRecords.EMPTY;
        } else {
            MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, fetchOffset);
            for (int i = 0; i < fetchCount; i++) builder.append(0L, ("key-" + i).getBytes(), ("value-" + i).getBytes());
            records = builder.build();
        }
        tpResponses.put(new TopicIdPartition(topicIds.get(partition.topic()), partition), new FetchResponseData.PartitionData().setPartitionIndex(partition.partition()).setHighWatermark(highWatermark).setLogStartOffset(logStartOffset).setRecords(records));
    }
    return FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, tpResponses);
}

Example 37 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class Writable method writeRecords.

default void writeRecords(BaseRecords records) {
    if (records instanceof MemoryRecords) {
        MemoryRecords memRecords = (MemoryRecords) records;
        writeByteBuffer(memRecords.buffer());
    } else {
        throw new UnsupportedOperationException("Unsupported record type " + records.getClass());
    }
}

Example 38 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class FetcherTest method testFetchResponseMetrics.

@Test
public void testFetchResponseMetrics() {
    buildFetcher();
    String topic1 = "foo";
    String topic2 = "bar";
    TopicPartition tp1 = new TopicPartition(topic1, 0);
    TopicPartition tp2 = new TopicPartition(topic2, 0);
    subscriptions.assignFromUser(mkSet(tp1, tp2));
    Map<String, Integer> partitionCounts = new HashMap<>();
    partitionCounts.put(topic1, 1);
    partitionCounts.put(topic2, 1);
    topicIds.put(topic1, Uuid.randomUuid());
    topicIds.put(topic2, Uuid.randomUuid());
    TopicIdPartition tidp1 = new TopicIdPartition(topicIds.get(topic1), tp1);
    TopicIdPartition tidp2 = new TopicIdPartition(topicIds.get(topic2), tp2);
    client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, partitionCounts, tp -> validLeaderEpoch, topicIds));
    int expectedBytes = 0;
    LinkedHashMap<TopicIdPartition, FetchResponseData.PartitionData> fetchPartitionData = new LinkedHashMap<>();
    for (TopicIdPartition tp : mkSet(tidp1, tidp2)) {
        subscriptions.seek(tp.topicPartition(), 0);
        MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L);
        for (int v = 0; v < 3; v++) builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes());
        MemoryRecords records = builder.build();
        for (Record record : records.records()) expectedBytes += record.sizeInBytes();
        fetchPartitionData.put(tp, new FetchResponseData.PartitionData().setPartitionIndex(tp.topicPartition().partition()).setHighWatermark(15).setLogStartOffset(0).setRecords(records));
    }
    assertEquals(1, fetcher.sendFetches());
    client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, fetchPartitionData));
    consumerClient.poll(time.timer(0));
    Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
    assertEquals(3, fetchedRecords.get(tp1).size());
    assertEquals(3, fetchedRecords.get(tp2).size());
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg));
    KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg));
    assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON);
    assertEquals(6, (Double) recordsCountAverage.metricValue(), EPSILON);
}

Example 39 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class FetcherTest method testMissingLeaderEpochInRecords.

@Test
public void testMissingLeaderEpochInRecords() {
    buildFetcher();
    assignFromUser(singleton(tp0));
    subscriptions.seek(tp0, 0);
    ByteBuffer buffer = ByteBuffer.allocate(1024);
    MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V0, CompressionType.NONE, TimestampType.CREATE_TIME, 0L, System.currentTimeMillis(), RecordBatch.NO_PARTITION_LEADER_EPOCH);
    builder.append(0L, "key".getBytes(), "1".getBytes());
    builder.append(0L, "key".getBytes(), "2".getBytes());
    MemoryRecords records = builder.build();
    assertEquals(1, fetcher.sendFetches());
    assertFalse(fetcher.hasCompletedFetches());
    client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0));
    consumerClient.poll(time.timer(0));
    assertTrue(fetcher.hasCompletedFetches());
    Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
    assertTrue(partitionRecords.containsKey(tp0));
    assertEquals(2, partitionRecords.get(tp0).size());
    for (ConsumerRecord<byte[], byte[]> record : partitionRecords.get(tp0)) {
        assertEquals(Optional.empty(), record.leaderEpoch());
    }
}

Example 40 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class FetcherTest method testHeaders.

@Test
public void testHeaders() {
    buildFetcher();
    MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 1L);
    builder.append(0L, "key".getBytes(), "value-1".getBytes());
    Header[] headersArray = new Header[1];
    headersArray[0] = new RecordHeader("headerKey", "headerValue".getBytes(StandardCharsets.UTF_8));
    builder.append(0L, "key".getBytes(), "value-2".getBytes(), headersArray);
    Header[] headersArray2 = new Header[2];
    headersArray2[0] = new RecordHeader("headerKey", "headerValue".getBytes(StandardCharsets.UTF_8));
    headersArray2[1] = new RecordHeader("headerKey", "headerValue2".getBytes(StandardCharsets.UTF_8));
    builder.append(0L, "key".getBytes(), "value-3".getBytes(), headersArray2);
    MemoryRecords memoryRecords = builder.build();
    List<ConsumerRecord<byte[], byte[]>> records;
    assignFromUser(singleton(tp0));
    subscriptions.seek(tp0, 1);
    client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, memoryRecords, Errors.NONE, 100L, 0));
    assertEquals(1, fetcher.sendFetches());
    consumerClient.poll(time.timer(0));
    Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> recordsByPartition = fetchedRecords();
    records = recordsByPartition.get(tp0);
    assertEquals(3, records.size());
    Iterator<ConsumerRecord<byte[], byte[]>> recordIterator = records.iterator();
    ConsumerRecord<byte[], byte[]> record = recordIterator.next();
    assertNull(record.headers().lastHeader("headerKey"));
    record = recordIterator.next();
    assertEquals("headerValue", new String(record.headers().lastHeader("headerKey").value(), StandardCharsets.UTF_8));
    assertEquals("headerKey", record.headers().lastHeader("headerKey").key());
    record = recordIterator.next();
    assertEquals("headerValue2", new String(record.headers().lastHeader("headerKey").value(), StandardCharsets.UTF_8));
    assertEquals("headerKey", record.headers().lastHeader("headerKey").key());
}