Examples with MemoryRecords org.apache.kafka.common.record.MemoryRecords used on opensource projects

Example 51 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class BatchBuilderTest method testHasRoomForUncompressed.

@ParameterizedTest
@ValueSource(ints = { 128, 157, 256, 433, 512, 777, 1024 })
public void testHasRoomForUncompressed(int batchSize) {
    ByteBuffer buffer = ByteBuffer.allocate(batchSize);
    long baseOffset = 57;
    long logAppendTime = time.milliseconds();
    boolean isControlBatch = false;
    int leaderEpoch = 15;
    BatchBuilder<String> builder = new BatchBuilder<>(buffer, serde, CompressionType.NONE, baseOffset, logAppendTime, isControlBatch, leaderEpoch, buffer.limit());
    String record = "i am a record";
    while (!builder.bytesNeeded(Arrays.asList(record), null).isPresent()) {
        builder.appendRecord(record, null);
    }
    // Approximate size should be exact when compression is not used
    int sizeInBytes = builder.approximateSizeInBytes();
    MemoryRecords records = builder.build();
    assertEquals(sizeInBytes, records.sizeInBytes());
    assertTrue(sizeInBytes <= batchSize, "Built batch size " + sizeInBytes + " is larger than max batch size " + batchSize);
}

Example 52 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class RecordsBatchReaderTest method testReadFromMemoryRecords.

@ParameterizedTest
@EnumSource(CompressionType.class)
public void testReadFromMemoryRecords(CompressionType compressionType) {
    long baseOffset = 57;
    List<TestBatch<String>> batches = RecordsIteratorTest.createBatches(baseOffset);
    MemoryRecords memRecords = RecordsIteratorTest.buildRecords(compressionType, batches);
    testBatchReader(baseOffset, memRecords, batches);
}

Example 53 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class RecordsIteratorTest method testMemoryRecords.

@Property
public void testMemoryRecords(@ForAll CompressionType compressionType, @ForAll long seed) {
    List<TestBatch<String>> batches = createBatches(seed);
    MemoryRecords memRecords = buildRecords(compressionType, batches);
    testIterator(batches, memRecords);
}

Example 54 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class FetchResponseBenchmark method setup.

@Setup(Level.Trial)
public void setup() {
    MemoryRecords records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord(1000, "key1".getBytes(StandardCharsets.UTF_8), "value1".getBytes(StandardCharsets.UTF_8)), new SimpleRecord(1001, "key2".getBytes(StandardCharsets.UTF_8), "value2".getBytes(StandardCharsets.UTF_8)), new SimpleRecord(1002, "key3".getBytes(StandardCharsets.UTF_8), "value3".getBytes(StandardCharsets.UTF_8)));
    this.responseData = new LinkedHashMap<>();
    this.topicIds = new HashMap<>();
    this.topicNames = new HashMap<>();
    for (int topicIdx = 0; topicIdx < topicCount; topicIdx++) {
        String topic = UUID.randomUUID().toString();
        Uuid id = Uuid.randomUuid();
        topicIds.put(topic, id);
        topicNames.put(id, topic);
        for (int partitionId = 0; partitionId < partitionCount; partitionId++) {
            FetchResponseData.PartitionData partitionData = new FetchResponseData.PartitionData().setPartitionIndex(partitionId).setLastStableOffset(0).setLogStartOffset(0).setRecords(records);
            responseData.put(new TopicIdPartition(id, new TopicPartition(topic, partitionId)), partitionData);
        }
    }
    this.header = new ResponseHeader(100, ApiKeys.FETCH.responseHeaderVersion(ApiKeys.FETCH.latestVersion()));
    this.fetchResponse = FetchResponse.of(Errors.NONE, 0, 0, responseData);
    this.fetchResponseData = this.fetchResponse.data();
}

Example 55 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class KafkaRaftClientTest method testObserverFetchWithNoLocalId.

@Test
public void testObserverFetchWithNoLocalId() throws Exception {
    // When no `localId` is defined, the client will behave as an observer.
    // This is designed for tooling/debugging use cases.
    Set<Integer> voters = Utils.mkSet(1, 2);
    RaftClientTestContext context = new RaftClientTestContext.Builder(OptionalInt.empty(), voters).build();
    // First fetch discovers the current leader and epoch
    context.pollUntilRequest();
    RaftRequest.Outbound fetchRequest1 = context.assertSentFetchRequest();
    assertTrue(voters.contains(fetchRequest1.destinationId()));
    context.assertFetchRequestData(fetchRequest1, 0, 0L, 0);
    int leaderEpoch = 5;
    int leaderId = 1;
    context.deliverResponse(fetchRequest1.correlationId, fetchRequest1.destinationId(), context.fetchResponse(5, leaderId, MemoryRecords.EMPTY, 0L, Errors.FENCED_LEADER_EPOCH));
    context.client.poll();
    context.assertElectedLeader(leaderEpoch, leaderId);
    // Second fetch goes to the discovered leader
    context.pollUntilRequest();
    RaftRequest.Outbound fetchRequest2 = context.assertSentFetchRequest();
    assertEquals(leaderId, fetchRequest2.destinationId());
    context.assertFetchRequestData(fetchRequest2, leaderEpoch, 0L, 0);
    List<String> records = Arrays.asList("a", "b", "c");
    MemoryRecords batch1 = context.buildBatch(0L, 3, records);
    context.deliverResponse(fetchRequest2.correlationId, fetchRequest2.destinationId(), context.fetchResponse(leaderEpoch, leaderId, batch1, 0L, Errors.NONE));
    context.client.poll();
    assertEquals(3L, context.log.endOffset().offset);
    assertEquals(3, context.log.lastFetchedEpoch());
}