Examples with MemoryRecords org.apache.kafka.common.record.MemoryRecords used on opensource projects

Example 76 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class KafkaRaftClientSnapshotTest method testCreateSnapshotAsFollowerWithInvalidSnapshotId.

@Test
public void testCreateSnapshotAsFollowerWithInvalidSnapshotId() throws Exception {
    int localId = 0;
    int leaderId = 1;
    int otherFollowerId = 2;
    int epoch = 5;
    Set<Integer> voters = Utils.mkSet(localId, leaderId, otherFollowerId);
    RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).withElectedLeader(epoch, leaderId).build();
    context.assertElectedLeader(epoch, leaderId);
    // When follower creating snapshot:
    // 1) The high watermark cannot be empty
    assertEquals(OptionalLong.empty(), context.client.highWatermark());
    OffsetAndEpoch invalidSnapshotId1 = new OffsetAndEpoch(0, 0);
    assertThrows(IllegalArgumentException.class, () -> context.client.createSnapshot(invalidSnapshotId1.offset, invalidSnapshotId1.epoch, 0));
    // Poll for our first fetch request
    context.pollUntilRequest();
    RaftRequest.Outbound fetchRequest = context.assertSentFetchRequest();
    assertTrue(voters.contains(fetchRequest.destinationId()));
    context.assertFetchRequestData(fetchRequest, epoch, 0L, 0);
    // The response does not advance the high watermark
    List<String> records1 = Arrays.asList("a", "b", "c");
    MemoryRecords batch1 = context.buildBatch(0L, 3, records1);
    context.deliverResponse(fetchRequest.correlationId, fetchRequest.destinationId(), context.fetchResponse(epoch, leaderId, batch1, 0L, Errors.NONE));
    context.client.poll();
    // 2) The high watermark must be larger than or equal to the snapshotId's endOffset
    int currentEpoch = context.currentEpoch();
    OffsetAndEpoch invalidSnapshotId2 = new OffsetAndEpoch(context.client.highWatermark().getAsLong() + 1, currentEpoch);
    assertThrows(IllegalArgumentException.class, () -> context.client.createSnapshot(invalidSnapshotId2.offset, invalidSnapshotId2.epoch, 0));
    // 3) The quorum epoch must be larger than or equal to the snapshotId's epoch
    OffsetAndEpoch invalidSnapshotId3 = new OffsetAndEpoch(context.client.highWatermark().getAsLong(), currentEpoch + 1);
    assertThrows(IllegalArgumentException.class, () -> context.client.createSnapshot(invalidSnapshotId3.offset, invalidSnapshotId3.epoch, 0));
    // The high watermark advances to be larger than log.endOffsetForEpoch(3), to test the case 3
    context.pollUntilRequest();
    fetchRequest = context.assertSentFetchRequest();
    assertTrue(voters.contains(fetchRequest.destinationId()));
    context.assertFetchRequestData(fetchRequest, epoch, 3L, 3);
    List<String> records2 = Arrays.asList("d", "e", "f");
    MemoryRecords batch2 = context.buildBatch(3L, 4, records2);
    context.deliverResponse(fetchRequest.correlationId, fetchRequest.destinationId(), context.fetchResponse(epoch, leaderId, batch2, 6L, Errors.NONE));
    context.client.poll();
    assertEquals(6L, context.client.highWatermark().getAsLong());
    // 4) The snapshotId should be validated against endOffsetForEpoch
    OffsetAndEpoch endOffsetForEpoch = context.log.endOffsetForEpoch(3);
    assertEquals(3, endOffsetForEpoch.epoch);
    OffsetAndEpoch invalidSnapshotId4 = new OffsetAndEpoch(endOffsetForEpoch.offset + 1, epoch);
    assertThrows(IllegalArgumentException.class, () -> context.client.createSnapshot(invalidSnapshotId4.offset, invalidSnapshotId4.epoch, 0));
}

Example 77 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class KafkaRaftClientTest method testPurgatoryFetchCompletedByFollowerTransition.

@Test
public void testPurgatoryFetchCompletedByFollowerTransition() throws Exception {
    int localId = 0;
    int voter1 = localId;
    int voter2 = localId + 1;
    int voter3 = localId + 2;
    int epoch = 5;
    Set<Integer> voters = Utils.mkSet(voter1, voter2, voter3);
    RaftClientTestContext context = RaftClientTestContext.initializeAsLeader(localId, voters, epoch);
    // Follower sends a fetch which cannot be satisfied immediately
    context.deliverRequest(context.fetchRequest(epoch, voter2, 1L, epoch, 500));
    context.client.poll();
    assertTrue(context.channel.drainSendQueue().stream().noneMatch(msg -> msg.data() instanceof FetchResponseData));
    // Now we get a BeginEpoch from the other voter and become a follower
    context.deliverRequest(context.beginEpochRequest(epoch + 1, voter3));
    context.pollUntilResponse();
    context.assertElectedLeader(epoch + 1, voter3);
    // We expect the BeginQuorumEpoch response and a failed Fetch response
    context.assertSentBeginQuorumEpochResponse(Errors.NONE, epoch + 1, OptionalInt.of(voter3));
    // The fetch should be satisfied immediately and return an error
    MemoryRecords fetchedRecords = context.assertSentFetchPartitionResponse(Errors.NOT_LEADER_OR_FOLLOWER, epoch + 1, OptionalInt.of(voter3));
    assertEquals(0, fetchedRecords.sizeInBytes());
}

Example 78 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class KafkaRaftClientTest method testPurgatoryFetchTimeout.

@Test
public void testPurgatoryFetchTimeout() throws Exception {
    int localId = 0;
    int otherNodeId = 1;
    int epoch = 5;
    Set<Integer> voters = Utils.mkSet(localId, otherNodeId);
    RaftClientTestContext context = RaftClientTestContext.initializeAsLeader(localId, voters, epoch);
    // Follower sends a fetch which cannot be satisfied immediately
    int maxWaitTimeMs = 500;
    context.deliverRequest(context.fetchRequest(epoch, otherNodeId, 1L, epoch, maxWaitTimeMs));
    context.client.poll();
    assertEquals(0, context.channel.drainSendQueue().size());
    // After expiration of the max wait time, the fetch returns an empty record set
    context.time.sleep(maxWaitTimeMs);
    context.client.poll();
    MemoryRecords fetchedRecords = context.assertSentFetchPartitionResponse(Errors.NONE, epoch, OptionalInt.of(localId));
    assertEquals(0, fetchedRecords.sizeInBytes());
}

Example 79 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class KafkaRaftClientTest method testPurgatoryFetchSatisfiedByWrite.

@Test
public void testPurgatoryFetchSatisfiedByWrite() throws Exception {
    int localId = 0;
    int otherNodeId = 1;
    int epoch = 5;
    Set<Integer> voters = Utils.mkSet(localId, otherNodeId);
    RaftClientTestContext context = RaftClientTestContext.initializeAsLeader(localId, voters, epoch);
    // Follower sends a fetch which cannot be satisfied immediately
    context.deliverRequest(context.fetchRequest(epoch, otherNodeId, 1L, epoch, 500));
    context.client.poll();
    assertEquals(0, context.channel.drainSendQueue().size());
    // Append some records that can fulfill the Fetch request
    String[] appendRecords = new String[] { "a", "b", "c" };
    context.client.scheduleAppend(epoch, Arrays.asList(appendRecords));
    context.client.poll();
    MemoryRecords fetchedRecords = context.assertSentFetchPartitionResponse(Errors.NONE, epoch, OptionalInt.of(localId));
    RaftClientTestContext.assertMatchingRecords(appendRecords, fetchedRecords);
}

Example 80 with MemoryRecords

use of org.apache.kafka.common.record.MemoryRecords in project kafka by apache.

the class RecordsBatchReaderTest method testReadFromFileRecords.

@ParameterizedTest
@EnumSource(CompressionType.class)
public void testReadFromFileRecords(CompressionType compressionType) throws Exception {
    long baseOffset = 57;
    List<TestBatch<String>> batches = RecordsIteratorTest.createBatches(baseOffset);
    MemoryRecords memRecords = RecordsIteratorTest.buildRecords(compressionType, batches);
    FileRecords fileRecords = FileRecords.open(tempFile());
    fileRecords.append(memRecords);
    testBatchReader(baseOffset, fileRecords, batches);
}