Examples with RawSnapshotReader org.apache.kafka.snapshot.RawSnapshotReader used on opensource projects

Example 6 with RawSnapshotReader

use of org.apache.kafka.snapshot.RawSnapshotReader in project kafka by apache.

the class KafkaRaftClientSnapshotTest method testFetchResponseWithSnapshotId.

@Test
public void testFetchResponseWithSnapshotId() throws Exception {
    int localId = 0;
    int leaderId = localId + 1;
    Set<Integer> voters = Utils.mkSet(localId, leaderId);
    int epoch = 2;
    OffsetAndEpoch snapshotId = new OffsetAndEpoch(100L, 1);
    RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).withElectedLeader(epoch, leaderId).build();
    context.pollUntilRequest();
    RaftRequest.Outbound fetchRequest = context.assertSentFetchRequest();
    context.assertFetchRequestData(fetchRequest, epoch, 0L, 0);
    context.deliverResponse(fetchRequest.correlationId, fetchRequest.destinationId(), snapshotFetchResponse(context.metadataPartition, context.metadataTopicId, epoch, leaderId, snapshotId, 200L));
    context.pollUntilRequest();
    RaftRequest.Outbound snapshotRequest = context.assertSentFetchSnapshotRequest();
    FetchSnapshotRequestData.PartitionSnapshot request = assertFetchSnapshotRequest(snapshotRequest, context.metadataPartition, localId, Integer.MAX_VALUE).get();
    assertEquals(snapshotId.offset, request.snapshotId().endOffset());
    assertEquals(snapshotId.epoch, request.snapshotId().epoch());
    assertEquals(0, request.position());
    List<String> records = Arrays.asList("foo", "bar");
    MemorySnapshotWriter memorySnapshot = new MemorySnapshotWriter(snapshotId);
    try (SnapshotWriter<String> snapshotWriter = snapshotWriter(context, memorySnapshot)) {
        snapshotWriter.append(records);
        snapshotWriter.freeze();
    }
    context.deliverResponse(snapshotRequest.correlationId, snapshotRequest.destinationId(), fetchSnapshotResponse(context.metadataPartition, epoch, leaderId, snapshotId, memorySnapshot.buffer().remaining(), 0L, memorySnapshot.buffer().slice()));
    context.pollUntilRequest();
    fetchRequest = context.assertSentFetchRequest();
    context.assertFetchRequestData(fetchRequest, epoch, snapshotId.offset, snapshotId.epoch);
    // Check that the snapshot was written to the log
    RawSnapshotReader snapshot = context.log.readSnapshot(snapshotId).get();
    assertEquals(memorySnapshot.buffer().remaining(), snapshot.sizeInBytes());
    SnapshotWriterReaderTest.assertSnapshot(Arrays.asList(records), snapshot);
    // Check that listener was notified of the new snapshot
    try (SnapshotReader<String> reader = context.listener.drainHandledSnapshot().get()) {
        assertEquals(snapshotId, reader.snapshotId());
        SnapshotWriterReaderTest.assertSnapshot(Arrays.asList(records), reader);
    }
}

Example 7 with RawSnapshotReader

use of org.apache.kafka.snapshot.RawSnapshotReader in project kafka by apache.

the class KafkaRaftClientSnapshotTest method testFetchSnapshotRequestAsLeader.

@Test
public void testFetchSnapshotRequestAsLeader() throws Exception {
    int localId = 0;
    Set<Integer> voters = Utils.mkSet(localId, localId + 1);
    OffsetAndEpoch snapshotId = new OffsetAndEpoch(1, 1);
    List<String> records = Arrays.asList("foo", "bar");
    RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).appendToLog(snapshotId.epoch, Arrays.asList("a")).build();
    context.becomeLeader();
    int epoch = context.currentEpoch();
    context.advanceLocalLeaderHighWatermarkToLogEndOffset();
    try (SnapshotWriter<String> snapshot = context.client.createSnapshot(snapshotId.offset - 1, snapshotId.epoch, 0).get()) {
        assertEquals(snapshotId, snapshot.snapshotId());
        snapshot.append(records);
        snapshot.freeze();
    }
    RawSnapshotReader snapshot = context.log.readSnapshot(snapshotId).get();
    context.deliverRequest(fetchSnapshotRequest(context.metadataPartition, epoch, snapshotId, Integer.MAX_VALUE, 0));
    context.client.poll();
    FetchSnapshotResponseData.PartitionSnapshot response = context.assertSentFetchSnapshotResponse(context.metadataPartition).get();
    assertEquals(Errors.NONE, Errors.forCode(response.errorCode()));
    assertEquals(snapshot.sizeInBytes(), response.size());
    assertEquals(0, response.position());
    assertEquals(snapshot.sizeInBytes(), response.unalignedRecords().sizeInBytes());
    UnalignedMemoryRecords memoryRecords = (UnalignedMemoryRecords) snapshot.slice(0, Math.toIntExact(snapshot.sizeInBytes()));
    assertEquals(memoryRecords.buffer(), ((UnalignedMemoryRecords) response.unalignedRecords()).buffer());
}

Example 8 with RawSnapshotReader

use of org.apache.kafka.snapshot.RawSnapshotReader in project kafka by apache.

the class KafkaRaftClientSnapshotTest method testFetchSnapshotRequestWithInvalidPosition.

@Test
public void testFetchSnapshotRequestWithInvalidPosition() throws Exception {
    int localId = 0;
    Set<Integer> voters = Utils.mkSet(localId, localId + 1);
    OffsetAndEpoch snapshotId = new OffsetAndEpoch(1, 1);
    List<String> records = Arrays.asList("foo", "bar");
    RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).appendToLog(snapshotId.epoch, Arrays.asList("a")).build();
    context.becomeLeader();
    int epoch = context.currentEpoch();
    context.advanceLocalLeaderHighWatermarkToLogEndOffset();
    try (SnapshotWriter<String> snapshot = context.client.createSnapshot(snapshotId.offset - 1, snapshotId.epoch, 0).get()) {
        assertEquals(snapshotId, snapshot.snapshotId());
        snapshot.append(records);
        snapshot.freeze();
    }
    context.deliverRequest(fetchSnapshotRequest(context.metadataPartition, epoch, snapshotId, Integer.MAX_VALUE, -1));
    context.client.poll();
    FetchSnapshotResponseData.PartitionSnapshot response = context.assertSentFetchSnapshotResponse(context.metadataPartition).get();
    assertEquals(Errors.POSITION_OUT_OF_RANGE, Errors.forCode(response.errorCode()));
    assertEquals(epoch, response.currentLeader().leaderEpoch());
    assertEquals(localId, response.currentLeader().leaderId());
    RawSnapshotReader snapshot = context.log.readSnapshot(snapshotId).get();
    context.deliverRequest(fetchSnapshotRequest(context.metadataPartition, epoch, snapshotId, Integer.MAX_VALUE, snapshot.sizeInBytes()));
    context.client.poll();
    response = context.assertSentFetchSnapshotResponse(context.metadataPartition).get();
    assertEquals(Errors.POSITION_OUT_OF_RANGE, Errors.forCode(response.errorCode()));
    assertEquals(epoch, response.currentLeader().leaderEpoch());
    assertEquals(localId, response.currentLeader().leaderId());
}

Example 9 with RawSnapshotReader

use of org.apache.kafka.snapshot.RawSnapshotReader in project kafka by apache.

the class QuorumControllerTest method testSnapshotSaveAndLoad.

@Test
public void testSnapshotSaveAndLoad() throws Throwable {
    final int numBrokers = 4;
    Map<Integer, Long> brokerEpochs = new HashMap<>();
    RawSnapshotReader reader = null;
    Uuid fooId;
    try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(3, Optional.empty())) {
        try (QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) {
            QuorumController active = controlEnv.activeController();
            for (int i = 0; i < numBrokers; i++) {
                BrokerRegistrationReply reply = active.registerBroker(new BrokerRegistrationRequestData().setBrokerId(i).setRack(null).setClusterId(active.clusterId()).setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB" + i)).setListeners(new ListenerCollection(Arrays.asList(new Listener().setName("PLAINTEXT").setHost("localhost").setPort(9092 + i)).iterator()))).get();
                brokerEpochs.put(i, reply.epoch());
            }
            for (int i = 0; i < numBrokers - 1; i++) {
                assertEquals(new BrokerHeartbeatReply(true, false, false, false), active.processBrokerHeartbeat(new BrokerHeartbeatRequestData().setWantFence(false).setBrokerEpoch(brokerEpochs.get(i)).setBrokerId(i).setCurrentMetadataOffset(100000L)).get());
            }
            CreateTopicsResponseData fooData = active.createTopics(new CreateTopicsRequestData().setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName("foo").setNumPartitions(-1).setReplicationFactor((short) -1).setAssignments(new CreatableReplicaAssignmentCollection(Arrays.asList(new CreatableReplicaAssignment().setPartitionIndex(0).setBrokerIds(Arrays.asList(0, 1, 2)), new CreatableReplicaAssignment().setPartitionIndex(1).setBrokerIds(Arrays.asList(1, 2, 0))).iterator()))).iterator()))).get();
            fooId = fooData.topics().find("foo").topicId();
            active.allocateProducerIds(new AllocateProducerIdsRequestData().setBrokerId(0).setBrokerEpoch(brokerEpochs.get(0))).get();
            long snapshotLogOffset = active.beginWritingSnapshot().get();
            reader = logEnv.waitForSnapshot(snapshotLogOffset);
            SnapshotReader<ApiMessageAndVersion> snapshot = createSnapshotReader(reader);
            assertEquals(snapshotLogOffset, snapshot.lastContainedLogOffset());
            checkSnapshotContent(expectedSnapshotContent(fooId, brokerEpochs), snapshot);
        }
    }
    try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(3, Optional.of(reader))) {
        try (QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) {
            QuorumController active = controlEnv.activeController();
            long snapshotLogOffset = active.beginWritingSnapshot().get();
            SnapshotReader<ApiMessageAndVersion> snapshot = createSnapshotReader(logEnv.waitForSnapshot(snapshotLogOffset));
            assertEquals(snapshotLogOffset, snapshot.lastContainedLogOffset());
            checkSnapshotContent(expectedSnapshotContent(fooId, brokerEpochs), snapshot);
        }
    }
}