Examples with InstallSnapshot org.opendaylight.controller.cluster.raft.messages.InstallSnapshot used on opensource projects

Example 16 with InstallSnapshot

use of org.opendaylight.controller.cluster.raft.messages.InstallSnapshot in project controller by opendaylight.

the class ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest method testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries.

/**
 * Send payloads to trigger a leader snapshot due to snapshotBatchCount reached with follower 2
 * lagging but not enough for the leader to trim its log from the last applied index. Follower 2's log
 * will be behind by several entries and, when it is resumed, it should be caught up via AppendEntries
 * sent by the leader.
 */
@Test
public void testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries() throws Exception {
    testLog.info("testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries starting");
    setup();
    sendInitialPayloadsReplicatedToAllFollowers("zero", "one");
    // Configure follower 2 to drop messages and lag.
    follower2Actor.underlyingActor().startDropMessages(AppendEntries.class);
    // Send the first payload and verify it gets applied by the leader and follower 1.
    MockPayload payload2 = sendPayloadData(leaderActor, "two");
    ApplyState applyState = MessageCollectorActor.expectFirstMatching(leaderCollectorActor, ApplyState.class);
    verifyApplyState(applyState, leaderCollectorActor, payload2.toString(), currentTerm, 2, payload2);
    applyState = MessageCollectorActor.expectFirstMatching(follower1CollectorActor, ApplyState.class);
    verifyApplyState(applyState, null, null, currentTerm, 2, payload2);
    expSnapshotState.add(payload2);
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.clearMessages(follower1CollectorActor);
    // Send another payload - this should cause a snapshot due to snapshotBatchCount reached.
    MockPayload payload3 = sendPayloadData(leaderActor, "three");
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, SaveSnapshotSuccess.class);
    testLog.info("testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries: sending 2 more payloads");
    // Send 2 more payloads - not enough to trigger another snapshot.
    MockPayload payload4 = sendPayloadData(leaderActor, "four");
    MockPayload payload5 = sendPayloadData(leaderActor, "five");
    // Verify the leader got consensus and applies each log entry even though follower 2 didn't respond.
    List<ApplyState> applyStates = MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 3);
    verifyApplyState(applyStates.get(0), leaderCollectorActor, payload3.toString(), currentTerm, 3, payload3);
    verifyApplyState(applyStates.get(1), leaderCollectorActor, payload4.toString(), currentTerm, 4, payload4);
    verifyApplyState(applyStates.get(2), leaderCollectorActor, payload5.toString(), currentTerm, 5, payload5);
    // Verify follower 1 applies each log entry.
    applyStates = MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, 3);
    verifyApplyState(applyStates.get(0), null, null, currentTerm, 3, payload3);
    verifyApplyState(applyStates.get(1), null, null, currentTerm, 4, payload4);
    verifyApplyState(applyStates.get(2), null, null, currentTerm, 5, payload5);
    // The snapshot should have caused the leader to advanced the snapshot index to the
    // last previously applied index (1) that was replicated to all followers at the time of capture.
    // Note: since the log size (3) did not exceed the snapshot batch count (4), the leader should not
    // have trimmed the log to the last index actually applied (5).
    assertEquals("Leader snapshot term", currentTerm, leaderContext.getReplicatedLog().getSnapshotTerm());
    assertEquals("Leader snapshot index", 1, leaderContext.getReplicatedLog().getSnapshotIndex());
    assertEquals("Leader journal log size", 4, leaderContext.getReplicatedLog().size());
    assertEquals("Leader journal last index", 5, leaderContext.getReplicatedLog().lastIndex());
    assertEquals("Leader commit index", 5, leaderContext.getCommitIndex());
    assertEquals("Leader last applied", 5, leaderContext.getLastApplied());
    assertEquals("Leader replicatedToAllIndex", 1, leader.getReplicatedToAllIndex());
    // Now stop dropping AppendEntries in follower 2.
    follower2Actor.underlyingActor().stopDropMessages(AppendEntries.class);
    // Verify follower 2 applies each log entry. The leader should not install a snapshot b/c
    // follower 2's next index (3) is still present in the log.
    applyStates = MessageCollectorActor.expectMatching(follower2CollectorActor, ApplyState.class, 4);
    verifyApplyState(applyStates.get(0), null, null, currentTerm, 2, payload2);
    verifyApplyState(applyStates.get(1), null, null, currentTerm, 3, payload3);
    verifyApplyState(applyStates.get(2), null, null, currentTerm, 4, payload4);
    verifyApplyState(applyStates.get(3), null, null, currentTerm, 5, payload5);
    // Verify the leader did not try to install a snapshot to catch up follower 2.
    InstallSnapshot installSnapshot = MessageCollectorActor.getFirstMatching(follower2CollectorActor, InstallSnapshot.class);
    Assert.assertNull("Follower 2 received unexpected InstallSnapshot", installSnapshot);
    // Ensure there's at least 1 more heartbeat.
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, AppendEntriesReply.class);
    // The leader should now have performed fake snapshots to advance the snapshot index and to trim
    // the log. In addition replicatedToAllIndex should've advanced.
    verifyLeadersTrimmedLog(5);
    // Verify the leader's persisted snapshot.
    List<Snapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(leaderId, Snapshot.class);
    assertEquals("Persisted snapshots size", 1, persistedSnapshots.size());
    verifySnapshot("Persisted", persistedSnapshots.get(0), currentTerm, 2, currentTerm, 3);
    List<ReplicatedLogEntry> unAppliedEntry = persistedSnapshots.get(0).getUnAppliedEntries();
    assertEquals("Persisted Snapshot getUnAppliedEntries size", 1, unAppliedEntry.size());
    verifyReplicatedLogEntry(unAppliedEntry.get(0), currentTerm, 3, payload3);
    // Verify follower 1's log and snapshot indexes.
    MessageCollectorActor.clearMessages(follower1CollectorActor);
    MessageCollectorActor.expectFirstMatching(follower1CollectorActor, AppendEntries.class);
    verifyFollowersTrimmedLog(1, follower1Actor, 5);
    // Verify follower 2's log and snapshot indexes.
    MessageCollectorActor.clearMessages(follower2CollectorActor);
    MessageCollectorActor.expectFirstMatching(follower2CollectorActor, AppendEntries.class);
    verifyFollowersTrimmedLog(2, follower2Actor, 5);
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.clearMessages(follower1CollectorActor);
    MessageCollectorActor.clearMessages(follower2CollectorActor);
    expSnapshotState.add(payload3);
    expSnapshotState.add(payload4);
    expSnapshotState.add(payload5);
    testLog.info("testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries complete");
}

Example 17 with InstallSnapshot

use of org.opendaylight.controller.cluster.raft.messages.InstallSnapshot in project controller by opendaylight.

the class FollowerTest method testHandleOutOfSequenceInstallSnapshot.

@Test
public void testHandleOutOfSequenceInstallSnapshot() {
    logStart("testHandleOutOfSequenceInstallSnapshot");
    MockRaftActorContext context = createActorContext();
    follower = createBehavior(context);
    ByteString bsSnapshot = createSnapshot();
    InstallSnapshot installSnapshot = new InstallSnapshot(1, "leader", 3, 1, getNextChunk(bsSnapshot, 10, 50), 3, 3);
    follower.handleMessage(leaderActor, installSnapshot);
    InstallSnapshotReply reply = MessageCollectorActor.expectFirstMatching(leaderActor, InstallSnapshotReply.class);
    assertEquals("isSuccess", false, reply.isSuccess());
    assertEquals("getChunkIndex", -1, reply.getChunkIndex());
    assertEquals("getTerm", 1, reply.getTerm());
    assertEquals("getFollowerId", context.getId(), reply.getFollowerId());
    assertNull("Expected null SnapshotTracker", follower.getSnapshotTracker());
}

Example 18 with InstallSnapshot

use of org.opendaylight.controller.cluster.raft.messages.InstallSnapshot in project controller by opendaylight.

the class FollowerTest method testReceivingAppendEntriesDuringInstallSnapshotFromDifferentLeader.

@Test
public void testReceivingAppendEntriesDuringInstallSnapshotFromDifferentLeader() {
    logStart("testReceivingAppendEntriesDuringInstallSnapshotFromDifferentLeader");
    MockRaftActorContext context = createActorContext();
    follower = createBehavior(context);
    ByteString bsSnapshot = createSnapshot();
    int snapshotLength = bsSnapshot.size();
    int chunkSize = 50;
    int totalChunks = snapshotLength / chunkSize + (snapshotLength % chunkSize > 0 ? 1 : 0);
    int lastIncludedIndex = 1;
    // Check that snapshot installation is not in progress
    assertNull(follower.getSnapshotTracker());
    // Make sure that we have more than 1 chunk to send
    assertTrue(totalChunks > 1);
    // Send an install snapshot with the first chunk to start the process of installing a snapshot
    byte[] chunkData = getNextChunk(bsSnapshot, 0, chunkSize);
    follower.handleMessage(leaderActor, new InstallSnapshot(1, "leader", lastIncludedIndex, 1, chunkData, 1, totalChunks));
    // Check if snapshot installation is in progress now
    assertNotNull(follower.getSnapshotTracker());
    // Send appendEntries with a new term and leader.
    AppendEntries appendEntries = new AppendEntries(2, "new-leader", 1, 1, Arrays.asList(newReplicatedLogEntry(2, 2, "3")), 2, -1, (short) 1);
    follower.handleMessage(leaderActor, appendEntries);
    AppendEntriesReply reply = MessageCollectorActor.expectFirstMatching(leaderActor, AppendEntriesReply.class);
    assertEquals("isSuccess", true, reply.isSuccess());
    assertEquals("getLogLastIndex", 2, reply.getLogLastIndex());
    assertEquals("getLogLastTerm", 2, reply.getLogLastTerm());
    assertEquals("getTerm", 2, reply.getTerm());
    assertNull(follower.getSnapshotTracker());
}