use of org.opendaylight.controller.cluster.raft.persisted.SimpleReplicatedLogEntry in project controller by opendaylight.
the class ReplicationAndSnapshotsIntegrationTest method testFirstSnapshot.
/**
* Send a payload to the TestRaftActor to persist and replicate. Since snapshotBatchCount is set to
* 4 and we already have 3 entries in the journal log, this should initiate a snapshot. In this
* scenario, the follower consensus and application of state is delayed until after the snapshot
* completes.
*/
private void testFirstSnapshot() throws Exception {
testLog.info("testFirstSnapshot starting");
expSnapshotState.add(recoveredPayload0);
expSnapshotState.add(recoveredPayload1);
expSnapshotState.add(recoveredPayload2);
// Delay the consensus by temporarily dropping the AppendEntries to both followers.
follower1Actor.underlyingActor().startDropMessages(AppendEntries.class);
follower2Actor.underlyingActor().startDropMessages(AppendEntries.class);
// Send the payload.
payload3 = sendPayloadData(leaderActor, "three");
// Wait for snapshot complete.
MessageCollectorActor.expectFirstMatching(leaderCollectorActor, SaveSnapshotSuccess.class);
// The snapshot index should not be advanced nor the log trimmed because replicatedToAllIndex
// is behind due the followers not being replicated yet via AppendEntries.
assertEquals("Leader snapshot term", initialTerm, leaderContext.getReplicatedLog().getSnapshotTerm());
assertEquals("Leader snapshot index", 1, leaderContext.getReplicatedLog().getSnapshotIndex());
assertEquals("Leader journal log size", 2, leaderContext.getReplicatedLog().size());
assertEquals("Leader journal last index", 3, leaderContext.getReplicatedLog().lastIndex());
// Verify the persisted snapshot in the leader. This should reflect the advanced snapshot index as
// the last applied log entry (2) even though the leader hasn't yet advanced its cached snapshot index.
List<Snapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(leaderId, Snapshot.class);
assertEquals("Persisted snapshots size", 1, persistedSnapshots.size());
verifySnapshot("Persisted", persistedSnapshots.get(0), initialTerm, 2, currentTerm, 3);
List<ReplicatedLogEntry> unAppliedEntry = persistedSnapshots.get(0).getUnAppliedEntries();
assertEquals("Persisted Snapshot getUnAppliedEntries size", 1, unAppliedEntry.size());
verifyReplicatedLogEntry(unAppliedEntry.get(0), currentTerm, 3, payload3);
// The leader's persisted journal log should be cleared since we snapshotted.
List<SimpleReplicatedLogEntry> persistedLeaderJournal = InMemoryJournal.get(leaderId, SimpleReplicatedLogEntry.class);
assertEquals("Persisted journal log size", 0, persistedLeaderJournal.size());
// Allow AppendEntries to both followers to proceed. This should catch up the followers and cause a
// "fake" snapshot in the leader to advance the snapshot index to 2. Also the state should be applied
// in all members (via ApplyState).
follower1Actor.underlyingActor().stopDropMessages(AppendEntries.class);
follower2Actor.underlyingActor().stopDropMessages(AppendEntries.class);
ApplyState applyState = MessageCollectorActor.expectFirstMatching(leaderCollectorActor, ApplyState.class);
verifyApplyState(applyState, leaderCollectorActor, payload3.toString(), currentTerm, 3, payload3);
verifyApplyJournalEntries(leaderCollectorActor, 3);
assertEquals("Leader commit index", 3, leaderContext.getCommitIndex());
applyState = MessageCollectorActor.expectFirstMatching(follower1CollectorActor, ApplyState.class);
verifyApplyState(applyState, null, null, currentTerm, 3, payload3);
verifyApplyJournalEntries(follower1CollectorActor, 3);
applyState = MessageCollectorActor.expectFirstMatching(follower2CollectorActor, ApplyState.class);
verifyApplyState(applyState, null, null, currentTerm, 3, payload3);
verifyApplyJournalEntries(follower2CollectorActor, 3);
assertEquals("Leader snapshot term", initialTerm, leaderContext.getReplicatedLog().getSnapshotTerm());
assertEquals("Leader snapshot index", 2, leaderContext.getReplicatedLog().getSnapshotIndex());
assertEquals("Leader journal log size", 1, leaderContext.getReplicatedLog().size());
assertEquals("Leader commit index", 3, leaderContext.getCommitIndex());
assertEquals("Leader last applied", 3, leaderContext.getLastApplied());
assertEquals("Leader replicatedToAllIndex", 2, leader.getReplicatedToAllIndex());
// The followers should also snapshot so verify.
MessageCollectorActor.expectFirstMatching(follower1CollectorActor, SaveSnapshotSuccess.class);
persistedSnapshots = InMemorySnapshotStore.getSnapshots(follower1Id, Snapshot.class);
assertEquals("Persisted snapshots size", 1, persistedSnapshots.size());
// The last applied index in the snapshot may or may not be the last log entry depending on
// timing so to avoid intermittent test failures, we'll just verify the snapshot's last term/index.
assertEquals("Follower1 Snapshot getLastTerm", currentTerm, persistedSnapshots.get(0).getLastTerm());
assertEquals("Follower1 Snapshot getLastIndex", 3, persistedSnapshots.get(0).getLastIndex());
MessageCollectorActor.expectFirstMatching(follower2CollectorActor, SaveSnapshotSuccess.class);
MessageCollectorActor.clearMessages(leaderCollectorActor);
MessageCollectorActor.clearMessages(follower1CollectorActor);
MessageCollectorActor.clearMessages(follower2CollectorActor);
testLog.info("testFirstSnapshot ending");
}
use of org.opendaylight.controller.cluster.raft.persisted.SimpleReplicatedLogEntry in project controller by opendaylight.
the class ReplicationAndSnapshotsIntegrationTest method testSecondSnapshot.
/**
* Send one more payload to trigger another snapshot. In this scenario, we delay the snapshot until
* consensus occurs and the leader applies the state.
*/
private void testSecondSnapshot() throws Exception {
testLog.info("testSecondSnapshot starting");
expSnapshotState.add(payload3);
expSnapshotState.add(payload4);
expSnapshotState.add(payload5);
expSnapshotState.add(payload6);
// Delay the CaptureSnapshot message to the leader actor.
leaderActor.underlyingActor().startDropMessages(CaptureSnapshotReply.class);
// Send the payload.
payload7 = sendPayloadData(leaderActor, "seven");
// Capture the CaptureSnapshotReply message so we can send it later.
final CaptureSnapshotReply captureSnapshotReply = MessageCollectorActor.expectFirstMatching(leaderCollectorActor, CaptureSnapshotReply.class);
// Wait for the state to be applied in the leader.
ApplyState applyState = MessageCollectorActor.expectFirstMatching(leaderCollectorActor, ApplyState.class);
verifyApplyState(applyState, leaderCollectorActor, payload7.toString(), currentTerm, 7, payload7);
// At this point the leader has applied the new state but the cached snapshot index should not be
// advanced by a "fake" snapshot because we're in the middle of a snapshot. We'll wait for at least
// one more heartbeat AppendEntriesReply to ensure this does not occur.
MessageCollectorActor.clearMessages(leaderCollectorActor);
MessageCollectorActor.expectFirstMatching(leaderCollectorActor, AppendEntriesReply.class);
assertEquals("Leader snapshot term", currentTerm, leaderContext.getReplicatedLog().getSnapshotTerm());
assertEquals("Leader snapshot index", 5, leaderContext.getReplicatedLog().getSnapshotIndex());
assertEquals("Leader journal log size", 2, leaderContext.getReplicatedLog().size());
assertEquals("Leader journal last index", 7, leaderContext.getReplicatedLog().lastIndex());
assertEquals("Leader commit index", 7, leaderContext.getCommitIndex());
assertEquals("Leader last applied", 7, leaderContext.getLastApplied());
assertEquals("Leader replicatedToAllIndex", 5, leader.getReplicatedToAllIndex());
// Now deliver the CaptureSnapshotReply.
leaderActor.underlyingActor().stopDropMessages(CaptureSnapshotReply.class);
leaderActor.tell(captureSnapshotReply, leaderActor);
// Wait for snapshot complete.
MessageCollectorActor.expectFirstMatching(leaderCollectorActor, SaveSnapshotSuccess.class);
// Wait for another heartbeat AppendEntriesReply. This should cause a "fake" snapshot to advance the
// snapshot index and trimmed the log since we're no longer in a snapshot.
MessageCollectorActor.clearMessages(leaderCollectorActor);
MessageCollectorActor.expectFirstMatching(leaderCollectorActor, AppendEntriesReply.class);
assertEquals("Leader snapshot term", currentTerm, leaderContext.getReplicatedLog().getSnapshotTerm());
assertEquals("Leader snapshot index", 6, leaderContext.getReplicatedLog().getSnapshotIndex());
assertEquals("Leader journal log size", 1, leaderContext.getReplicatedLog().size());
assertEquals("Leader journal last index", 7, leaderContext.getReplicatedLog().lastIndex());
assertEquals("Leader commit index", 7, leaderContext.getCommitIndex());
// Verify the persisted snapshot. This should reflect the snapshot index as the last applied
// log entry (7) and shouldn't contain any unapplied entries as we capture persisted the snapshot data
// when the snapshot is created (ie when the CaptureSnapshot is processed).
List<Snapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(leaderId, Snapshot.class);
assertEquals("Persisted snapshots size", 1, persistedSnapshots.size());
verifySnapshot("Persisted", persistedSnapshots.get(0), currentTerm, 6, currentTerm, 7);
List<ReplicatedLogEntry> unAppliedEntry = persistedSnapshots.get(0).getUnAppliedEntries();
assertEquals("Persisted Snapshot getUnAppliedEntries size", 1, unAppliedEntry.size());
verifyReplicatedLogEntry(unAppliedEntry.get(0), currentTerm, 7, payload7);
// The leader's persisted journal log should be cleared since we did a snapshot.
List<SimpleReplicatedLogEntry> persistedLeaderJournal = InMemoryJournal.get(leaderId, SimpleReplicatedLogEntry.class);
assertEquals("Persisted journal log size", 0, persistedLeaderJournal.size());
// Verify the followers apply all 4 new log entries.
List<ApplyState> applyStates = MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, 4);
verifyApplyState(applyStates.get(0), null, null, currentTerm, 4, payload4);
verifyApplyState(applyStates.get(1), null, null, currentTerm, 5, payload5);
verifyApplyState(applyStates.get(2), null, null, currentTerm, 6, payload6);
verifyApplyState(applyStates.get(3), null, null, currentTerm, 7, payload7);
applyStates = MessageCollectorActor.expectMatching(follower2CollectorActor, ApplyState.class, 4);
verifyApplyState(applyStates.get(0), null, null, currentTerm, 4, payload4);
verifyApplyState(applyStates.get(1), null, null, currentTerm, 5, payload5);
verifyApplyState(applyStates.get(2), null, null, currentTerm, 6, payload6);
verifyApplyState(applyStates.get(3), null, null, currentTerm, 7, payload7);
// Verify the follower's snapshot index has also advanced. (after another AppendEntries heartbeat
// to be safe).
MessageCollectorActor.clearMessages(follower1CollectorActor);
MessageCollectorActor.expectFirstMatching(follower1CollectorActor, AppendEntries.class);
follower1Context = follower1Actor.underlyingActor().getRaftActorContext();
assertEquals("Follower 1 snapshot term", currentTerm, follower1Context.getReplicatedLog().getSnapshotTerm());
assertEquals("Follower 1 snapshot index", 6, follower1Context.getReplicatedLog().getSnapshotIndex());
assertEquals("Follower 1 journal log size", 1, follower1Context.getReplicatedLog().size());
assertEquals("Follower 1 journal last index", 7, follower1Context.getReplicatedLog().lastIndex());
assertEquals("Follower 1 commit index", 7, follower1Context.getCommitIndex());
MessageCollectorActor.clearMessages(follower2CollectorActor);
MessageCollectorActor.expectFirstMatching(follower2CollectorActor, AppendEntries.class);
follower2Context = follower2Actor.underlyingActor().getRaftActorContext();
assertEquals("Follower 2 snapshot term", currentTerm, follower2Context.getReplicatedLog().getSnapshotTerm());
assertEquals("Follower 2 snapshot index", 6, follower2Context.getReplicatedLog().getSnapshotIndex());
assertEquals("Follower 2 journal log size", 1, follower2Context.getReplicatedLog().size());
assertEquals("Follower 2 journal last index", 7, follower2Context.getReplicatedLog().lastIndex());
assertEquals("Follower 2 commit index", 7, follower2Context.getCommitIndex());
expSnapshotState.add(payload7);
testLog.info("testSecondSnapshot ending");
}
use of org.opendaylight.controller.cluster.raft.persisted.SimpleReplicatedLogEntry in project controller by opendaylight.
the class SnapshotManagerTest method testPersistWhenReplicatedLogSizeExceedsSnapshotBatchCount.
@Test
public void testPersistWhenReplicatedLogSizeExceedsSnapshotBatchCount() {
// matches snapshotBatchCount
doReturn(10L).when(mockReplicatedLog).size();
doReturn(100).when(mockReplicatedLog).dataSize();
doReturn(5L).when(mockReplicatedLog).getSnapshotIndex();
doReturn(5L).when(mockReplicatedLog).getSnapshotTerm();
long replicatedToAllIndex = 1;
ReplicatedLogEntry replicatedLogEntry = mock(ReplicatedLogEntry.class);
doReturn(replicatedLogEntry).when(mockReplicatedLog).get(replicatedToAllIndex);
doReturn(6L).when(replicatedLogEntry).getTerm();
doReturn(replicatedToAllIndex).when(replicatedLogEntry).getIndex();
snapshotManager.capture(new SimpleReplicatedLogEntry(9, 6, new MockRaftActorContext.MockPayload()), replicatedToAllIndex);
snapshotManager.persist(ByteState.empty(), Optional.empty(), 2000000L);
verify(mockDataPersistenceProvider).saveSnapshot(any(Snapshot.class));
verify(mockReplicatedLog).snapshotPreCommit(9L, 6L);
verify(mockRaftActorBehavior).setReplicatedToAllIndex(replicatedToAllIndex);
}
use of org.opendaylight.controller.cluster.raft.persisted.SimpleReplicatedLogEntry in project controller by opendaylight.
the class SnapshotManagerTest method testCallingRollbackMultipleTimesCausesNoHarm.
@Test
public void testCallingRollbackMultipleTimesCausesNoHarm() {
// when replicatedToAllIndex = -1
snapshotManager.capture(new SimpleReplicatedLogEntry(9, 6, new MockRaftActorContext.MockPayload()), -1);
snapshotManager.persist(ByteState.empty(), Optional.empty(), Runtime.getRuntime().totalMemory());
snapshotManager.rollback();
snapshotManager.rollback();
verify(mockReplicatedLog, times(1)).snapshotRollback();
}
use of org.opendaylight.controller.cluster.raft.persisted.SimpleReplicatedLogEntry in project controller by opendaylight.
the class SnapshotManagerTest method testPersistSendInstallSnapshot.
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testPersistSendInstallSnapshot() throws Exception {
doReturn(Integer.MAX_VALUE).when(mockReplicatedLog).dataSize();
doNothing().when(mockProcedure).accept(anyObject());
// when replicatedToAllIndex = -1
boolean capture = snapshotManager.captureToInstall(new SimpleReplicatedLogEntry(9, 6, new MockRaftActorContext.MockPayload()), -1, "follower-1");
assertTrue(capture);
ByteState snapshotState = ByteState.of(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 });
ArgumentCaptor<Optional> installSnapshotStreamCapture = ArgumentCaptor.forClass(Optional.class);
verify(mockProcedure).accept(installSnapshotStreamCapture.capture());
Optional<OutputStream> installSnapshotStream = installSnapshotStreamCapture.getValue();
assertEquals("isPresent", true, installSnapshotStream.isPresent());
installSnapshotStream.get().write(snapshotState.getBytes());
snapshotManager.persist(snapshotState, installSnapshotStream, Runtime.getRuntime().totalMemory());
assertEquals(true, snapshotManager.isCapturing());
verify(mockDataPersistenceProvider).saveSnapshot(any(Snapshot.class));
verify(mockReplicatedLog).snapshotPreCommit(9L, 6L);
ArgumentCaptor<SendInstallSnapshot> sendInstallSnapshotArgumentCaptor = ArgumentCaptor.forClass(SendInstallSnapshot.class);
verify(mockRaftActorBehavior).handleMessage(any(ActorRef.class), sendInstallSnapshotArgumentCaptor.capture());
SendInstallSnapshot sendInstallSnapshot = sendInstallSnapshotArgumentCaptor.getValue();
assertEquals("state", snapshotState, sendInstallSnapshot.getSnapshot().getState());
assertArrayEquals("state", snapshotState.getBytes(), sendInstallSnapshot.getSnapshotBytes().read());
}
Aggregations