use of org.opendaylight.controller.cluster.raft.persisted.Snapshot in project controller by opendaylight.
the class ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest method verifyReplicationsAndSnapshotWithNoLaggingAfterInstallSnapshot.
/**
* Do another round of payloads and snapshot to verify replicatedToAllIndex gets back on track and
* snapshots works as expected after doing a follower snapshot. In this step we don't lag a follower.
*/
private long verifyReplicationsAndSnapshotWithNoLaggingAfterInstallSnapshot() throws Exception {
testLog.info("verifyReplicationsAndSnapshotWithNoLaggingAfterInstallSnapshot starting: replicatedToAllIndex: {}", leader.getReplicatedToAllIndex());
// Send another payload - a snapshot should occur.
MockPayload payload4 = sendPayloadData(leaderActor, "four");
// Wait for the snapshot to complete.
MessageCollectorActor.expectFirstMatching(leaderCollectorActor, SaveSnapshotSuccess.class);
ApplyState applyState = MessageCollectorActor.expectFirstMatching(leaderCollectorActor, ApplyState.class);
verifyApplyState(applyState, leaderCollectorActor, payload4.toString(), currentTerm, 4, payload4);
// Verify the leader's last persisted snapshot (previous ones may not be purged yet).
List<Snapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(leaderId, Snapshot.class);
Snapshot persistedSnapshot = persistedSnapshots.get(persistedSnapshots.size() - 1);
// The last (fourth) payload may or may not have been applied when the snapshot is captured depending on the
// timing when the async persistence completes.
List<ReplicatedLogEntry> unAppliedEntry = persistedSnapshot.getUnAppliedEntries();
long leadersSnapshotIndex;
if (unAppliedEntry.isEmpty()) {
leadersSnapshotIndex = 4;
expSnapshotState.add(payload4);
verifySnapshot("Persisted", persistedSnapshot, currentTerm, 4, currentTerm, 4);
} else {
leadersSnapshotIndex = 3;
verifySnapshot("Persisted", persistedSnapshot, currentTerm, 3, currentTerm, 4);
assertEquals("Persisted Snapshot getUnAppliedEntries size", 1, unAppliedEntry.size());
verifyReplicatedLogEntry(unAppliedEntry.get(0), currentTerm, 4, payload4);
expSnapshotState.add(payload4);
}
// Send a couple more payloads.
MockPayload payload5 = sendPayloadData(leaderActor, "five");
MockPayload payload6 = sendPayloadData(leaderActor, "six");
// Verify the leader applies the 2 log entries.
List<ApplyState> applyStates = MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 3);
verifyApplyState(applyStates.get(1), leaderCollectorActor, payload5.toString(), currentTerm, 5, payload5);
verifyApplyState(applyStates.get(2), leaderCollectorActor, payload6.toString(), currentTerm, 6, payload6);
// Verify the leader applies a log entry for at least the last entry index.
verifyApplyJournalEntries(leaderCollectorActor, 6);
// Ensure there's at least 1 more heartbeat to trim the log.
MessageCollectorActor.clearMessages(leaderCollectorActor);
MessageCollectorActor.expectFirstMatching(leaderCollectorActor, AppendEntriesReply.class);
// Verify the leader's final state.
verifyLeadersTrimmedLog(6);
InMemoryJournal.dumpJournal(leaderId);
// Verify the leaders's persisted journal log - it should only contain the last 2 ReplicatedLogEntries
// added after the snapshot as the persisted journal should've been purged to the snapshot
// sequence number.
verifyPersistedJournal(leaderId, Arrays.asList(new SimpleReplicatedLogEntry(5, currentTerm, payload5), new SimpleReplicatedLogEntry(6, currentTerm, payload6)));
// Verify the leaders's persisted journal contains an ApplyJournalEntries for at least the last entry index.
List<ApplyJournalEntries> persistedApplyJournalEntries = InMemoryJournal.get(leaderId, ApplyJournalEntries.class);
boolean found = false;
for (ApplyJournalEntries entry : persistedApplyJournalEntries) {
if (entry.getToIndex() == 6) {
found = true;
break;
}
}
Assert.assertTrue(String.format("ApplyJournalEntries with index %d not found in leader's persisted journal", 6), found);
// Verify follower 1 applies the 3 log entries.
applyStates = MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, 3);
verifyApplyState(applyStates.get(0), null, null, currentTerm, 4, payload4);
verifyApplyState(applyStates.get(1), null, null, currentTerm, 5, payload5);
verifyApplyState(applyStates.get(2), null, null, currentTerm, 6, payload6);
// Verify follower 1's log state.
verifyFollowersTrimmedLog(1, follower1Actor, 6);
// Verify follower 2 applies the 3 log entries.
applyStates = MessageCollectorActor.expectMatching(follower2CollectorActor, ApplyState.class, 3);
verifyApplyState(applyStates.get(0), null, null, currentTerm, 4, payload4);
verifyApplyState(applyStates.get(1), null, null, currentTerm, 5, payload5);
verifyApplyState(applyStates.get(2), null, null, currentTerm, 6, payload6);
// Verify follower 2's log state.
verifyFollowersTrimmedLog(2, follower2Actor, 6);
expSnapshotState.add(payload5);
expSnapshotState.add(payload6);
testLog.info("verifyReplicationsAndSnapshotWithNoLaggingAfterInstallSnapshot ending");
return leadersSnapshotIndex;
}
use of org.opendaylight.controller.cluster.raft.persisted.Snapshot in project controller by opendaylight.
the class ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest method testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries.
/**
* Send payloads to trigger a leader snapshot due to snapshotBatchCount reached with follower 2
* lagging but not enough for the leader to trim its log from the last applied index. Follower 2's log
* will be behind by several entries and, when it is resumed, it should be caught up via AppendEntries
* sent by the leader.
*/
@Test
public void testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries() throws Exception {
testLog.info("testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries starting");
setup();
sendInitialPayloadsReplicatedToAllFollowers("zero", "one");
// Configure follower 2 to drop messages and lag.
follower2Actor.underlyingActor().startDropMessages(AppendEntries.class);
// Send the first payload and verify it gets applied by the leader and follower 1.
MockPayload payload2 = sendPayloadData(leaderActor, "two");
ApplyState applyState = MessageCollectorActor.expectFirstMatching(leaderCollectorActor, ApplyState.class);
verifyApplyState(applyState, leaderCollectorActor, payload2.toString(), currentTerm, 2, payload2);
applyState = MessageCollectorActor.expectFirstMatching(follower1CollectorActor, ApplyState.class);
verifyApplyState(applyState, null, null, currentTerm, 2, payload2);
expSnapshotState.add(payload2);
MessageCollectorActor.clearMessages(leaderCollectorActor);
MessageCollectorActor.clearMessages(follower1CollectorActor);
// Send another payload - this should cause a snapshot due to snapshotBatchCount reached.
MockPayload payload3 = sendPayloadData(leaderActor, "three");
MessageCollectorActor.expectFirstMatching(leaderCollectorActor, SaveSnapshotSuccess.class);
testLog.info("testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries: sending 2 more payloads");
// Send 2 more payloads - not enough to trigger another snapshot.
MockPayload payload4 = sendPayloadData(leaderActor, "four");
MockPayload payload5 = sendPayloadData(leaderActor, "five");
// Verify the leader got consensus and applies each log entry even though follower 2 didn't respond.
List<ApplyState> applyStates = MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 3);
verifyApplyState(applyStates.get(0), leaderCollectorActor, payload3.toString(), currentTerm, 3, payload3);
verifyApplyState(applyStates.get(1), leaderCollectorActor, payload4.toString(), currentTerm, 4, payload4);
verifyApplyState(applyStates.get(2), leaderCollectorActor, payload5.toString(), currentTerm, 5, payload5);
// Verify follower 1 applies each log entry.
applyStates = MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, 3);
verifyApplyState(applyStates.get(0), null, null, currentTerm, 3, payload3);
verifyApplyState(applyStates.get(1), null, null, currentTerm, 4, payload4);
verifyApplyState(applyStates.get(2), null, null, currentTerm, 5, payload5);
// The snapshot should have caused the leader to advanced the snapshot index to the
// last previously applied index (1) that was replicated to all followers at the time of capture.
// Note: since the log size (3) did not exceed the snapshot batch count (4), the leader should not
// have trimmed the log to the last index actually applied (5).
assertEquals("Leader snapshot term", currentTerm, leaderContext.getReplicatedLog().getSnapshotTerm());
assertEquals("Leader snapshot index", 1, leaderContext.getReplicatedLog().getSnapshotIndex());
assertEquals("Leader journal log size", 4, leaderContext.getReplicatedLog().size());
assertEquals("Leader journal last index", 5, leaderContext.getReplicatedLog().lastIndex());
assertEquals("Leader commit index", 5, leaderContext.getCommitIndex());
assertEquals("Leader last applied", 5, leaderContext.getLastApplied());
assertEquals("Leader replicatedToAllIndex", 1, leader.getReplicatedToAllIndex());
// Now stop dropping AppendEntries in follower 2.
follower2Actor.underlyingActor().stopDropMessages(AppendEntries.class);
// Verify follower 2 applies each log entry. The leader should not install a snapshot b/c
// follower 2's next index (3) is still present in the log.
applyStates = MessageCollectorActor.expectMatching(follower2CollectorActor, ApplyState.class, 4);
verifyApplyState(applyStates.get(0), null, null, currentTerm, 2, payload2);
verifyApplyState(applyStates.get(1), null, null, currentTerm, 3, payload3);
verifyApplyState(applyStates.get(2), null, null, currentTerm, 4, payload4);
verifyApplyState(applyStates.get(3), null, null, currentTerm, 5, payload5);
// Verify the leader did not try to install a snapshot to catch up follower 2.
InstallSnapshot installSnapshot = MessageCollectorActor.getFirstMatching(follower2CollectorActor, InstallSnapshot.class);
Assert.assertNull("Follower 2 received unexpected InstallSnapshot", installSnapshot);
// Ensure there's at least 1 more heartbeat.
MessageCollectorActor.clearMessages(leaderCollectorActor);
MessageCollectorActor.expectFirstMatching(leaderCollectorActor, AppendEntriesReply.class);
// The leader should now have performed fake snapshots to advance the snapshot index and to trim
// the log. In addition replicatedToAllIndex should've advanced.
verifyLeadersTrimmedLog(5);
// Verify the leader's persisted snapshot.
List<Snapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(leaderId, Snapshot.class);
assertEquals("Persisted snapshots size", 1, persistedSnapshots.size());
verifySnapshot("Persisted", persistedSnapshots.get(0), currentTerm, 2, currentTerm, 3);
List<ReplicatedLogEntry> unAppliedEntry = persistedSnapshots.get(0).getUnAppliedEntries();
assertEquals("Persisted Snapshot getUnAppliedEntries size", 1, unAppliedEntry.size());
verifyReplicatedLogEntry(unAppliedEntry.get(0), currentTerm, 3, payload3);
// Verify follower 1's log and snapshot indexes.
MessageCollectorActor.clearMessages(follower1CollectorActor);
MessageCollectorActor.expectFirstMatching(follower1CollectorActor, AppendEntries.class);
verifyFollowersTrimmedLog(1, follower1Actor, 5);
// Verify follower 2's log and snapshot indexes.
MessageCollectorActor.clearMessages(follower2CollectorActor);
MessageCollectorActor.expectFirstMatching(follower2CollectorActor, AppendEntries.class);
verifyFollowersTrimmedLog(2, follower2Actor, 5);
MessageCollectorActor.clearMessages(leaderCollectorActor);
MessageCollectorActor.clearMessages(follower1CollectorActor);
MessageCollectorActor.clearMessages(follower2CollectorActor);
expSnapshotState.add(payload3);
expSnapshotState.add(payload4);
expSnapshotState.add(payload5);
testLog.info("testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries complete");
}
use of org.opendaylight.controller.cluster.raft.persisted.Snapshot in project controller by opendaylight.
the class Follower method processNewEntries.
private boolean processNewEntries(final AppendEntries appendEntries, final ActorRef sender) {
int numLogEntries = appendEntries.getEntries().size();
if (numLogEntries == 0) {
return true;
}
log.debug("{}: Number of entries to be appended = {}", logName(), numLogEntries);
long lastIndex = lastIndex();
int addEntriesFrom = 0;
// term), delete the existing entry and all that follow it (ยง5.3)
if (context.getReplicatedLog().size() > 0) {
// Find the entry up until the one that is not in the follower's log
for (int i = 0; i < numLogEntries; i++, addEntriesFrom++) {
ReplicatedLogEntry matchEntry = appendEntries.getEntries().get(i);
if (!isLogEntryPresent(matchEntry.getIndex())) {
// newEntry not found in the log
break;
}
long existingEntryTerm = getLogEntryTerm(matchEntry.getIndex());
log.debug("{}: matchEntry {} is present: existingEntryTerm: {}", logName(), matchEntry, existingEntryTerm);
// what the term was so we'll assume it matches.
if (existingEntryTerm == -1 || existingEntryTerm == matchEntry.getTerm()) {
continue;
}
if (!context.getRaftPolicy().applyModificationToStateBeforeConsensus()) {
log.info("{}: Removing entries from log starting at {}, commitIndex: {}, lastApplied: {}", logName(), matchEntry.getIndex(), context.getCommitIndex(), context.getLastApplied());
// been applied to the state yet.
if (matchEntry.getIndex() <= context.getLastApplied() || !context.getReplicatedLog().removeFromAndPersist(matchEntry.getIndex())) {
// Could not remove the entries - this means the matchEntry index must be in the
// snapshot and not the log. In this case the prior entries are part of the state
// so we must send back a reply to force a snapshot to completely re-sync the
// follower's log and state.
log.info("{}: Could not remove entries - sending reply to force snapshot", logName());
sender.tell(new AppendEntriesReply(context.getId(), currentTerm(), false, lastIndex, lastTerm(), context.getPayloadVersion(), true), actor());
return false;
}
break;
} else {
sender.tell(new AppendEntriesReply(context.getId(), currentTerm(), false, lastIndex, lastTerm(), context.getPayloadVersion(), true), actor());
return false;
}
}
}
lastIndex = lastIndex();
log.debug("{}: After cleanup, lastIndex: {}, entries to be added from: {}", logName(), lastIndex, addEntriesFrom);
// When persistence successfully completes for each new log entry appended, we need to determine if we
// should capture a snapshot to compact the persisted log. shouldCaptureSnapshot tracks whether or not
// one of the log entries has exceeded the log size threshold whereby a snapshot should be taken. However
// we don't initiate the snapshot at that log entry but rather after the last log entry has been persisted.
// This is done because subsequent log entries after the one that tripped the threshold may have been
// applied to the state already, as the persistence callback occurs async, and we want those entries
// purged from the persisted log as well.
final AtomicBoolean shouldCaptureSnapshot = new AtomicBoolean(false);
final Procedure<ReplicatedLogEntry> appendAndPersistCallback = logEntry -> {
final List<ReplicatedLogEntry> entries = appendEntries.getEntries();
final ReplicatedLogEntry lastEntryToAppend = entries.get(entries.size() - 1);
if (shouldCaptureSnapshot.get() && logEntry == lastEntryToAppend) {
context.getSnapshotManager().capture(context.getReplicatedLog().last(), getReplicatedToAllIndex());
}
};
// Append any new entries not already in the log
for (int i = addEntriesFrom; i < numLogEntries; i++) {
ReplicatedLogEntry entry = appendEntries.getEntries().get(i);
log.debug("{}: Append entry to log {}", logName(), entry.getData());
context.getReplicatedLog().appendAndPersist(entry, appendAndPersistCallback, false);
shouldCaptureSnapshot.compareAndSet(false, context.getReplicatedLog().shouldCaptureSnapshot(entry.getIndex()));
if (entry.getData() instanceof ServerConfigurationPayload) {
context.updatePeerIds((ServerConfigurationPayload) entry.getData());
}
}
log.debug("{}: Log size is now {}", logName(), context.getReplicatedLog().size());
return true;
}
use of org.opendaylight.controller.cluster.raft.persisted.Snapshot in project controller by opendaylight.
the class MigratedMessagesTest method doTestSnapshotAfterStartupWithMigratedMessage.
@SuppressWarnings("checkstyle:IllegalCatch")
private TestActorRef<MockRaftActor> doTestSnapshotAfterStartupWithMigratedMessage(String id, boolean persistent, Consumer<Snapshot> snapshotVerifier, final State snapshotState) {
InMemorySnapshotStore.addSnapshotSavedLatch(id);
InMemoryJournal.addDeleteMessagesCompleteLatch(id);
DefaultConfigParamsImpl config = new DefaultConfigParamsImpl();
config.setCustomRaftPolicyImplementationClass(DisableElectionsRaftPolicy.class.getName());
RaftActorSnapshotCohort snapshotCohort = new RaftActorSnapshotCohort() {
@Override
public void createSnapshot(ActorRef actorRef, java.util.Optional<OutputStream> installSnapshotStream) {
actorRef.tell(new CaptureSnapshotReply(snapshotState, installSnapshotStream), actorRef);
}
@Override
public void applySnapshot(State newState) {
}
@Override
public State deserializeSnapshot(ByteSource snapshotBytes) {
throw new UnsupportedOperationException();
}
};
TestActorRef<MockRaftActor> raftActorRef = factory.createTestActor(MockRaftActor.builder().id(id).config(config).snapshotCohort(snapshotCohort).persistent(Optional.of(persistent)).peerAddresses(ImmutableMap.of("peer", "")).props().withDispatcher(Dispatchers.DefaultDispatcherId()), id);
MockRaftActor mockRaftActor = raftActorRef.underlyingActor();
mockRaftActor.waitForRecoveryComplete();
Snapshot snapshot = InMemorySnapshotStore.waitForSavedSnapshot(id, Snapshot.class);
snapshotVerifier.accept(snapshot);
InMemoryJournal.waitForDeleteMessagesComplete(id);
assertEquals("InMemoryJournal size", 0, InMemoryJournal.get(id).size());
return raftActorRef;
}
use of org.opendaylight.controller.cluster.raft.persisted.Snapshot in project controller by opendaylight.
the class RaftActorRecoverySupport method possiblyRestoreFromSnapshot.
@SuppressWarnings("checkstyle:IllegalCatch")
private void possiblyRestoreFromSnapshot() {
Snapshot restoreFromSnapshot = cohort.getRestoreFromSnapshot();
if (restoreFromSnapshot == null) {
return;
}
if (anyDataRecovered) {
log.warn("{}: The provided restore snapshot was not applied because the persistence store is not empty", context.getId());
return;
}
log.debug("{}: Restore snapshot: {}", context.getId(), restoreFromSnapshot);
context.getSnapshotManager().apply(new ApplySnapshot(restoreFromSnapshot));
}
Aggregations