use of org.opendaylight.controller.cluster.raft.persisted.UpdateElectionTerm in project controller by opendaylight.
the class FollowerTest method testCaptureSnapshotOnMiddleEntryInAppendEntries.
@Test
public void testCaptureSnapshotOnMiddleEntryInAppendEntries() {
String id = "testCaptureSnapshotOnMiddleEntryInAppendEntries";
logStart(id);
InMemoryJournal.addEntry(id, 1, new UpdateElectionTerm(1, null));
DefaultConfigParamsImpl config = new DefaultConfigParamsImpl();
config.setSnapshotBatchCount(2);
config.setCustomRaftPolicyImplementationClass(DisableElectionsRaftPolicy.class.getName());
final AtomicReference<MockRaftActor> followerRaftActor = new AtomicReference<>();
RaftActorSnapshotCohort snapshotCohort = newRaftActorSnapshotCohort(followerRaftActor);
Builder builder = MockRaftActor.builder().persistent(Optional.of(true)).id(id).peerAddresses(ImmutableMap.of("leader", "")).config(config).snapshotCohort(snapshotCohort);
TestActorRef<MockRaftActor> followerActorRef = actorFactory.createTestActor(builder.props().withDispatcher(Dispatchers.DefaultDispatcherId()), id);
followerRaftActor.set(followerActorRef.underlyingActor());
followerRaftActor.get().waitForInitializeBehaviorComplete();
InMemorySnapshotStore.addSnapshotSavedLatch(id);
InMemoryJournal.addDeleteMessagesCompleteLatch(id);
InMemoryJournal.addWriteMessagesCompleteLatch(id, 1, ApplyJournalEntries.class);
List<ReplicatedLogEntry> entries = Arrays.asList(newReplicatedLogEntry(1, 0, "one"), newReplicatedLogEntry(1, 1, "two"), newReplicatedLogEntry(1, 2, "three"));
AppendEntries appendEntries = new AppendEntries(1, "leader", -1, -1, entries, 2, -1, (short) 0);
followerActorRef.tell(appendEntries, leaderActor);
AppendEntriesReply reply = MessageCollectorActor.expectFirstMatching(leaderActor, AppendEntriesReply.class);
assertEquals("isSuccess", true, reply.isSuccess());
final Snapshot snapshot = InMemorySnapshotStore.waitForSavedSnapshot(id, Snapshot.class);
InMemoryJournal.waitForDeleteMessagesComplete(id);
InMemoryJournal.waitForWriteMessagesComplete(id);
// We expect the ApplyJournalEntries for index 2 to remain in the persisted log b/c it's still queued for
// persistence by the time we initiate capture so the last persisted journal sequence number doesn't include it.
// This is OK - on recovery it will be a no-op since index 2 has already been applied.
List<Object> journalEntries = InMemoryJournal.get(id, Object.class);
assertEquals("Persisted journal entries size: " + journalEntries, 1, journalEntries.size());
assertEquals("Persisted journal entry type", ApplyJournalEntries.class, journalEntries.get(0).getClass());
assertEquals("ApplyJournalEntries index", 2, ((ApplyJournalEntries) journalEntries.get(0)).getToIndex());
assertEquals("Snapshot unapplied size", 0, snapshot.getUnAppliedEntries().size());
assertEquals("Snapshot getLastAppliedTerm", 1, snapshot.getLastAppliedTerm());
assertEquals("Snapshot getLastAppliedIndex", 2, snapshot.getLastAppliedIndex());
assertEquals("Snapshot getLastTerm", 1, snapshot.getLastTerm());
assertEquals("Snapshot getLastIndex", 2, snapshot.getLastIndex());
assertEquals("Snapshot state", ImmutableList.of(entries.get(0).getData(), entries.get(1).getData(), entries.get(2).getData()), MockRaftActor.fromState(snapshot.getState()));
assertEquals("Journal size", 0, followerRaftActor.get().getReplicatedLog().size());
assertEquals("Snapshot index", 2, followerRaftActor.get().getReplicatedLog().getSnapshotIndex());
// Reinstate the actor from persistence
actorFactory.killActor(followerActorRef, new TestKit(getSystem()));
followerActorRef = actorFactory.createTestActor(builder.props().withDispatcher(Dispatchers.DefaultDispatcherId()), id);
followerRaftActor.set(followerActorRef.underlyingActor());
followerRaftActor.get().waitForInitializeBehaviorComplete();
assertEquals("Journal size", 0, followerRaftActor.get().getReplicatedLog().size());
assertEquals("Last index", 2, followerRaftActor.get().getReplicatedLog().lastIndex());
assertEquals("Last applied index", 2, followerRaftActor.get().getRaftActorContext().getLastApplied());
assertEquals("Commit index", 2, followerRaftActor.get().getRaftActorContext().getCommitIndex());
assertEquals("State", ImmutableList.of(entries.get(0).getData(), entries.get(1).getData(), entries.get(2).getData()), followerRaftActor.get().getState());
}
use of org.opendaylight.controller.cluster.raft.persisted.UpdateElectionTerm in project controller by opendaylight.
the class NonVotingFollowerIntegrationTest method testFollowerResyncWithMoreLeaderLogEntriesAndDownPeerAfterNonPersistentLeaderRestart.
/**
* Tests non-voting follower re-sync after the non-persistent leader restarts and commits new log
* entries prior to re-connecting to the follower. The leader's last index will be greater than the
* follower's last index corresponding to the previous data retained in memory. So the follower's log
* will be behind the leader's log but the leader's log entries will have a higher term. It also adds a
* "down" peer on restart so the leader doesn't trim its log as it's trying to resync the follower.
* Eventually the follower should force the leader to install snapshot to re-sync its state.
*/
@Test
public void testFollowerResyncWithMoreLeaderLogEntriesAndDownPeerAfterNonPersistentLeaderRestart() {
testLog.info("testFollowerResyncWithMoreLeaderLogEntriesAndDownPeerAfterNonPersistentLeaderRestart starting");
setupLeaderAndNonVotingFollower();
// Add log entries and verify they are committed and applied by both nodes.
expSnapshotState.add(sendPayloadData(leaderActor, "zero"));
expSnapshotState.add(sendPayloadData(leaderActor, "one"));
expSnapshotState.add(sendPayloadData(leaderActor, "two"));
MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, expSnapshotState.size());
MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, expSnapshotState.size());
long lastIndex = 2;
assertEquals("Leader journal lastIndex", lastIndex, leaderContext.getReplicatedLog().lastIndex());
assertEquals("Leader commit index", lastIndex, leaderContext.getCommitIndex());
assertEquals("Follower journal lastIndex", lastIndex, follower1Context.getReplicatedLog().lastIndex());
assertEquals("Follower commit index", lastIndex, follower1Context.getCommitIndex());
assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
MessageCollectorActor.clearMessages(follower1CollectorActor);
MessageCollectorActor.expectFirstMatching(follower1CollectorActor, AppendEntries.class);
assertEquals("Follower snapshot index", lastIndex - 1, follower1Context.getReplicatedLog().getSnapshotIndex());
assertEquals("Follower journal size", 1, leaderContext.getReplicatedLog().size());
// Restart the leader
killActor(leaderActor);
MessageCollectorActor.clearMessages(follower1CollectorActor);
// Temporarily drop AppendEntries to simulate a disconnect when the leader restarts.
followerInstance.startDropMessages(AppendEntries.class);
// Add a "down" peer so the leader doesn't trim its log as it's trying to resync the follower. The
// leader will keep decrementing the follower's nextIndex to try to find a matching index. Since
// there is no matching index it will eventually hit index 0 which should cause the follower to
// force an install snapshot upon failure to remove the conflicting indexes due to indexes 0 and 1
// being in the prior snapshot and not the log.
//
// We also add another voting follower actor into the mix even though it shoildn't affect the
// outcome.
ServerConfigurationPayload persistedServerConfig = new ServerConfigurationPayload(Arrays.asList(new ServerInfo(leaderId, true), new ServerInfo(follower1Id, false), new ServerInfo(follower2Id, true), new ServerInfo("downPeer", false)));
SimpleReplicatedLogEntry persistedServerConfigEntry = new SimpleReplicatedLogEntry(0, currentTerm, persistedServerConfig);
InMemoryJournal.clear();
InMemoryJournal.addEntry(leaderId, 1, new UpdateElectionTerm(currentTerm, leaderId));
InMemoryJournal.addEntry(leaderId, 2, persistedServerConfigEntry);
InMemoryJournal.addEntry(follower2Id, 1, persistedServerConfigEntry);
DefaultConfigParamsImpl follower2ConfigParams = newFollowerConfigParams();
follower2ConfigParams.setCustomRaftPolicyImplementationClass(DisableElectionsRaftPolicy.class.getName());
follower2Actor = newTestRaftActor(follower2Id, TestRaftActor.newBuilder().peerAddresses(ImmutableMap.of(leaderId, testActorPath(leaderId), follower1Id, follower1Actor.path().toString())).config(follower2ConfigParams).persistent(Optional.of(false)));
TestRaftActor follower2Instance = follower2Actor.underlyingActor();
follower2Instance.waitForRecoveryComplete();
follower2CollectorActor = follower2Instance.collectorActor();
peerAddresses = ImmutableMap.of(follower1Id, follower1Actor.path().toString(), follower2Id, follower2Actor.path().toString());
createNewLeaderActor();
currentTerm++;
assertEquals("Leader term", currentTerm, leaderContext.getTermInformation().getCurrentTerm());
assertEquals("Leader journal lastIndex", -1, leaderContext.getReplicatedLog().lastIndex());
assertEquals("Leader commit index", -1, leaderContext.getCommitIndex());
// Add new log entries to the leader - several more than the prior log entries
expSnapshotState.add(sendPayloadData(leaderActor, "zero-1"));
expSnapshotState.add(sendPayloadData(leaderActor, "one-1"));
expSnapshotState.add(sendPayloadData(leaderActor, "two-1"));
expSnapshotState.add(sendPayloadData(leaderActor, "three-1"));
expSnapshotState.add(sendPayloadData(leaderActor, "four-1"));
MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, expSnapshotState.size());
MessageCollectorActor.expectMatching(follower2CollectorActor, ApplyState.class, expSnapshotState.size());
lastIndex = 4;
assertEquals("Leader journal lastIndex", lastIndex, leaderContext.getReplicatedLog().lastIndex());
assertEquals("Leader commit index", lastIndex, leaderContext.getCommitIndex());
assertEquals("Leader snapshot index", -1, leaderContext.getReplicatedLog().getSnapshotIndex());
assertEquals("Leader replicatedToAllIndex", -1, leaderInstance.getCurrentBehavior().getReplicatedToAllIndex());
// Re-enable AppendEntries to the follower. The follower's log will be out of sync and it should
// should force the leader to install snapshot to re-sync the entire follower's log and state.
followerInstance.stopDropMessages(AppendEntries.class);
MessageCollectorActor.expectFirstMatching(follower1CollectorActor, SnapshotComplete.class);
assertEquals("Follower term", currentTerm, follower1Context.getTermInformation().getCurrentTerm());
assertEquals("Follower journal lastIndex", lastIndex, follower1Context.getReplicatedLog().lastIndex());
assertEquals("Follower journal lastTerm", currentTerm, follower1Context.getReplicatedLog().lastTerm());
assertEquals("Follower commit index", lastIndex, follower1Context.getCommitIndex());
assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
testLog.info("testFollowerResyncWithMoreLeaderLogEntriesAndDownPeerAfterNonPersistentLeaderRestart ending");
}
use of org.opendaylight.controller.cluster.raft.persisted.UpdateElectionTerm in project controller by opendaylight.
the class NonVotingFollowerIntegrationTest method setupLeaderAndNonVotingFollower.
private void setupLeaderAndNonVotingFollower() {
snapshotBatchCount = 100;
int persistedTerm = 1;
// Set up a persisted ServerConfigurationPayload with the leader voting and the follower non-voting.
ServerConfigurationPayload persistedServerConfig = new ServerConfigurationPayload(Arrays.asList(new ServerInfo(leaderId, true), new ServerInfo(follower1Id, false)));
SimpleReplicatedLogEntry persistedServerConfigEntry = new SimpleReplicatedLogEntry(0, persistedTerm, persistedServerConfig);
InMemoryJournal.addEntry(leaderId, 1, new UpdateElectionTerm(persistedTerm, leaderId));
InMemoryJournal.addEntry(leaderId, 2, persistedServerConfigEntry);
InMemoryJournal.addEntry(follower1Id, 1, new UpdateElectionTerm(persistedTerm, leaderId));
InMemoryJournal.addEntry(follower1Id, 2, persistedServerConfigEntry);
DefaultConfigParamsImpl followerConfigParams = newFollowerConfigParams();
follower1Actor = newTestRaftActor(follower1Id, follower1Builder.peerAddresses(ImmutableMap.of(leaderId, testActorPath(leaderId))).config(followerConfigParams).persistent(Optional.of(false)));
peerAddresses = ImmutableMap.<String, String>builder().put(follower1Id, follower1Actor.path().toString()).build();
leaderConfigParams = newLeaderConfigParams();
leaderActor = newTestRaftActor(leaderId, TestRaftActor.newBuilder().peerAddresses(peerAddresses).config(leaderConfigParams).persistent(Optional.of(false)));
followerInstance = follower1Actor.underlyingActor();
follower1CollectorActor = followerInstance.collectorActor();
leaderInstance = leaderActor.underlyingActor();
leaderCollectorActor = leaderInstance.collectorActor();
leaderContext = leaderInstance.getRaftActorContext();
follower1Context = followerInstance.getRaftActorContext();
waitUntilLeader(leaderActor);
// Verify leader's context after startup
currentTerm = persistedTerm + 1;
assertEquals("Leader term", currentTerm, leaderContext.getTermInformation().getCurrentTerm());
assertEquals("Leader server config", Sets.newHashSet(persistedServerConfig.getServerConfig()), Sets.newHashSet(leaderContext.getPeerServerInfo(true).getServerConfig()));
assertEquals("Leader isVotingMember", true, leaderContext.isVotingMember());
// Verify follower's context after startup
MessageCollectorActor.expectFirstMatching(follower1CollectorActor, AppendEntries.class);
assertEquals("Follower term", currentTerm, follower1Context.getTermInformation().getCurrentTerm());
assertEquals("Follower server config", Sets.newHashSet(persistedServerConfig.getServerConfig()), Sets.newHashSet(follower1Context.getPeerServerInfo(true).getServerConfig()));
assertEquals("FollowerisVotingMember", false, follower1Context.isVotingMember());
}
use of org.opendaylight.controller.cluster.raft.persisted.UpdateElectionTerm in project controller by opendaylight.
the class RaftActorRecoverySupportTest method testNoDataRecoveredWithPersistenceDisabled.
@Test
public void testNoDataRecoveredWithPersistenceDisabled() {
doReturn(false).when(mockPersistence).isRecoveryApplicable();
sendMessageToSupport(new UpdateElectionTerm(5, "member2"));
assertEquals("Current term", 5, context.getTermInformation().getCurrentTerm());
assertEquals("Voted For", "member2", context.getTermInformation().getVotedFor());
sendMessageToSupport(RecoveryCompleted.getInstance(), true);
verify(mockCohort).getRestoreFromSnapshot();
verifyNoMoreInteractions(mockCohort, mockPersistentProvider);
}
use of org.opendaylight.controller.cluster.raft.persisted.UpdateElectionTerm in project controller by opendaylight.
the class RaftActorRecoverySupportTest method testUpdateElectionTerm.
@Test
public void testUpdateElectionTerm() {
sendMessageToSupport(new UpdateElectionTerm(5, "member2"));
assertEquals("Current term", 5, context.getTermInformation().getCurrentTerm());
assertEquals("Voted For", "member2", context.getTermInformation().getVotedFor());
}
Aggregations