use of org.opendaylight.controller.cluster.raft.persisted.UpdateElectionTerm in project controller by opendaylight.
the class RaftActorServerConfigurationSupportTest method testChangeToVotingWithNoLeaderAndForwardedToOtherNodeAfterElectionTimeout.
@Test
public void testChangeToVotingWithNoLeaderAndForwardedToOtherNodeAfterElectionTimeout() {
LOG.info("testChangeToVotingWithNoLeaderAndForwardedToOtherNodeAfterElectionTimeout starting");
final String node1ID = "node1";
final String node2ID = "node2";
final PeerAddressResolver peerAddressResolver = peerId -> peerId.equals(node1ID) ? actorFactory.createTestActorPath(node1ID) : peerId.equals(node2ID) ? actorFactory.createTestActorPath(node2ID) : null;
DefaultConfigParamsImpl configParams = new DefaultConfigParamsImpl();
configParams.setHeartBeatInterval(new FiniteDuration(100, TimeUnit.MILLISECONDS));
configParams.setElectionTimeoutFactor(3);
configParams.setPeerAddressResolver(peerAddressResolver);
ServerConfigurationPayload persistedServerConfig = new ServerConfigurationPayload(Arrays.asList(new ServerInfo(node1ID, false), new ServerInfo(node2ID, false)));
SimpleReplicatedLogEntry persistedServerConfigEntry = new SimpleReplicatedLogEntry(0, 1, persistedServerConfig);
InMemoryJournal.addEntry(node1ID, 1, new UpdateElectionTerm(1, "node1"));
InMemoryJournal.addEntry(node1ID, 2, persistedServerConfigEntry);
InMemoryJournal.addEntry(node2ID, 1, new UpdateElectionTerm(1, "node1"));
InMemoryJournal.addEntry(node2ID, 2, persistedServerConfigEntry);
InMemoryJournal.addEntry(node2ID, 3, new SimpleReplicatedLogEntry(1, 1, new MockRaftActorContext.MockPayload("2")));
InMemoryJournal.addEntry(node2ID, 4, new ApplyJournalEntries(1));
ActorRef node1Collector = actorFactory.createActor(MessageCollectorActor.props(), actorFactory.generateActorId("collector"));
TestActorRef<CollectingMockRaftActor> node1RaftActorRef = actorFactory.createTestActor(CollectingMockRaftActor.props(node1ID, ImmutableMap.<String, String>of(), configParams, PERSISTENT, node1Collector).withDispatcher(Dispatchers.DefaultDispatcherId()), node1ID);
final CollectingMockRaftActor node1RaftActor = node1RaftActorRef.underlyingActor();
ActorRef node2Collector = actorFactory.createActor(MessageCollectorActor.props(), actorFactory.generateActorId("collector"));
TestActorRef<CollectingMockRaftActor> node2RaftActorRef = actorFactory.createTestActor(CollectingMockRaftActor.props(node2ID, ImmutableMap.<String, String>of(), configParams, PERSISTENT, node2Collector).withDispatcher(Dispatchers.DefaultDispatcherId()), node2ID);
final CollectingMockRaftActor node2RaftActor = node2RaftActorRef.underlyingActor();
// Send a ChangeServersVotingStatus message to node1 to change mode1 to voting. This should cause
// node1 to try to elect itself as leader in order to apply the new server config. However node1's log
// is behind node2's so node2 should not grant node1's vote. This should cause node1 to time out and
// forward the request to node2.
ChangeServersVotingStatus changeServers = new ChangeServersVotingStatus(ImmutableMap.of(node1ID, true, node2ID, true));
node1RaftActorRef.tell(changeServers, testKit.getRef());
ServerChangeReply reply = testKit.expectMsgClass(testKit.duration("5 seconds"), ServerChangeReply.class);
assertEquals("getStatus", ServerChangeStatus.OK, reply.getStatus());
MessageCollectorActor.expectFirstMatching(node2Collector, ApplyJournalEntries.class);
verifyServerConfigurationPayloadEntry(node2RaftActor.getRaftActorContext().getReplicatedLog(), votingServer(node1ID), votingServer(node2ID));
assertEquals("getRaftState", RaftState.Leader, node2RaftActor.getRaftState());
MessageCollectorActor.expectFirstMatching(node1Collector, ApplyJournalEntries.class);
verifyServerConfigurationPayloadEntry(node1RaftActor.getRaftActorContext().getReplicatedLog(), votingServer(node1ID), votingServer(node2ID));
assertEquals("isVotingMember", true, node1RaftActor.getRaftActorContext().isVotingMember());
assertEquals("getRaftState", RaftState.Follower, node1RaftActor.getRaftState());
LOG.info("testChangeToVotingWithNoLeaderAndForwardedToOtherNodeAfterElectionTimeout ending");
}
use of org.opendaylight.controller.cluster.raft.persisted.UpdateElectionTerm in project controller by opendaylight.
the class RaftActorServerConfigurationSupportTest method testChangeToVotingWithNoLeader.
@Test
public void testChangeToVotingWithNoLeader() {
LOG.info("testChangeToVotingWithNoLeader starting");
DefaultConfigParamsImpl configParams = new DefaultConfigParamsImpl();
configParams.setHeartBeatInterval(new FiniteDuration(100, TimeUnit.MILLISECONDS));
configParams.setElectionTimeoutFactor(5);
final String node1ID = "node1";
final String node2ID = "node2";
// Set up a persisted ServerConfigurationPayload. Initially node1 and node2 will come up as non-voting.
// via the server config. The server config will also contain 2 voting peers that are down (ie no
// actors created).
ServerConfigurationPayload persistedServerConfig = new ServerConfigurationPayload(Arrays.asList(new ServerInfo(node1ID, false), new ServerInfo(node2ID, false), new ServerInfo("downNode1", true), new ServerInfo("downNode2", true)));
SimpleReplicatedLogEntry persistedServerConfigEntry = new SimpleReplicatedLogEntry(0, 1, persistedServerConfig);
InMemoryJournal.addEntry(node1ID, 1, new UpdateElectionTerm(1, "downNode1"));
InMemoryJournal.addEntry(node1ID, 2, persistedServerConfigEntry);
InMemoryJournal.addEntry(node1ID, 3, new ApplyJournalEntries(0));
InMemoryJournal.addEntry(node2ID, 1, new UpdateElectionTerm(1, "downNode2"));
InMemoryJournal.addEntry(node2ID, 2, persistedServerConfigEntry);
InMemoryJournal.addEntry(node2ID, 3, new ApplyJournalEntries(0));
ActorRef node1Collector = actorFactory.createActor(MessageCollectorActor.props(), actorFactory.generateActorId("collector"));
TestActorRef<CollectingMockRaftActor> node1RaftActorRef = actorFactory.createTestActor(CollectingMockRaftActor.props(node1ID, ImmutableMap.<String, String>of(), configParams, PERSISTENT, node1Collector).withDispatcher(Dispatchers.DefaultDispatcherId()), node1ID);
CollectingMockRaftActor node1RaftActor = node1RaftActorRef.underlyingActor();
ActorRef node2Collector = actorFactory.createActor(MessageCollectorActor.props(), actorFactory.generateActorId("collector"));
TestActorRef<CollectingMockRaftActor> node2RaftActorRef = actorFactory.createTestActor(CollectingMockRaftActor.props(node2ID, ImmutableMap.<String, String>of(), configParams, PERSISTENT, node2Collector).withDispatcher(Dispatchers.DefaultDispatcherId()), node2ID);
CollectingMockRaftActor node2RaftActor = node2RaftActorRef.underlyingActor();
node1RaftActor.waitForInitializeBehaviorComplete();
node2RaftActor.waitForInitializeBehaviorComplete();
// Verify the intended server config was loaded and applied.
verifyServerConfigurationPayloadEntry(node1RaftActor.getRaftActorContext().getReplicatedLog(), nonVotingServer(node1ID), nonVotingServer(node2ID), votingServer("downNode1"), votingServer("downNode2"));
assertEquals("isVotingMember", false, node1RaftActor.getRaftActorContext().isVotingMember());
assertEquals("getRaftState", RaftState.Follower, node1RaftActor.getRaftState());
assertEquals("getLeaderId", null, node1RaftActor.getLeaderId());
verifyServerConfigurationPayloadEntry(node2RaftActor.getRaftActorContext().getReplicatedLog(), nonVotingServer(node1ID), nonVotingServer(node2ID), votingServer("downNode1"), votingServer("downNode2"));
assertEquals("isVotingMember", false, node2RaftActor.getRaftActorContext().isVotingMember());
// For the test, we send a ChangeServersVotingStatus message to node1 to flip the voting states for
// each server, ie node1 and node2 to voting and the 2 down nodes to non-voting. This should cause
// node1 to try to elect itself as leader in order to apply the new server config. Since the 2
// down nodes are switched to non-voting, node1 should only need a vote from node2.
// First send the message such that node1 has no peer address for node2 - should fail.
ChangeServersVotingStatus changeServers = new ChangeServersVotingStatus(ImmutableMap.of(node1ID, true, node2ID, true, "downNode1", false, "downNode2", false));
node1RaftActorRef.tell(changeServers, testKit.getRef());
ServerChangeReply reply = testKit.expectMsgClass(testKit.duration("5 seconds"), ServerChangeReply.class);
assertEquals("getStatus", ServerChangeStatus.NO_LEADER, reply.getStatus());
assertEquals("getRaftState", RaftState.Follower, node1RaftActor.getRaftState());
// Send an AppendEntries so node1 has a leaderId
long term = node1RaftActor.getRaftActorContext().getTermInformation().getCurrentTerm();
node1RaftActorRef.tell(new AppendEntries(term, "downNode1", -1L, -1L, Collections.<ReplicatedLogEntry>emptyList(), 0, -1, (short) 1), ActorRef.noSender());
// Wait for the ElectionTimeout to clear the leaderId. The leaderId must be null so on the next
// ChangeServersVotingStatus message, it will try to elect a leader.
AbstractRaftActorIntegrationTest.verifyRaftState(node1RaftActorRef, rs -> assertEquals("getLeader", null, rs.getLeader()));
// Update node2's peer address and send the message again
node1RaftActor.setPeerAddress(node2ID, node2RaftActorRef.path().toString());
node1RaftActorRef.tell(changeServers, testKit.getRef());
reply = testKit.expectMsgClass(testKit.duration("5 seconds"), ServerChangeReply.class);
assertEquals("getStatus", ServerChangeStatus.OK, reply.getStatus());
ApplyJournalEntries apply = MessageCollectorActor.expectFirstMatching(node1Collector, ApplyJournalEntries.class);
assertEquals("getToIndex", 1, apply.getToIndex());
verifyServerConfigurationPayloadEntry(node1RaftActor.getRaftActorContext().getReplicatedLog(), votingServer(node1ID), votingServer(node2ID), nonVotingServer("downNode1"), nonVotingServer("downNode2"));
assertEquals("isVotingMember", true, node1RaftActor.getRaftActorContext().isVotingMember());
assertEquals("getRaftState", RaftState.Leader, node1RaftActor.getRaftState());
apply = MessageCollectorActor.expectFirstMatching(node2Collector, ApplyJournalEntries.class);
assertEquals("getToIndex", 1, apply.getToIndex());
verifyServerConfigurationPayloadEntry(node2RaftActor.getRaftActorContext().getReplicatedLog(), votingServer(node1ID), votingServer(node2ID), nonVotingServer("downNode1"), nonVotingServer("downNode2"));
assertEquals("isVotingMember", true, node2RaftActor.getRaftActorContext().isVotingMember());
assertEquals("getRaftState", RaftState.Follower, node2RaftActor.getRaftState());
LOG.info("testChangeToVotingWithNoLeader ending");
}
use of org.opendaylight.controller.cluster.raft.persisted.UpdateElectionTerm in project controller by opendaylight.
the class FollowerTest method testCaptureSnapshotOnLastEntryInAppendEntries.
@Test
public void testCaptureSnapshotOnLastEntryInAppendEntries() {
String id = "testCaptureSnapshotOnLastEntryInAppendEntries";
logStart(id);
InMemoryJournal.addEntry(id, 1, new UpdateElectionTerm(1, null));
DefaultConfigParamsImpl config = new DefaultConfigParamsImpl();
config.setSnapshotBatchCount(2);
config.setCustomRaftPolicyImplementationClass(DisableElectionsRaftPolicy.class.getName());
final AtomicReference<MockRaftActor> followerRaftActor = new AtomicReference<>();
RaftActorSnapshotCohort snapshotCohort = newRaftActorSnapshotCohort(followerRaftActor);
Builder builder = MockRaftActor.builder().persistent(Optional.of(true)).id(id).peerAddresses(ImmutableMap.of("leader", "")).config(config).snapshotCohort(snapshotCohort);
TestActorRef<MockRaftActor> followerActorRef = actorFactory.createTestActor(builder.props().withDispatcher(Dispatchers.DefaultDispatcherId()), id);
followerRaftActor.set(followerActorRef.underlyingActor());
followerRaftActor.get().waitForInitializeBehaviorComplete();
InMemorySnapshotStore.addSnapshotSavedLatch(id);
InMemoryJournal.addDeleteMessagesCompleteLatch(id);
InMemoryJournal.addWriteMessagesCompleteLatch(id, 1, ApplyJournalEntries.class);
List<ReplicatedLogEntry> entries = Arrays.asList(newReplicatedLogEntry(1, 0, "one"), newReplicatedLogEntry(1, 1, "two"));
AppendEntries appendEntries = new AppendEntries(1, "leader", -1, -1, entries, 1, -1, (short) 0);
followerActorRef.tell(appendEntries, leaderActor);
AppendEntriesReply reply = MessageCollectorActor.expectFirstMatching(leaderActor, AppendEntriesReply.class);
assertEquals("isSuccess", true, reply.isSuccess());
final Snapshot snapshot = InMemorySnapshotStore.waitForSavedSnapshot(id, Snapshot.class);
InMemoryJournal.waitForDeleteMessagesComplete(id);
InMemoryJournal.waitForWriteMessagesComplete(id);
// We expect the ApplyJournalEntries for index 1 to remain in the persisted log b/c it's still queued for
// persistence by the time we initiate capture so the last persisted journal sequence number doesn't include it.
// This is OK - on recovery it will be a no-op since index 1 has already been applied.
List<Object> journalEntries = InMemoryJournal.get(id, Object.class);
assertEquals("Persisted journal entries size: " + journalEntries, 1, journalEntries.size());
assertEquals("Persisted journal entry type", ApplyJournalEntries.class, journalEntries.get(0).getClass());
assertEquals("ApplyJournalEntries index", 1, ((ApplyJournalEntries) journalEntries.get(0)).getToIndex());
assertEquals("Snapshot unapplied size", 0, snapshot.getUnAppliedEntries().size());
assertEquals("Snapshot getLastAppliedTerm", 1, snapshot.getLastAppliedTerm());
assertEquals("Snapshot getLastAppliedIndex", 1, snapshot.getLastAppliedIndex());
assertEquals("Snapshot getLastTerm", 1, snapshot.getLastTerm());
assertEquals("Snapshot getLastIndex", 1, snapshot.getLastIndex());
assertEquals("Snapshot state", ImmutableList.of(entries.get(0).getData(), entries.get(1).getData()), MockRaftActor.fromState(snapshot.getState()));
}
use of org.opendaylight.controller.cluster.raft.persisted.UpdateElectionTerm in project controller by opendaylight.
the class ReplicationAndSnapshotsIntegrationTest method runTest.
@Test
public void runTest() throws Exception {
testLog.info("testReplicationAndSnapshots starting");
// Setup the persistent journal for the leader. We'll start up with 3 journal log entries (one less
// than the snapshotBatchCount).
long seqId = 1;
InMemoryJournal.addEntry(leaderId, seqId++, new UpdateElectionTerm(initialTerm, leaderId));
recoveredPayload0 = new MockPayload("zero");
InMemoryJournal.addEntry(leaderId, seqId++, new SimpleReplicatedLogEntry(0, initialTerm, recoveredPayload0));
recoveredPayload1 = new MockPayload("one");
InMemoryJournal.addEntry(leaderId, seqId++, new SimpleReplicatedLogEntry(1, initialTerm, recoveredPayload1));
recoveredPayload2 = new MockPayload("two");
InMemoryJournal.addEntry(leaderId, seqId++, new SimpleReplicatedLogEntry(2, initialTerm, recoveredPayload2));
InMemoryJournal.addEntry(leaderId, seqId++, new ApplyJournalEntries(2));
origLeaderJournal = InMemoryJournal.get(leaderId, SimpleReplicatedLogEntry.class);
// Create the leader and 2 follower actors and verify initial syncing of the followers after leader
// persistence recovery.
DefaultConfigParamsImpl followerConfigParams = newFollowerConfigParams();
followerConfigParams.setSnapshotBatchCount(snapshotBatchCount);
follower1Actor = newTestRaftActor(follower1Id, ImmutableMap.of(leaderId, testActorPath(leaderId), follower2Id, testActorPath(follower2Id)), followerConfigParams);
follower2Actor = newTestRaftActor(follower2Id, ImmutableMap.of(leaderId, testActorPath(leaderId), follower1Id, testActorPath(follower1Id)), followerConfigParams);
peerAddresses = ImmutableMap.<String, String>builder().put(follower1Id, follower1Actor.path().toString()).put(follower2Id, follower2Actor.path().toString()).build();
leaderConfigParams = newLeaderConfigParams();
leaderActor = newTestRaftActor(leaderId, peerAddresses, leaderConfigParams);
follower1CollectorActor = follower1Actor.underlyingActor().collectorActor();
follower2CollectorActor = follower2Actor.underlyingActor().collectorActor();
leaderCollectorActor = leaderActor.underlyingActor().collectorActor();
leaderContext = leaderActor.underlyingActor().getRaftActorContext();
verifyLeaderRecoveryAndInitialization();
testFirstSnapshot();
testSubsequentReplications();
testSecondSnapshot();
testLeaderReinstatement();
testLog.info("testReplicationAndSnapshots ending");
}
use of org.opendaylight.controller.cluster.raft.persisted.UpdateElectionTerm in project controller by opendaylight.
the class ClusterAdminRpcServiceTest method setupPersistedServerConfigPayload.
private static void setupPersistedServerConfigPayload(ServerConfigurationPayload serverConfig, String member, String datastoreTypeSuffix, String... shards) {
String[] datastoreTypes = { "config_", "oper_" };
for (String type : datastoreTypes) {
for (String shard : shards) {
List<ServerInfo> newServerInfo = new ArrayList<>(serverConfig.getServerConfig().size());
for (ServerInfo info : serverConfig.getServerConfig()) {
newServerInfo.add(new ServerInfo(ShardIdentifier.create(shard, MemberName.forName(info.getId()), type + datastoreTypeSuffix).toString(), info.isVoting()));
}
String shardID = ShardIdentifier.create(shard, MemberName.forName(member), type + datastoreTypeSuffix).toString();
InMemoryJournal.addEntry(shardID, 1, new UpdateElectionTerm(1, null));
InMemoryJournal.addEntry(shardID, 2, new SimpleReplicatedLogEntry(0, 1, new ServerConfigurationPayload(newServerInfo)));
}
}
}
Aggregations