use of org.opendaylight.controller.cluster.raft.persisted.ServerInfo in project controller by opendaylight.
the class RaftActorServerConfigurationSupportTest method testChangeToVotingWithNoLeaderAndElectionTimeout.
@Test
public void testChangeToVotingWithNoLeaderAndElectionTimeout() {
LOG.info("testChangeToVotingWithNoLeaderAndElectionTimeout starting");
final String node1ID = "node1";
final String node2ID = "node2";
final PeerAddressResolver peerAddressResolver = peerId -> peerId.equals(node1ID) ? actorFactory.createTestActorPath(node1ID) : peerId.equals(node2ID) ? actorFactory.createTestActorPath(node2ID) : null;
ServerConfigurationPayload persistedServerConfig = new ServerConfigurationPayload(Arrays.asList(new ServerInfo(node1ID, false), new ServerInfo(node2ID, true)));
SimpleReplicatedLogEntry persistedServerConfigEntry = new SimpleReplicatedLogEntry(0, 1, persistedServerConfig);
InMemoryJournal.addEntry(node1ID, 1, new UpdateElectionTerm(1, "node1"));
InMemoryJournal.addEntry(node1ID, 2, persistedServerConfigEntry);
InMemoryJournal.addEntry(node2ID, 1, new UpdateElectionTerm(1, "node1"));
InMemoryJournal.addEntry(node2ID, 2, persistedServerConfigEntry);
DefaultConfigParamsImpl configParams1 = new DefaultConfigParamsImpl();
configParams1.setHeartBeatInterval(new FiniteDuration(100, TimeUnit.MILLISECONDS));
configParams1.setElectionTimeoutFactor(1);
configParams1.setPeerAddressResolver(peerAddressResolver);
ActorRef node1Collector = actorFactory.createActor(MessageCollectorActor.props(), actorFactory.generateActorId("collector"));
TestActorRef<CollectingMockRaftActor> node1RaftActorRef = actorFactory.createTestActor(CollectingMockRaftActor.props(node1ID, ImmutableMap.<String, String>of(), configParams1, PERSISTENT, node1Collector).withDispatcher(Dispatchers.DefaultDispatcherId()), node1ID);
final CollectingMockRaftActor node1RaftActor = node1RaftActorRef.underlyingActor();
DefaultConfigParamsImpl configParams2 = new DefaultConfigParamsImpl();
configParams2.setElectionTimeoutFactor(1000000);
configParams2.setPeerAddressResolver(peerAddressResolver);
ActorRef node2Collector = actorFactory.createActor(MessageCollectorActor.props(), actorFactory.generateActorId("collector"));
TestActorRef<CollectingMockRaftActor> node2RaftActorRef = actorFactory.createTestActor(CollectingMockRaftActor.props(node2ID, ImmutableMap.<String, String>of(), configParams2, PERSISTENT, node2Collector).withDispatcher(Dispatchers.DefaultDispatcherId()), node2ID);
CollectingMockRaftActor node2RaftActor = node2RaftActorRef.underlyingActor();
// Send a ChangeServersVotingStatus message to node1 to change mode1 to voting. This should cause
// node1 to try to elect itself as leader in order to apply the new server config. But we'll drop
// RequestVote messages in node2 which should cause node1 to time out and revert back to the previous
// server config and fail with NO_LEADER. Note that node1 shouldn't forward the request to node2 b/c
// node2 was previously voting.
node2RaftActor.setDropMessageOfType(RequestVote.class);
ChangeServersVotingStatus changeServers = new ChangeServersVotingStatus(ImmutableMap.of(node1ID, true));
node1RaftActorRef.tell(changeServers, testKit.getRef());
ServerChangeReply reply = testKit.expectMsgClass(testKit.duration("5 seconds"), ServerChangeReply.class);
assertEquals("getStatus", ServerChangeStatus.NO_LEADER, reply.getStatus());
assertEquals("Server config", Sets.newHashSet(nonVotingServer(node1ID), votingServer(node2ID)), Sets.newHashSet(node1RaftActor.getRaftActorContext().getPeerServerInfo(true).getServerConfig()));
assertEquals("getRaftState", RaftState.Follower, node1RaftActor.getRaftState());
LOG.info("testChangeToVotingWithNoLeaderAndElectionTimeout ending");
}
use of org.opendaylight.controller.cluster.raft.persisted.ServerInfo in project controller by opendaylight.
the class LeadershipTransferIntegrationTest method createRaftActors.
private void createRaftActors() {
testLog.info("createRaftActors starting");
final Snapshot snapshot = Snapshot.create(EmptyState.INSTANCE, Collections.emptyList(), -1, -1, -1, -1, 1, null, new org.opendaylight.controller.cluster.raft.persisted.ServerConfigurationPayload(Arrays.asList(new ServerInfo(leaderId, true), new ServerInfo(follower1Id, true), new ServerInfo(follower2Id, true), new ServerInfo(follower3Id, false))));
InMemorySnapshotStore.addSnapshot(leaderId, snapshot);
InMemorySnapshotStore.addSnapshot(follower1Id, snapshot);
InMemorySnapshotStore.addSnapshot(follower2Id, snapshot);
InMemorySnapshotStore.addSnapshot(follower3Id, snapshot);
follower1NotifierActor = factory.createActor(MessageCollectorActor.props(), factory.generateActorId(follower1Id + "-notifier"));
follower1Actor = newTestRaftActor(follower1Id, TestRaftActor.newBuilder().peerAddresses(ImmutableMap.of(leaderId, testActorPath(leaderId), follower2Id, testActorPath(follower2Id), follower3Id, testActorPath(follower3Id))).config(newFollowerConfigParams()).roleChangeNotifier(follower1NotifierActor));
follower2NotifierActor = factory.createActor(MessageCollectorActor.props(), factory.generateActorId(follower2Id + "-notifier"));
follower2Actor = newTestRaftActor(follower2Id, TestRaftActor.newBuilder().peerAddresses(ImmutableMap.of(leaderId, testActorPath(leaderId), follower1Id, follower1Actor.path().toString(), follower3Id, testActorPath(follower3Id))).config(newFollowerConfigParams()).roleChangeNotifier(follower2NotifierActor));
follower3NotifierActor = factory.createActor(MessageCollectorActor.props(), factory.generateActorId(follower3Id + "-notifier"));
follower3Actor = newTestRaftActor(follower3Id, TestRaftActor.newBuilder().peerAddresses(ImmutableMap.of(leaderId, testActorPath(leaderId), follower1Id, follower1Actor.path().toString(), follower2Id, follower2Actor.path().toString())).config(newFollowerConfigParams()).roleChangeNotifier(follower3NotifierActor));
peerAddresses = ImmutableMap.<String, String>builder().put(follower1Id, follower1Actor.path().toString()).put(follower2Id, follower2Actor.path().toString()).put(follower3Id, follower3Actor.path().toString()).build();
leaderConfigParams = newLeaderConfigParams();
leaderConfigParams.setElectionTimeoutFactor(3);
leaderNotifierActor = factory.createActor(MessageCollectorActor.props(), factory.generateActorId(leaderId + "-notifier"));
leaderActor = newTestRaftActor(leaderId, TestRaftActor.newBuilder().peerAddresses(peerAddresses).config(leaderConfigParams).roleChangeNotifier(leaderNotifierActor));
follower1CollectorActor = follower1Actor.underlyingActor().collectorActor();
follower2CollectorActor = follower2Actor.underlyingActor().collectorActor();
follower3CollectorActor = follower3Actor.underlyingActor().collectorActor();
leaderCollectorActor = leaderActor.underlyingActor().collectorActor();
leaderContext = leaderActor.underlyingActor().getRaftActorContext();
waitUntilLeader(leaderActor);
testLog.info("createRaftActors starting");
}
use of org.opendaylight.controller.cluster.raft.persisted.ServerInfo in project controller by opendaylight.
the class InstallSnapshotTest method testSerialization.
@Test
public void testSerialization() {
byte[] data = new byte[1000];
for (int i = 0, j = 0; i < data.length; i++) {
data[i] = (byte) j;
if (++j >= 255) {
j = 0;
}
}
ServerConfigurationPayload serverConfig = new ServerConfigurationPayload(Arrays.asList(new ServerInfo("leader", true), new ServerInfo("follower", false)));
InstallSnapshot expected = new InstallSnapshot(3L, "leaderId", 11L, 2L, data, 5, 6, Optional.<Integer>of(54321), Optional.of(serverConfig));
Object serialized = expected.toSerializable(RaftVersions.CURRENT_VERSION);
assertEquals("Serialized type", InstallSnapshot.class, serialized.getClass());
InstallSnapshot actual = (InstallSnapshot) SerializationUtils.clone((Serializable) serialized);
verifyInstallSnapshot(expected, actual);
expected = new InstallSnapshot(3L, "leaderId", 11L, 2L, data, 5, 6);
actual = (InstallSnapshot) SerializationUtils.clone((Serializable) expected.toSerializable(RaftVersions.CURRENT_VERSION));
verifyInstallSnapshot(expected, actual);
}
use of org.opendaylight.controller.cluster.raft.persisted.ServerInfo in project controller by opendaylight.
the class ClusterAdminRpcServiceTest method testFlipMemberVotingStates.
@Test
public void testFlipMemberVotingStates() throws Exception {
String name = "testFlipMemberVotingStates";
ServerConfigurationPayload persistedServerConfig = new ServerConfigurationPayload(Arrays.asList(new ServerInfo("member-1", true), new ServerInfo("member-2", true), new ServerInfo("member-3", false)));
setupPersistedServerConfigPayload(persistedServerConfig, "member-1", name, "cars", "people");
setupPersistedServerConfigPayload(persistedServerConfig, "member-2", name, "cars", "people");
setupPersistedServerConfigPayload(persistedServerConfig, "member-3", name, "cars", "people");
String moduleShardsConfig = "module-shards-member1-and-2-and-3.conf";
final MemberNode leaderNode1 = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name).moduleShardsConfig(moduleShardsConfig).datastoreContextBuilder(DatastoreContext.newBuilder().shardHeartbeatIntervalInMillis(300).shardElectionTimeoutFactor(1)).build();
final MemberNode replicaNode2 = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name).moduleShardsConfig(moduleShardsConfig).build();
final MemberNode replicaNode3 = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name).moduleShardsConfig(moduleShardsConfig).build();
leaderNode1.configDataStore().waitTillReady();
leaderNode1.operDataStore().waitTillReady();
replicaNode3.configDataStore().waitTillReady();
replicaNode3.operDataStore().waitTillReady();
verifyVotingStates(leaderNode1.configDataStore(), "cars", new SimpleEntry<>("member-1", TRUE), new SimpleEntry<>("member-2", TRUE), new SimpleEntry<>("member-3", FALSE));
ClusterAdminRpcService service3 = new ClusterAdminRpcService(replicaNode3.configDataStore(), replicaNode3.operDataStore(), null);
RpcResult<FlipMemberVotingStatesForAllShardsOutput> rpcResult = service3.flipMemberVotingStatesForAllShards().get(10, TimeUnit.SECONDS);
FlipMemberVotingStatesForAllShardsOutput result = verifySuccessfulRpcResult(rpcResult);
verifyShardResults(result.getShardResult(), successShardResult("cars", DataStoreType.Config), successShardResult("people", DataStoreType.Config), successShardResult("cars", DataStoreType.Operational), successShardResult("people", DataStoreType.Operational));
verifyVotingStates(new AbstractDataStore[] { leaderNode1.configDataStore(), leaderNode1.operDataStore(), replicaNode2.configDataStore(), replicaNode2.operDataStore(), replicaNode3.configDataStore(), replicaNode3.operDataStore() }, new String[] { "cars", "people" }, new SimpleEntry<>("member-1", FALSE), new SimpleEntry<>("member-2", FALSE), new SimpleEntry<>("member-3", TRUE));
// Leadership should have transferred to member 3 since it is the only remaining voting member.
verifyRaftState(leaderNode1.configDataStore(), "cars", raftState -> {
assertNotNull("Expected non-null leader Id", raftState.getLeader());
assertTrue("Expected leader member-1. Actual: " + raftState.getLeader(), raftState.getLeader().contains("member-3"));
});
verifyRaftState(leaderNode1.operDataStore(), "cars", raftState -> {
assertNotNull("Expected non-null leader Id", raftState.getLeader());
assertTrue("Expected leader member-1. Actual: " + raftState.getLeader(), raftState.getLeader().contains("member-3"));
});
// Flip the voting states back to the original states.
rpcResult = service3.flipMemberVotingStatesForAllShards().get(10, TimeUnit.SECONDS);
result = verifySuccessfulRpcResult(rpcResult);
verifyShardResults(result.getShardResult(), successShardResult("cars", DataStoreType.Config), successShardResult("people", DataStoreType.Config), successShardResult("cars", DataStoreType.Operational), successShardResult("people", DataStoreType.Operational));
verifyVotingStates(new AbstractDataStore[] { leaderNode1.configDataStore(), leaderNode1.operDataStore(), replicaNode2.configDataStore(), replicaNode2.operDataStore(), replicaNode3.configDataStore(), replicaNode3.operDataStore() }, new String[] { "cars", "people" }, new SimpleEntry<>("member-1", TRUE), new SimpleEntry<>("member-2", TRUE), new SimpleEntry<>("member-3", FALSE));
// Leadership should have transferred to member 1 or 2.
verifyRaftState(leaderNode1.configDataStore(), "cars", raftState -> {
assertNotNull("Expected non-null leader Id", raftState.getLeader());
assertTrue("Expected leader member-1 or member-2. Actual: " + raftState.getLeader(), raftState.getLeader().contains("member-1") || raftState.getLeader().contains("member-2"));
});
}
use of org.opendaylight.controller.cluster.raft.persisted.ServerInfo in project controller by opendaylight.
the class NonVotingFollowerIntegrationTest method testFollowerResyncWithMoreLeaderLogEntriesAndDownPeerAfterNonPersistentLeaderRestart.
/**
* Tests non-voting follower re-sync after the non-persistent leader restarts and commits new log
* entries prior to re-connecting to the follower. The leader's last index will be greater than the
* follower's last index corresponding to the previous data retained in memory. So the follower's log
* will be behind the leader's log but the leader's log entries will have a higher term. It also adds a
* "down" peer on restart so the leader doesn't trim its log as it's trying to resync the follower.
* Eventually the follower should force the leader to install snapshot to re-sync its state.
*/
@Test
public void testFollowerResyncWithMoreLeaderLogEntriesAndDownPeerAfterNonPersistentLeaderRestart() {
testLog.info("testFollowerResyncWithMoreLeaderLogEntriesAndDownPeerAfterNonPersistentLeaderRestart starting");
setupLeaderAndNonVotingFollower();
// Add log entries and verify they are committed and applied by both nodes.
expSnapshotState.add(sendPayloadData(leaderActor, "zero"));
expSnapshotState.add(sendPayloadData(leaderActor, "one"));
expSnapshotState.add(sendPayloadData(leaderActor, "two"));
MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, expSnapshotState.size());
MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, expSnapshotState.size());
long lastIndex = 2;
assertEquals("Leader journal lastIndex", lastIndex, leaderContext.getReplicatedLog().lastIndex());
assertEquals("Leader commit index", lastIndex, leaderContext.getCommitIndex());
assertEquals("Follower journal lastIndex", lastIndex, follower1Context.getReplicatedLog().lastIndex());
assertEquals("Follower commit index", lastIndex, follower1Context.getCommitIndex());
assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
MessageCollectorActor.clearMessages(follower1CollectorActor);
MessageCollectorActor.expectFirstMatching(follower1CollectorActor, AppendEntries.class);
assertEquals("Follower snapshot index", lastIndex - 1, follower1Context.getReplicatedLog().getSnapshotIndex());
assertEquals("Follower journal size", 1, leaderContext.getReplicatedLog().size());
// Restart the leader
killActor(leaderActor);
MessageCollectorActor.clearMessages(follower1CollectorActor);
// Temporarily drop AppendEntries to simulate a disconnect when the leader restarts.
followerInstance.startDropMessages(AppendEntries.class);
// Add a "down" peer so the leader doesn't trim its log as it's trying to resync the follower. The
// leader will keep decrementing the follower's nextIndex to try to find a matching index. Since
// there is no matching index it will eventually hit index 0 which should cause the follower to
// force an install snapshot upon failure to remove the conflicting indexes due to indexes 0 and 1
// being in the prior snapshot and not the log.
//
// We also add another voting follower actor into the mix even though it shoildn't affect the
// outcome.
ServerConfigurationPayload persistedServerConfig = new ServerConfigurationPayload(Arrays.asList(new ServerInfo(leaderId, true), new ServerInfo(follower1Id, false), new ServerInfo(follower2Id, true), new ServerInfo("downPeer", false)));
SimpleReplicatedLogEntry persistedServerConfigEntry = new SimpleReplicatedLogEntry(0, currentTerm, persistedServerConfig);
InMemoryJournal.clear();
InMemoryJournal.addEntry(leaderId, 1, new UpdateElectionTerm(currentTerm, leaderId));
InMemoryJournal.addEntry(leaderId, 2, persistedServerConfigEntry);
InMemoryJournal.addEntry(follower2Id, 1, persistedServerConfigEntry);
DefaultConfigParamsImpl follower2ConfigParams = newFollowerConfigParams();
follower2ConfigParams.setCustomRaftPolicyImplementationClass(DisableElectionsRaftPolicy.class.getName());
follower2Actor = newTestRaftActor(follower2Id, TestRaftActor.newBuilder().peerAddresses(ImmutableMap.of(leaderId, testActorPath(leaderId), follower1Id, follower1Actor.path().toString())).config(follower2ConfigParams).persistent(Optional.of(false)));
TestRaftActor follower2Instance = follower2Actor.underlyingActor();
follower2Instance.waitForRecoveryComplete();
follower2CollectorActor = follower2Instance.collectorActor();
peerAddresses = ImmutableMap.of(follower1Id, follower1Actor.path().toString(), follower2Id, follower2Actor.path().toString());
createNewLeaderActor();
currentTerm++;
assertEquals("Leader term", currentTerm, leaderContext.getTermInformation().getCurrentTerm());
assertEquals("Leader journal lastIndex", -1, leaderContext.getReplicatedLog().lastIndex());
assertEquals("Leader commit index", -1, leaderContext.getCommitIndex());
// Add new log entries to the leader - several more than the prior log entries
expSnapshotState.add(sendPayloadData(leaderActor, "zero-1"));
expSnapshotState.add(sendPayloadData(leaderActor, "one-1"));
expSnapshotState.add(sendPayloadData(leaderActor, "two-1"));
expSnapshotState.add(sendPayloadData(leaderActor, "three-1"));
expSnapshotState.add(sendPayloadData(leaderActor, "four-1"));
MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, expSnapshotState.size());
MessageCollectorActor.expectMatching(follower2CollectorActor, ApplyState.class, expSnapshotState.size());
lastIndex = 4;
assertEquals("Leader journal lastIndex", lastIndex, leaderContext.getReplicatedLog().lastIndex());
assertEquals("Leader commit index", lastIndex, leaderContext.getCommitIndex());
assertEquals("Leader snapshot index", -1, leaderContext.getReplicatedLog().getSnapshotIndex());
assertEquals("Leader replicatedToAllIndex", -1, leaderInstance.getCurrentBehavior().getReplicatedToAllIndex());
// Re-enable AppendEntries to the follower. The follower's log will be out of sync and it should
// should force the leader to install snapshot to re-sync the entire follower's log and state.
followerInstance.stopDropMessages(AppendEntries.class);
MessageCollectorActor.expectFirstMatching(follower1CollectorActor, SnapshotComplete.class);
assertEquals("Follower term", currentTerm, follower1Context.getTermInformation().getCurrentTerm());
assertEquals("Follower journal lastIndex", lastIndex, follower1Context.getReplicatedLog().lastIndex());
assertEquals("Follower journal lastTerm", currentTerm, follower1Context.getReplicatedLog().lastTerm());
assertEquals("Follower commit index", lastIndex, follower1Context.getCommitIndex());
assertEquals("Follower applied state", expSnapshotState, followerInstance.getState());
testLog.info("testFollowerResyncWithMoreLeaderLogEntriesAndDownPeerAfterNonPersistentLeaderRestart ending");
}
Aggregations