use of org.opendaylight.controller.cluster.datastore.entityownership.messages.RegisterListenerLocal in project controller by opendaylight.
the class DistributedEntityOwnershipService method registerListener.
@Override
public DOMEntityOwnershipListenerRegistration registerListener(final String entityType, final DOMEntityOwnershipListener listener) {
Preconditions.checkNotNull(entityType, "entityType cannot be null");
Preconditions.checkNotNull(listener, "listener cannot be null");
RegisterListenerLocal registerListener = new RegisterListenerLocal(listener, entityType);
LOG.debug("Registering listener with message: {}", registerListener);
executeLocalEntityOwnershipShardOperation(registerListener);
return new DistributedEntityOwnershipListenerRegistration(listener, entityType, this);
}
use of org.opendaylight.controller.cluster.datastore.entityownership.messages.RegisterListenerLocal in project controller by opendaylight.
the class EntityOwnershipShardTest method testListenerRegistration.
@Test
public void testListenerRegistration() throws Exception {
testLog.info("testListenerRegistration starting");
ShardTestKit kit = new ShardTestKit(getSystem());
ShardIdentifier leaderId = newShardId(LOCAL_MEMBER_NAME);
ShardIdentifier peerId = newShardId(PEER_MEMBER_1_NAME);
TestActorRef<TestEntityOwnershipShard> peer = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId, peerMap(leaderId.toString()), PEER_MEMBER_1_NAME)), peerId.toString());
peer.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
TestActorRef<EntityOwnershipShard> leader = actorFactory.createTestActor(newShardProps(leaderId, peerMap(peerId.toString()), LOCAL_MEMBER_NAME), leaderId.toString());
ShardTestKit.waitUntilLeader(leader);
String otherEntityType = "otherEntityType";
final DOMEntity entity1 = new DOMEntity(ENTITY_TYPE, ENTITY_ID1);
final DOMEntity entity2 = new DOMEntity(ENTITY_TYPE, ENTITY_ID2);
final DOMEntity entity3 = new DOMEntity(ENTITY_TYPE, ENTITY_ID3);
final DOMEntity entity4 = new DOMEntity(otherEntityType, ENTITY_ID3);
DOMEntityOwnershipListener listener = mock(DOMEntityOwnershipListener.class);
// Register listener
leader.tell(new RegisterListenerLocal(listener, ENTITY_TYPE), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
// Register a couple candidates for the desired entity type and verify listener is notified.
leader.tell(new RegisterCandidateLocal(entity1), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verify(listener, timeout(5000)).ownershipChanged(ownershipChange(entity1, false, true, true));
leader.tell(new RegisterCandidateLocal(entity2), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verify(listener, timeout(5000)).ownershipChanged(ownershipChange(entity2, false, true, true));
reset(listener);
// Register another candidate for another entity type and verify listener is not notified.
leader.tell(new RegisterCandidateLocal(entity4), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
verify(listener, never()).ownershipChanged(ownershipChange(entity4));
// Register remote candidate for entity1
peer.tell(new RegisterCandidateLocal(entity1), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verifyCommittedEntityCandidate(leader, ENTITY_TYPE, entity1.getIdentifier(), PEER_MEMBER_1_NAME);
// Unregister the local candidate for entity1 and verify listener is notified
leader.tell(new UnregisterCandidateLocal(entity1), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verify(listener, timeout(5000)).ownershipChanged(ownershipChange(entity1, true, false, true));
reset(listener);
// Unregister the listener, add a candidate for entity3 and verify listener isn't notified
leader.tell(new UnregisterListenerLocal(listener, ENTITY_TYPE), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
leader.tell(new RegisterCandidateLocal(entity3), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verifyOwner(leader, ENTITY_TYPE, entity3.getIdentifier(), LOCAL_MEMBER_NAME);
Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
verify(listener, never()).ownershipChanged(any(DOMEntityOwnershipChange.class));
// Re-register the listener and verify it gets notified of currently owned entities
reset(listener);
leader.tell(new RegisterListenerLocal(listener, ENTITY_TYPE), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verify(listener, timeout(5000).times(2)).ownershipChanged(or(ownershipChange(entity2, false, true, true), ownershipChange(entity3, false, true, true)));
Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
verify(listener, never()).ownershipChanged(ownershipChange(entity4));
verify(listener, times(1)).ownershipChanged(ownershipChange(entity1));
testLog.info("testListenerRegistration ending");
}
use of org.opendaylight.controller.cluster.datastore.entityownership.messages.RegisterListenerLocal in project controller by opendaylight.
the class EntityOwnershipShardTest method testLeaderIsolationWithPendingCandidateAdded.
@Test
public void testLeaderIsolationWithPendingCandidateAdded() throws Exception {
testLog.info("testLeaderIsolationWithPendingCandidateAdded starting");
final ShardTestKit kit = new ShardTestKit(getSystem());
ShardIdentifier leaderId = newShardId(LOCAL_MEMBER_NAME);
ShardIdentifier peerId1 = newShardId(PEER_MEMBER_1_NAME);
ShardIdentifier peerId2 = newShardId(PEER_MEMBER_2_NAME);
dataStoreContextBuilder.shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(4).shardIsolatedLeaderCheckIntervalInMillis(100000);
TestActorRef<TestEntityOwnershipShard> peer1 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId1, peerMap(leaderId.toString(), peerId2.toString()), PEER_MEMBER_1_NAME), actorFactory.createActor(MessageCollectorActor.props())), peerId1.toString());
peer1.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
TestActorRef<TestEntityOwnershipShard> peer2 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId2, peerMap(leaderId.toString(), peerId1.toString()), PEER_MEMBER_2_NAME), actorFactory.createTestActor(MessageCollectorActor.props())), peerId2.toString());
peer2.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
dataStoreContextBuilder = DatastoreContext.newBuilderFrom(dataStoreContextBuilder.build()).shardIsolatedLeaderCheckIntervalInMillis(500);
TestActorRef<TestEntityOwnershipShard> leader = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(leaderId, peerMap(peerId1.toString(), peerId2.toString()), LOCAL_MEMBER_NAME), actorFactory.createTestActor(MessageCollectorActor.props())), leaderId.toString());
ShardTestKit.waitUntilLeader(leader);
// Add listeners on all members
DOMEntityOwnershipListener leaderListener = mock(DOMEntityOwnershipListener.class, "DOMEntityOwnershipListener-" + LOCAL_MEMBER_NAME);
leader.tell(new RegisterListenerLocal(leaderListener, ENTITY_TYPE), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
DOMEntityOwnershipListener peer1Listener = mock(DOMEntityOwnershipListener.class, "DOMEntityOwnershipListener-" + PEER_MEMBER_1_NAME);
peer1.tell(new RegisterListenerLocal(peer1Listener, ENTITY_TYPE), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
DOMEntityOwnershipListener peer2Listener = mock(DOMEntityOwnershipListener.class, "DOMEntityOwnershipListener-" + PEER_MEMBER_2_NAME);
peer2.tell(new RegisterListenerLocal(peer2Listener, ENTITY_TYPE), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
// Drop the CandidateAdded message to the leader for now.
leader.underlyingActor().startDroppingMessagesOfType(CandidateAdded.class);
// Add an entity candidates for the leader. Since we've blocked the CandidateAdded message, it won't be
// assigned the owner.
DOMEntity entity1 = new DOMEntity(ENTITY_TYPE, ENTITY_ID1);
leader.tell(new RegisterCandidateLocal(entity1), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verifyCommittedEntityCandidate(leader, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
verifyCommittedEntityCandidate(peer1, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
verifyCommittedEntityCandidate(peer2, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
DOMEntity entity2 = new DOMEntity(ENTITY_TYPE, ENTITY_ID2);
leader.tell(new RegisterCandidateLocal(entity2), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verifyCommittedEntityCandidate(leader, entity2.getType(), entity2.getIdentifier(), LOCAL_MEMBER_NAME);
verifyCommittedEntityCandidate(peer1, entity2.getType(), entity2.getIdentifier(), LOCAL_MEMBER_NAME);
verifyCommittedEntityCandidate(peer2, entity2.getType(), entity2.getIdentifier(), LOCAL_MEMBER_NAME);
// Capture the CandidateAdded messages.
final List<CandidateAdded> candidateAdded = expectMatching(leader.underlyingActor().collectorActor(), CandidateAdded.class, 2);
// Drop AppendEntries to the followers containing a log entry, which will be for the owner writes after we
// forward the CandidateAdded messages to the leader. This will leave the pending owner write tx's uncommitted.
peer1.underlyingActor().startDroppingMessagesOfType(AppendEntries.class, ae -> ae.getEntries().size() > 0);
peer2.underlyingActor().startDroppingMessagesOfType(AppendEntries.class, ae -> ae.getEntries().size() > 0);
// Now forward the CandidateAdded messages to the leader and wait for it to send out the AppendEntries.
leader.underlyingActor().stopDroppingMessagesOfType(CandidateAdded.class);
leader.tell(candidateAdded.get(0), leader);
leader.tell(candidateAdded.get(1), leader);
expectMatching(peer1.underlyingActor().collectorActor(), AppendEntries.class, 2, ae -> ae.getEntries().size() > 0);
// Verify no owner assigned.
verifyNoOwnerSet(leader, entity1.getType(), entity1.getIdentifier());
verifyNoOwnerSet(leader, entity2.getType(), entity2.getIdentifier());
// Isolate the leader by dropping AppendEntries to the followers and incoming messages from the followers.
leader.underlyingActor().startDroppingMessagesOfType(RequestVote.class);
leader.underlyingActor().startDroppingMessagesOfType(AppendEntries.class);
peer2.underlyingActor().startDroppingMessagesOfType(AppendEntries.class, ae -> ae.getLeaderId().equals(leaderId.toString()));
peer1.underlyingActor().startDroppingMessagesOfType(AppendEntries.class);
// Send PeerDown to the isolated leader - should be no-op since there's no owned entities.
leader.tell(new PeerDown(peerId1.getMemberName(), peerId1.toString()), ActorRef.noSender());
leader.tell(new PeerDown(peerId2.getMemberName(), peerId2.toString()), ActorRef.noSender());
// Verify the leader transitions to IsolatedLeader.
verifyRaftState(leader, state -> assertEquals("getRaftState", RaftState.IsolatedLeader.toString(), state.getRaftState()));
// Send PeerDown to the new leader peer1.
peer1.tell(new PeerDown(leaderId.getMemberName(), leaderId.toString()), ActorRef.noSender());
// Make peer1 start an election and become leader by sending the TimeoutNow message.
peer1.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
// Verify the peer1 transitions to Leader.
verifyRaftState(peer1, state -> assertEquals("getRaftState", RaftState.Leader.toString(), state.getRaftState()));
verifyNoOwnerSet(peer1, entity1.getType(), entity1.getIdentifier());
verifyNoOwnerSet(peer2, entity1.getType(), entity2.getIdentifier());
verifyNoMoreInteractions(peer1Listener);
verifyNoMoreInteractions(peer2Listener);
// Add candidate peer1 candidate for entity2.
peer1.tell(new RegisterCandidateLocal(entity2), kit.getRef());
verifyOwner(peer1, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
verify(peer1Listener, timeout(5000)).ownershipChanged(ownershipChange(entity2, false, true, true));
verify(peer2Listener, timeout(5000)).ownershipChanged(ownershipChange(entity2, false, false, true));
reset(leaderListener, peer1Listener, peer2Listener);
// Remove the isolation.
leader.underlyingActor().stopDroppingMessagesOfType(RequestVote.class);
leader.underlyingActor().stopDroppingMessagesOfType(AppendEntries.class);
peer2.underlyingActor().stopDroppingMessagesOfType(AppendEntries.class);
peer1.underlyingActor().stopDroppingMessagesOfType(AppendEntries.class);
// Previous leader should switch to Follower.
verifyRaftState(leader, state -> assertEquals("getRaftState", RaftState.Follower.toString(), state.getRaftState()));
// Send PeerUp to peer1 and peer2.
peer1.tell(new PeerUp(leaderId.getMemberName(), leaderId.toString()), ActorRef.noSender());
peer2.tell(new PeerUp(leaderId.getMemberName(), leaderId.toString()), ActorRef.noSender());
// The previous leader should become the owner of entity1.
verifyOwner(leader, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
// The previous leader's DOMEntityOwnershipListener should get 4 total notifications:
// - inJeopardy cleared for entity1 (wasOwner=false, isOwner=false, hasOwner=false, inJeopardy=false)
// - inJeopardy cleared for entity2 (wasOwner=false, isOwner=false, hasOwner=false, inJeopardy=false)
// - local owner granted for entity1 (wasOwner=false, isOwner=true, hasOwner=true, inJeopardy=false)
// - remote owner for entity2 (wasOwner=false, isOwner=false, hasOwner=true, inJeopardy=false)
verify(leaderListener, timeout(5000).times(4)).ownershipChanged(or(or(ownershipChange(entity1, false, false, false), ownershipChange(entity2, false, false, false)), or(ownershipChange(entity1, false, true, true), ownershipChange(entity2, false, false, true))));
verify(peer1Listener, timeout(5000)).ownershipChanged(ownershipChange(entity1, false, false, true));
verify(peer2Listener, timeout(5000)).ownershipChanged(ownershipChange(entity1, false, false, true));
// Verify entity2's owner doesn't change.
Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
verifyOwner(peer1, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
verifyNoMoreInteractions(leaderListener);
verifyNoMoreInteractions(peer1Listener);
verifyNoMoreInteractions(peer2Listener);
testLog.info("testLeaderIsolationWithPendingCandidateAdded ending");
}
use of org.opendaylight.controller.cluster.datastore.entityownership.messages.RegisterListenerLocal in project controller by opendaylight.
the class DistributedEntityOwnershipServiceTest method testListenerRegistration.
@Test
public void testListenerRegistration() {
DistributedEntityOwnershipService service = spy(DistributedEntityOwnershipService.start(dataStore.getActorContext(), EntityOwnerSelectionStrategyConfig.newBuilder().build()));
YangInstanceIdentifier entityId = YangInstanceIdentifier.of(QNAME);
DOMEntity entity = new DOMEntity(ENTITY_TYPE, entityId);
DOMEntityOwnershipListener listener = mock(DOMEntityOwnershipListener.class);
DOMEntityOwnershipListenerRegistration reg = service.registerListener(entity.getType(), listener);
assertNotNull("EntityOwnershipListenerRegistration null", reg);
assertEquals("getEntityType", entity.getType(), reg.getEntityType());
assertEquals("getInstance", listener, reg.getInstance());
RegisterListenerLocal regListener = verifyMessage(service, RegisterListenerLocal.class);
assertSame("getListener", listener, regListener.getListener());
assertEquals("getEntityType", entity.getType(), regListener.getEntityType());
reset(service);
reg.close();
UnregisterListenerLocal unregListener = verifyMessage(service, UnregisterListenerLocal.class);
assertEquals("getEntityType", entity.getType(), unregListener.getEntityType());
assertSame("getListener", listener, unregListener.getListener());
service.close();
}
use of org.opendaylight.controller.cluster.datastore.entityownership.messages.RegisterListenerLocal in project controller by opendaylight.
the class EntityOwnershipShardTest method testLeaderIsolation.
@Test
public void testLeaderIsolation() throws Exception {
testLog.info("testLeaderIsolation starting");
final ShardTestKit kit = new ShardTestKit(getSystem());
ShardIdentifier leaderId = newShardId(LOCAL_MEMBER_NAME);
ShardIdentifier peerId1 = newShardId(PEER_MEMBER_1_NAME);
ShardIdentifier peerId2 = newShardId(PEER_MEMBER_2_NAME);
dataStoreContextBuilder.shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(4).shardIsolatedLeaderCheckIntervalInMillis(100000);
TestActorRef<TestEntityOwnershipShard> peer1 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId1, peerMap(leaderId.toString(), peerId2.toString()), PEER_MEMBER_1_NAME)), peerId1.toString());
peer1.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
TestActorRef<TestEntityOwnershipShard> peer2 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId2, peerMap(leaderId.toString(), peerId1.toString()), PEER_MEMBER_2_NAME)), peerId2.toString());
peer2.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
dataStoreContextBuilder = DatastoreContext.newBuilderFrom(dataStoreContextBuilder.build()).shardIsolatedLeaderCheckIntervalInMillis(500);
TestActorRef<TestEntityOwnershipShard> leader = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(leaderId, peerMap(peerId1.toString(), peerId2.toString()), LOCAL_MEMBER_NAME)), leaderId.toString());
ShardTestKit.waitUntilLeader(leader);
// Add entity1 candidates for all members with the leader as the owner
DOMEntity entity1 = new DOMEntity(ENTITY_TYPE, ENTITY_ID1);
leader.tell(new RegisterCandidateLocal(entity1), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verifyCommittedEntityCandidate(leader, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
peer1.tell(new RegisterCandidateLocal(entity1), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verifyCommittedEntityCandidate(leader, entity1.getType(), entity1.getIdentifier(), PEER_MEMBER_1_NAME);
peer2.tell(new RegisterCandidateLocal(entity1), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verifyCommittedEntityCandidate(leader, entity1.getType(), entity1.getIdentifier(), PEER_MEMBER_2_NAME);
verifyOwner(leader, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
verifyOwner(peer1, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
verifyOwner(peer2, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
// Add entity2 candidates for all members with peer1 as the owner
DOMEntity entity2 = new DOMEntity(ENTITY_TYPE, ENTITY_ID2);
peer1.tell(new RegisterCandidateLocal(entity2), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verifyCommittedEntityCandidate(leader, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
peer2.tell(new RegisterCandidateLocal(entity2), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verifyCommittedEntityCandidate(leader, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_2_NAME);
leader.tell(new RegisterCandidateLocal(entity2), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verifyCommittedEntityCandidate(leader, entity2.getType(), entity2.getIdentifier(), LOCAL_MEMBER_NAME);
verifyOwner(leader, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
verifyOwner(peer1, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
verifyOwner(peer2, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
// Add entity3 candidates for all members with peer2 as the owner
DOMEntity entity3 = new DOMEntity(ENTITY_TYPE, ENTITY_ID3);
peer2.tell(new RegisterCandidateLocal(entity3), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verifyCommittedEntityCandidate(leader, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_2_NAME);
leader.tell(new RegisterCandidateLocal(entity3), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verifyCommittedEntityCandidate(leader, entity3.getType(), entity3.getIdentifier(), LOCAL_MEMBER_NAME);
peer1.tell(new RegisterCandidateLocal(entity3), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verifyCommittedEntityCandidate(leader, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_1_NAME);
verifyOwner(leader, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_2_NAME);
verifyOwner(peer1, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_2_NAME);
verifyOwner(peer2, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_2_NAME);
// Add listeners on all members
DOMEntityOwnershipListener leaderListener = mock(DOMEntityOwnershipListener.class);
leader.tell(new RegisterListenerLocal(leaderListener, ENTITY_TYPE), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verify(leaderListener, timeout(5000).times(3)).ownershipChanged(or(or(ownershipChange(entity1, false, true, true), ownershipChange(entity2, false, false, true)), ownershipChange(entity3, false, false, true)));
reset(leaderListener);
DOMEntityOwnershipListener peer1Listener = mock(DOMEntityOwnershipListener.class);
peer1.tell(new RegisterListenerLocal(peer1Listener, ENTITY_TYPE), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verify(peer1Listener, timeout(5000).times(3)).ownershipChanged(or(or(ownershipChange(entity1, false, false, true), ownershipChange(entity2, false, true, true)), ownershipChange(entity3, false, false, true)));
reset(peer1Listener);
DOMEntityOwnershipListener peer2Listener = mock(DOMEntityOwnershipListener.class);
peer2.tell(new RegisterListenerLocal(peer2Listener, ENTITY_TYPE), kit.getRef());
kit.expectMsgClass(SuccessReply.class);
verify(peer2Listener, timeout(5000).times(3)).ownershipChanged(or(or(ownershipChange(entity1, false, false, true), ownershipChange(entity2, false, false, true)), ownershipChange(entity3, false, true, true)));
reset(peer2Listener);
// Isolate the leader by dropping AppendEntries to the followers and incoming messages from the followers.
leader.underlyingActor().startDroppingMessagesOfType(RequestVote.class);
leader.underlyingActor().startDroppingMessagesOfType(AppendEntries.class);
peer2.underlyingActor().startDroppingMessagesOfType(AppendEntries.class, ae -> ae.getLeaderId().equals(leaderId.toString()));
peer1.underlyingActor().startDroppingMessagesOfType(AppendEntries.class);
// Make peer1 start an election and become leader by enabling the ElectionTimeout message.
peer1.underlyingActor().stopDroppingMessagesOfType(ElectionTimeout.class);
// Send PeerDown to the isolated leader so it tries to re-assign ownership for the entities owned by the
// isolated peers.
leader.tell(new PeerDown(peerId1.getMemberName(), peerId1.toString()), ActorRef.noSender());
leader.tell(new PeerDown(peerId2.getMemberName(), peerId2.toString()), ActorRef.noSender());
verifyRaftState(leader, state -> assertEquals("getRaftState", RaftState.IsolatedLeader.toString(), state.getRaftState()));
// Expect inJeopardy notification on the isolated leader.
verify(leaderListener, timeout(5000).times(3)).ownershipChanged(or(or(ownershipChange(entity1, true, true, true, true), ownershipChange(entity2, false, false, true, true)), ownershipChange(entity3, false, false, true, true)));
reset(leaderListener);
verifyRaftState(peer1, state -> assertEquals("getRaftState", RaftState.Leader.toString(), state.getRaftState()));
// Send PeerDown to the new leader peer1 so it re-assigns ownership for the entities owned by the
// isolated leader.
peer1.tell(new PeerDown(leaderId.getMemberName(), leaderId.toString()), ActorRef.noSender());
verifyOwner(peer1, entity1.getType(), entity1.getIdentifier(), PEER_MEMBER_1_NAME);
verify(peer1Listener, timeout(5000)).ownershipChanged(ownershipChange(entity1, false, true, true));
reset(peer1Listener);
verify(peer2Listener, timeout(5000)).ownershipChanged(ownershipChange(entity1, false, false, true));
reset(peer2Listener);
// Remove the isolation.
leader.underlyingActor().stopDroppingMessagesOfType(RequestVote.class);
leader.underlyingActor().stopDroppingMessagesOfType(AppendEntries.class);
peer2.underlyingActor().stopDroppingMessagesOfType(AppendEntries.class);
peer1.underlyingActor().stopDroppingMessagesOfType(AppendEntries.class);
// Previous leader should switch to Follower and send inJeopardy cleared notifications for all entities.
verifyRaftState(leader, state -> assertEquals("getRaftState", RaftState.Follower.toString(), state.getRaftState()));
verify(leaderListener, timeout(5000).times(3)).ownershipChanged(or(or(ownershipChange(entity1, true, true, true), ownershipChange(entity2, false, false, true)), ownershipChange(entity3, false, false, true)));
verifyOwner(leader, entity1.getType(), entity1.getIdentifier(), PEER_MEMBER_1_NAME);
verify(leaderListener, timeout(5000)).ownershipChanged(ownershipChange(entity1, true, false, true));
Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
verifyOwner(leader, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
verifyOwner(leader, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_2_NAME);
verifyNoMoreInteractions(leaderListener);
verifyNoMoreInteractions(peer1Listener);
verifyNoMoreInteractions(peer2Listener);
testLog.info("testLeaderIsolation ending");
}
Aggregations