Examples with PeerDown org.opendaylight.controller.cluster.datastore.messages.PeerDown used on opensource projects

Example 1 with PeerDown

use of org.opendaylight.controller.cluster.datastore.messages.PeerDown in project controller by opendaylight.

the class EntityOwnershipShardTest method testOwnerChangesOnPeerAvailabilityChanges.

@Test
public void testOwnerChangesOnPeerAvailabilityChanges() throws Exception {
    testLog.info("testOwnerChangesOnPeerAvailabilityChanges starting");
    final ShardTestKit kit = new ShardTestKit(getSystem());
    dataStoreContextBuilder.shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(4).shardIsolatedLeaderCheckIntervalInMillis(100000);
    ShardIdentifier leaderId = newShardId(LOCAL_MEMBER_NAME);
    ShardIdentifier peerId1 = newShardId(PEER_MEMBER_1_NAME);
    ShardIdentifier peerId2 = newShardId(PEER_MEMBER_2_NAME);
    TestActorRef<TestEntityOwnershipShard> peer1 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId1, peerMap(leaderId.toString(), peerId2.toString()), PEER_MEMBER_1_NAME)), peerId1.toString());
    peer1.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
    TestActorRef<TestEntityOwnershipShard> peer2 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId2, peerMap(leaderId.toString(), peerId1.toString()), PEER_MEMBER_2_NAME)), peerId2.toString());
    peer2.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
    TestActorRef<EntityOwnershipShard> leader = actorFactory.createTestActor(newShardProps(leaderId, peerMap(peerId1.toString(), peerId2.toString()), LOCAL_MEMBER_NAME), leaderId.toString());
    verifyRaftState(leader, state -> assertEquals("getRaftState", RaftState.Leader.toString(), state.getRaftState()));
    // Send PeerDown and PeerUp with no entities
    leader.tell(new PeerDown(peerId2.getMemberName(), peerId2.toString()), ActorRef.noSender());
    leader.tell(new PeerUp(peerId2.getMemberName(), peerId2.toString()), ActorRef.noSender());
    // Add candidates for entity1 with the local leader as the owner
    leader.tell(new RegisterCandidateLocal(new DOMEntity(ENTITY_TYPE, ENTITY_ID1)), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID1, LOCAL_MEMBER_NAME);
    peer2.tell(new RegisterCandidateLocal(new DOMEntity(ENTITY_TYPE, ENTITY_ID1)), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID1, PEER_MEMBER_2_NAME);
    peer1.tell(new RegisterCandidateLocal(new DOMEntity(ENTITY_TYPE, ENTITY_ID1)), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID1, PEER_MEMBER_1_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID1, LOCAL_MEMBER_NAME);
    // Add candidates for entity2 with peerMember2 as the owner
    peer2.tell(new RegisterCandidateLocal(new DOMEntity(ENTITY_TYPE, ENTITY_ID2)), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_2_NAME);
    peer1.tell(new RegisterCandidateLocal(new DOMEntity(ENTITY_TYPE, ENTITY_ID2)), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_1_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_2_NAME);
    // Add candidates for entity3 with peerMember2 as the owner.
    peer2.tell(new RegisterCandidateLocal(new DOMEntity(ENTITY_TYPE, ENTITY_ID3)), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID3, PEER_MEMBER_2_NAME);
    leader.tell(new RegisterCandidateLocal(new DOMEntity(ENTITY_TYPE, ENTITY_ID3)), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID3, LOCAL_MEMBER_NAME);
    peer1.tell(new RegisterCandidateLocal(new DOMEntity(ENTITY_TYPE, ENTITY_ID3)), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID3, PEER_MEMBER_1_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID3, PEER_MEMBER_2_NAME);
    // Add only candidate peerMember2 for entity4.
    peer2.tell(new RegisterCandidateLocal(new DOMEntity(ENTITY_TYPE, ENTITY_ID4)), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID4, PEER_MEMBER_2_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID4, PEER_MEMBER_2_NAME);
    // Add only candidate peerMember1 for entity5.
    peer1.tell(new RegisterCandidateLocal(new DOMEntity(ENTITY_TYPE, ENTITY_ID5)), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID5, PEER_MEMBER_1_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID5, PEER_MEMBER_1_NAME);
    // Kill peerMember2 and send PeerDown - the entities (2, 3, 4) owned by peerMember2 should get a new
    // owner selected
    kit.watch(peer2);
    peer2.tell(PoisonPill.getInstance(), ActorRef.noSender());
    kit.expectMsgClass(kit.duration("5 seconds"), Terminated.class);
    kit.unwatch(peer2);
    leader.tell(new PeerDown(peerId2.getMemberName(), peerId2.toString()), ActorRef.noSender());
    // Send PeerDown again - should be noop
    leader.tell(new PeerDown(peerId2.getMemberName(), peerId2.toString()), ActorRef.noSender());
    peer1.tell(new PeerDown(peerId2.getMemberName(), peerId2.toString()), ActorRef.noSender());
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID1, LOCAL_MEMBER_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_1_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID3, LOCAL_MEMBER_NAME);
    // no other candidates for entity4 so peerMember2 should remain owner.
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID4, PEER_MEMBER_2_NAME);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID1, PEER_MEMBER_2_NAME);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_2_NAME);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID3, PEER_MEMBER_2_NAME);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID4, PEER_MEMBER_2_NAME);
    // Reinstate peerMember2
    peer2 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId2, peerMap(leaderId.toString(), peerId1.toString()), PEER_MEMBER_2_NAME)), peerId2.toString());
    peer2.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
    leader.tell(new PeerUp(peerId2.getMemberName(), peerId2.toString()), ActorRef.noSender());
    // Send PeerUp again - should be noop
    leader.tell(new PeerUp(peerId2.getMemberName(), peerId2.toString()), ActorRef.noSender());
    peer1.tell(new PeerUp(peerId2.getMemberName(), peerId2.toString()), ActorRef.noSender());
    // peerMember2's candidates should be removed on startup.
    verifyNoEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID1, PEER_MEMBER_2_NAME);
    verifyNoEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_2_NAME);
    verifyNoEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID3, PEER_MEMBER_2_NAME);
    verifyNoEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID4, PEER_MEMBER_2_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID1, LOCAL_MEMBER_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_1_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID3, LOCAL_MEMBER_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID4, "");
    // Add back candidate peerMember2 for entities 1, 2, & 3.
    peer2.tell(new RegisterCandidateLocal(new DOMEntity(ENTITY_TYPE, ENTITY_ID1)), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    peer2.tell(new RegisterCandidateLocal(new DOMEntity(ENTITY_TYPE, ENTITY_ID2)), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    peer2.tell(new RegisterCandidateLocal(new DOMEntity(ENTITY_TYPE, ENTITY_ID3)), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID1, PEER_MEMBER_2_NAME);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_2_NAME);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID3, PEER_MEMBER_2_NAME);
    verifyCommittedEntityCandidate(peer2, ENTITY_TYPE, ENTITY_ID1, PEER_MEMBER_2_NAME);
    verifyCommittedEntityCandidate(peer2, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_2_NAME);
    verifyCommittedEntityCandidate(peer2, ENTITY_TYPE, ENTITY_ID3, PEER_MEMBER_2_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID1, LOCAL_MEMBER_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_1_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID3, LOCAL_MEMBER_NAME);
    verifyOwner(peer2, ENTITY_TYPE, ENTITY_ID1, LOCAL_MEMBER_NAME);
    verifyOwner(peer2, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_1_NAME);
    verifyOwner(peer2, ENTITY_TYPE, ENTITY_ID3, LOCAL_MEMBER_NAME);
    verifyOwner(peer2, ENTITY_TYPE, ENTITY_ID4, "");
    // Kill peerMember1 and send PeerDown - entity 2 should get a new owner selected
    kit.watch(peer1);
    peer1.tell(PoisonPill.getInstance(), ActorRef.noSender());
    kit.expectMsgClass(kit.duration("5 seconds"), Terminated.class);
    kit.unwatch(peer1);
    leader.tell(new PeerDown(peerId1.getMemberName(), peerId1.toString()), ActorRef.noSender());
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_2_NAME);
    // Verify the reinstated peerMember2 is fully synced.
    verifyOwner(peer2, ENTITY_TYPE, ENTITY_ID1, LOCAL_MEMBER_NAME);
    verifyOwner(peer2, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_2_NAME);
    verifyOwner(peer2, ENTITY_TYPE, ENTITY_ID3, LOCAL_MEMBER_NAME);
    verifyOwner(peer2, ENTITY_TYPE, ENTITY_ID4, "");
    // Reinstate peerMember1 and verify no owner changes
    peer1 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId1, peerMap(leaderId.toString(), peerId2.toString()), PEER_MEMBER_1_NAME)), peerId1.toString());
    peer1.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
    leader.tell(new PeerUp(peerId1.getMemberName(), peerId1.toString()), ActorRef.noSender());
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID1, LOCAL_MEMBER_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_2_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID3, LOCAL_MEMBER_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID4, "");
    verifyNoEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID1, PEER_MEMBER_1_NAME);
    verifyNoEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_1_NAME);
    verifyNoEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID3, PEER_MEMBER_1_NAME);
    verifyNoEntityCandidate(peer2, ENTITY_TYPE, ENTITY_ID1, PEER_MEMBER_1_NAME);
    verifyNoEntityCandidate(peer2, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_1_NAME);
    verifyNoEntityCandidate(peer2, ENTITY_TYPE, ENTITY_ID3, PEER_MEMBER_1_NAME);
    // Verify the reinstated peerMember1 is fully synced.
    verifyOwner(peer1, ENTITY_TYPE, ENTITY_ID1, LOCAL_MEMBER_NAME);
    verifyOwner(peer1, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_2_NAME);
    verifyOwner(peer1, ENTITY_TYPE, ENTITY_ID3, LOCAL_MEMBER_NAME);
    verifyOwner(peer1, ENTITY_TYPE, ENTITY_ID4, "");
    AtomicLong leaderLastApplied = new AtomicLong();
    verifyRaftState(leader, rs -> {
        assertEquals("LastApplied up-to-date", rs.getLastApplied(), rs.getLastIndex());
        leaderLastApplied.set(rs.getLastApplied());
    });
    verifyRaftState(peer2, rs -> {
        assertEquals("LastApplied", leaderLastApplied.get(), rs.getLastIndex());
    });
    // Kill the local leader and elect peer2 the leader. This should cause a new owner to be selected for
    // the entities (1 and 3) previously owned by the local leader member.
    peer2.tell(new PeerAddressResolved(peerId1.toString(), peer1.path().toString()), ActorRef.noSender());
    peer2.tell(new PeerUp(leaderId.getMemberName(), leaderId.toString()), ActorRef.noSender());
    peer2.tell(new PeerUp(peerId1.getMemberName(), peerId1.toString()), ActorRef.noSender());
    kit.watch(leader);
    leader.tell(PoisonPill.getInstance(), ActorRef.noSender());
    kit.expectMsgClass(kit.duration("5 seconds"), Terminated.class);
    kit.unwatch(leader);
    peer2.tell(new PeerDown(leaderId.getMemberName(), leaderId.toString()), ActorRef.noSender());
    peer2.tell(TimeoutNow.INSTANCE, peer2);
    verifyRaftState(peer2, state -> assertEquals("getRaftState", RaftState.Leader.toString(), state.getRaftState()));
    verifyOwner(peer2, ENTITY_TYPE, ENTITY_ID1, PEER_MEMBER_2_NAME);
    verifyOwner(peer2, ENTITY_TYPE, ENTITY_ID2, PEER_MEMBER_2_NAME);
    verifyOwner(peer2, ENTITY_TYPE, ENTITY_ID3, PEER_MEMBER_2_NAME);
    verifyOwner(peer2, ENTITY_TYPE, ENTITY_ID4, "");
    testLog.info("testOwnerChangesOnPeerAvailabilityChanges ending");
}

Example 2 with PeerDown

use of org.opendaylight.controller.cluster.datastore.messages.PeerDown in project controller by opendaylight.

the class EntityOwnershipShardTest method testLeaderIsolationWithPendingCandidateAdded.

@Test
public void testLeaderIsolationWithPendingCandidateAdded() throws Exception {
    testLog.info("testLeaderIsolationWithPendingCandidateAdded starting");
    final ShardTestKit kit = new ShardTestKit(getSystem());
    ShardIdentifier leaderId = newShardId(LOCAL_MEMBER_NAME);
    ShardIdentifier peerId1 = newShardId(PEER_MEMBER_1_NAME);
    ShardIdentifier peerId2 = newShardId(PEER_MEMBER_2_NAME);
    dataStoreContextBuilder.shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(4).shardIsolatedLeaderCheckIntervalInMillis(100000);
    TestActorRef<TestEntityOwnershipShard> peer1 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId1, peerMap(leaderId.toString(), peerId2.toString()), PEER_MEMBER_1_NAME), actorFactory.createActor(MessageCollectorActor.props())), peerId1.toString());
    peer1.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
    TestActorRef<TestEntityOwnershipShard> peer2 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId2, peerMap(leaderId.toString(), peerId1.toString()), PEER_MEMBER_2_NAME), actorFactory.createTestActor(MessageCollectorActor.props())), peerId2.toString());
    peer2.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
    dataStoreContextBuilder = DatastoreContext.newBuilderFrom(dataStoreContextBuilder.build()).shardIsolatedLeaderCheckIntervalInMillis(500);
    TestActorRef<TestEntityOwnershipShard> leader = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(leaderId, peerMap(peerId1.toString(), peerId2.toString()), LOCAL_MEMBER_NAME), actorFactory.createTestActor(MessageCollectorActor.props())), leaderId.toString());
    ShardTestKit.waitUntilLeader(leader);
    // Add listeners on all members
    DOMEntityOwnershipListener leaderListener = mock(DOMEntityOwnershipListener.class, "DOMEntityOwnershipListener-" + LOCAL_MEMBER_NAME);
    leader.tell(new RegisterListenerLocal(leaderListener, ENTITY_TYPE), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    DOMEntityOwnershipListener peer1Listener = mock(DOMEntityOwnershipListener.class, "DOMEntityOwnershipListener-" + PEER_MEMBER_1_NAME);
    peer1.tell(new RegisterListenerLocal(peer1Listener, ENTITY_TYPE), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    DOMEntityOwnershipListener peer2Listener = mock(DOMEntityOwnershipListener.class, "DOMEntityOwnershipListener-" + PEER_MEMBER_2_NAME);
    peer2.tell(new RegisterListenerLocal(peer2Listener, ENTITY_TYPE), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    // Drop the CandidateAdded message to the leader for now.
    leader.underlyingActor().startDroppingMessagesOfType(CandidateAdded.class);
    // Add an entity candidates for the leader. Since we've blocked the CandidateAdded message, it won't be
    // assigned the owner.
    DOMEntity entity1 = new DOMEntity(ENTITY_TYPE, ENTITY_ID1);
    leader.tell(new RegisterCandidateLocal(entity1), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
    verifyCommittedEntityCandidate(peer1, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
    verifyCommittedEntityCandidate(peer2, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
    DOMEntity entity2 = new DOMEntity(ENTITY_TYPE, ENTITY_ID2);
    leader.tell(new RegisterCandidateLocal(entity2), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity2.getType(), entity2.getIdentifier(), LOCAL_MEMBER_NAME);
    verifyCommittedEntityCandidate(peer1, entity2.getType(), entity2.getIdentifier(), LOCAL_MEMBER_NAME);
    verifyCommittedEntityCandidate(peer2, entity2.getType(), entity2.getIdentifier(), LOCAL_MEMBER_NAME);
    // Capture the CandidateAdded messages.
    final List<CandidateAdded> candidateAdded = expectMatching(leader.underlyingActor().collectorActor(), CandidateAdded.class, 2);
    // Drop AppendEntries to the followers containing a log entry, which will be for the owner writes after we
    // forward the CandidateAdded messages to the leader. This will leave the pending owner write tx's uncommitted.
    peer1.underlyingActor().startDroppingMessagesOfType(AppendEntries.class, ae -> ae.getEntries().size() > 0);
    peer2.underlyingActor().startDroppingMessagesOfType(AppendEntries.class, ae -> ae.getEntries().size() > 0);
    // Now forward the CandidateAdded messages to the leader and wait for it to send out the AppendEntries.
    leader.underlyingActor().stopDroppingMessagesOfType(CandidateAdded.class);
    leader.tell(candidateAdded.get(0), leader);
    leader.tell(candidateAdded.get(1), leader);
    expectMatching(peer1.underlyingActor().collectorActor(), AppendEntries.class, 2, ae -> ae.getEntries().size() > 0);
    // Verify no owner assigned.
    verifyNoOwnerSet(leader, entity1.getType(), entity1.getIdentifier());
    verifyNoOwnerSet(leader, entity2.getType(), entity2.getIdentifier());
    // Isolate the leader by dropping AppendEntries to the followers and incoming messages from the followers.
    leader.underlyingActor().startDroppingMessagesOfType(RequestVote.class);
    leader.underlyingActor().startDroppingMessagesOfType(AppendEntries.class);
    peer2.underlyingActor().startDroppingMessagesOfType(AppendEntries.class, ae -> ae.getLeaderId().equals(leaderId.toString()));
    peer1.underlyingActor().startDroppingMessagesOfType(AppendEntries.class);
    // Send PeerDown to the isolated leader - should be no-op since there's no owned entities.
    leader.tell(new PeerDown(peerId1.getMemberName(), peerId1.toString()), ActorRef.noSender());
    leader.tell(new PeerDown(peerId2.getMemberName(), peerId2.toString()), ActorRef.noSender());
    // Verify the leader transitions to IsolatedLeader.
    verifyRaftState(leader, state -> assertEquals("getRaftState", RaftState.IsolatedLeader.toString(), state.getRaftState()));
    // Send PeerDown to the new leader peer1.
    peer1.tell(new PeerDown(leaderId.getMemberName(), leaderId.toString()), ActorRef.noSender());
    // Make peer1 start an election and become leader by sending the TimeoutNow message.
    peer1.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
    // Verify the peer1 transitions to Leader.
    verifyRaftState(peer1, state -> assertEquals("getRaftState", RaftState.Leader.toString(), state.getRaftState()));
    verifyNoOwnerSet(peer1, entity1.getType(), entity1.getIdentifier());
    verifyNoOwnerSet(peer2, entity1.getType(), entity2.getIdentifier());
    verifyNoMoreInteractions(peer1Listener);
    verifyNoMoreInteractions(peer2Listener);
    // Add candidate peer1 candidate for entity2.
    peer1.tell(new RegisterCandidateLocal(entity2), kit.getRef());
    verifyOwner(peer1, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
    verify(peer1Listener, timeout(5000)).ownershipChanged(ownershipChange(entity2, false, true, true));
    verify(peer2Listener, timeout(5000)).ownershipChanged(ownershipChange(entity2, false, false, true));
    reset(leaderListener, peer1Listener, peer2Listener);
    // Remove the isolation.
    leader.underlyingActor().stopDroppingMessagesOfType(RequestVote.class);
    leader.underlyingActor().stopDroppingMessagesOfType(AppendEntries.class);
    peer2.underlyingActor().stopDroppingMessagesOfType(AppendEntries.class);
    peer1.underlyingActor().stopDroppingMessagesOfType(AppendEntries.class);
    // Previous leader should switch to Follower.
    verifyRaftState(leader, state -> assertEquals("getRaftState", RaftState.Follower.toString(), state.getRaftState()));
    // Send PeerUp to peer1 and peer2.
    peer1.tell(new PeerUp(leaderId.getMemberName(), leaderId.toString()), ActorRef.noSender());
    peer2.tell(new PeerUp(leaderId.getMemberName(), leaderId.toString()), ActorRef.noSender());
    // The previous leader should become the owner of entity1.
    verifyOwner(leader, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
    // The previous leader's DOMEntityOwnershipListener should get 4 total notifications:
    // - inJeopardy cleared for entity1 (wasOwner=false, isOwner=false, hasOwner=false, inJeopardy=false)
    // - inJeopardy cleared for entity2 (wasOwner=false, isOwner=false, hasOwner=false, inJeopardy=false)
    // - local owner granted for entity1 (wasOwner=false, isOwner=true, hasOwner=true, inJeopardy=false)
    // - remote owner for entity2 (wasOwner=false, isOwner=false, hasOwner=true, inJeopardy=false)
    verify(leaderListener, timeout(5000).times(4)).ownershipChanged(or(or(ownershipChange(entity1, false, false, false), ownershipChange(entity2, false, false, false)), or(ownershipChange(entity1, false, true, true), ownershipChange(entity2, false, false, true))));
    verify(peer1Listener, timeout(5000)).ownershipChanged(ownershipChange(entity1, false, false, true));
    verify(peer2Listener, timeout(5000)).ownershipChanged(ownershipChange(entity1, false, false, true));
    // Verify entity2's owner doesn't change.
    Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
    verifyOwner(peer1, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
    verifyNoMoreInteractions(leaderListener);
    verifyNoMoreInteractions(peer1Listener);
    verifyNoMoreInteractions(peer2Listener);
    testLog.info("testLeaderIsolationWithPendingCandidateAdded ending");
}

Example 3 with PeerDown

use of org.opendaylight.controller.cluster.datastore.messages.PeerDown in project controller by opendaylight.

the class ShardManagerTest method testShardAvailabilityOnChangeOfMemberReachability.

@Test
public void testShardAvailabilityOnChangeOfMemberReachability() throws Exception {
    LOG.info("testShardAvailabilityOnChangeOfMemberReachability starting");
    String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
    // Create an ActorSystem ShardManager actor for member-1.
    final ActorSystem system1 = newActorSystem("Member1");
    Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
    final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
    final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1, newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(new ClusterWrapperImpl(system1)).props().withDispatcher(Dispatchers.DefaultDispatcherId()), shardManagerID);
    // Create an ActorSystem ShardManager actor for member-2.
    final ActorSystem system2 = newActorSystem("Member2");
    Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
    final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
    MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2")).build());
    final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2, newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(new ClusterWrapperImpl(system2)).props().withDispatcher(Dispatchers.DefaultDispatcherId()), shardManagerID);
    new TestKit(system1) {

        {
            shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
            shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
            shardManager1.tell(new ActorInitialized(), mockShardActor1);
            shardManager2.tell(new ActorInitialized(), mockShardActor2);
            String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
            String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
            shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class), DataStoreVersions.CURRENT_VERSION), mockShardActor1);
            shardManager1.tell(new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor1);
            shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), DataStoreVersions.CURRENT_VERSION), mockShardActor2);
            shardManager2.tell(new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()), mockShardActor2);
            shardManager1.underlyingActor().waitForMemberUp();
            shardManager1.tell(new FindPrimary("default", true), getRef());
            RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
            String path = found.getPrimaryPath();
            assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
            shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"), getRef());
            shardManager1.underlyingActor().waitForUnreachableMember();
            PeerDown peerDown = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
            assertEquals("getMemberName", MEMBER_2, peerDown.getMemberName());
            MessageCollectorActor.clearMessages(mockShardActor1);
            shardManager1.tell(MockClusterWrapper.createMemberRemoved("member-2", "akka://cluster-test@127.0.0.1:2558"), getRef());
            MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
            shardManager1.tell(new FindPrimary("default", true), getRef());
            expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
            shardManager1.tell(MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"), getRef());
            shardManager1.underlyingActor().waitForReachableMember();
            PeerUp peerUp = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
            assertEquals("getMemberName", MEMBER_2, peerUp.getMemberName());
            MessageCollectorActor.clearMessages(mockShardActor1);
            shardManager1.tell(new FindPrimary("default", true), getRef());
            RemotePrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
            String path1 = found1.getPrimaryPath();
            assertTrue("Unexpected primary path " + path1, path1.contains("member-2-shard-default-config"));
            shardManager1.tell(MockClusterWrapper.createMemberUp("member-2", "akka://cluster-test@127.0.0.1:2558"), getRef());
            MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
            // Test FindPrimary wait succeeds after reachable member event.
            shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"), getRef());
            shardManager1.underlyingActor().waitForUnreachableMember();
            shardManager1.tell(new FindPrimary("default", true), getRef());
            shardManager1.tell(MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"), getRef());
            RemotePrimaryShardFound found2 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
            String path2 = found2.getPrimaryPath();
            assertTrue("Unexpected primary path " + path2, path2.contains("member-2-shard-default-config"));
        }
    };
    LOG.info("testShardAvailabilityOnChangeOfMemberReachability ending");
}

Example 4 with PeerDown

use of org.opendaylight.controller.cluster.datastore.messages.PeerDown in project controller by opendaylight.

the class EntityOwnershipShardTest method testLeaderIsolation.

@Test
public void testLeaderIsolation() throws Exception {
    testLog.info("testLeaderIsolation starting");
    final ShardTestKit kit = new ShardTestKit(getSystem());
    ShardIdentifier leaderId = newShardId(LOCAL_MEMBER_NAME);
    ShardIdentifier peerId1 = newShardId(PEER_MEMBER_1_NAME);
    ShardIdentifier peerId2 = newShardId(PEER_MEMBER_2_NAME);
    dataStoreContextBuilder.shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(4).shardIsolatedLeaderCheckIntervalInMillis(100000);
    TestActorRef<TestEntityOwnershipShard> peer1 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId1, peerMap(leaderId.toString(), peerId2.toString()), PEER_MEMBER_1_NAME)), peerId1.toString());
    peer1.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
    TestActorRef<TestEntityOwnershipShard> peer2 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId2, peerMap(leaderId.toString(), peerId1.toString()), PEER_MEMBER_2_NAME)), peerId2.toString());
    peer2.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
    dataStoreContextBuilder = DatastoreContext.newBuilderFrom(dataStoreContextBuilder.build()).shardIsolatedLeaderCheckIntervalInMillis(500);
    TestActorRef<TestEntityOwnershipShard> leader = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(leaderId, peerMap(peerId1.toString(), peerId2.toString()), LOCAL_MEMBER_NAME)), leaderId.toString());
    ShardTestKit.waitUntilLeader(leader);
    // Add entity1 candidates for all members with the leader as the owner
    DOMEntity entity1 = new DOMEntity(ENTITY_TYPE, ENTITY_ID1);
    leader.tell(new RegisterCandidateLocal(entity1), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
    peer1.tell(new RegisterCandidateLocal(entity1), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity1.getType(), entity1.getIdentifier(), PEER_MEMBER_1_NAME);
    peer2.tell(new RegisterCandidateLocal(entity1), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity1.getType(), entity1.getIdentifier(), PEER_MEMBER_2_NAME);
    verifyOwner(leader, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
    verifyOwner(peer1, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
    verifyOwner(peer2, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
    // Add entity2 candidates for all members with peer1 as the owner
    DOMEntity entity2 = new DOMEntity(ENTITY_TYPE, ENTITY_ID2);
    peer1.tell(new RegisterCandidateLocal(entity2), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
    peer2.tell(new RegisterCandidateLocal(entity2), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_2_NAME);
    leader.tell(new RegisterCandidateLocal(entity2), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity2.getType(), entity2.getIdentifier(), LOCAL_MEMBER_NAME);
    verifyOwner(leader, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
    verifyOwner(peer1, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
    verifyOwner(peer2, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
    // Add entity3 candidates for all members with peer2 as the owner
    DOMEntity entity3 = new DOMEntity(ENTITY_TYPE, ENTITY_ID3);
    peer2.tell(new RegisterCandidateLocal(entity3), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_2_NAME);
    leader.tell(new RegisterCandidateLocal(entity3), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity3.getType(), entity3.getIdentifier(), LOCAL_MEMBER_NAME);
    peer1.tell(new RegisterCandidateLocal(entity3), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_1_NAME);
    verifyOwner(leader, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_2_NAME);
    verifyOwner(peer1, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_2_NAME);
    verifyOwner(peer2, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_2_NAME);
    // Add listeners on all members
    DOMEntityOwnershipListener leaderListener = mock(DOMEntityOwnershipListener.class);
    leader.tell(new RegisterListenerLocal(leaderListener, ENTITY_TYPE), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verify(leaderListener, timeout(5000).times(3)).ownershipChanged(or(or(ownershipChange(entity1, false, true, true), ownershipChange(entity2, false, false, true)), ownershipChange(entity3, false, false, true)));
    reset(leaderListener);
    DOMEntityOwnershipListener peer1Listener = mock(DOMEntityOwnershipListener.class);
    peer1.tell(new RegisterListenerLocal(peer1Listener, ENTITY_TYPE), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verify(peer1Listener, timeout(5000).times(3)).ownershipChanged(or(or(ownershipChange(entity1, false, false, true), ownershipChange(entity2, false, true, true)), ownershipChange(entity3, false, false, true)));
    reset(peer1Listener);
    DOMEntityOwnershipListener peer2Listener = mock(DOMEntityOwnershipListener.class);
    peer2.tell(new RegisterListenerLocal(peer2Listener, ENTITY_TYPE), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verify(peer2Listener, timeout(5000).times(3)).ownershipChanged(or(or(ownershipChange(entity1, false, false, true), ownershipChange(entity2, false, false, true)), ownershipChange(entity3, false, true, true)));
    reset(peer2Listener);
    // Isolate the leader by dropping AppendEntries to the followers and incoming messages from the followers.
    leader.underlyingActor().startDroppingMessagesOfType(RequestVote.class);
    leader.underlyingActor().startDroppingMessagesOfType(AppendEntries.class);
    peer2.underlyingActor().startDroppingMessagesOfType(AppendEntries.class, ae -> ae.getLeaderId().equals(leaderId.toString()));
    peer1.underlyingActor().startDroppingMessagesOfType(AppendEntries.class);
    // Make peer1 start an election and become leader by enabling the ElectionTimeout message.
    peer1.underlyingActor().stopDroppingMessagesOfType(ElectionTimeout.class);
    // Send PeerDown to the isolated leader so it tries to re-assign ownership for the entities owned by the
    // isolated peers.
    leader.tell(new PeerDown(peerId1.getMemberName(), peerId1.toString()), ActorRef.noSender());
    leader.tell(new PeerDown(peerId2.getMemberName(), peerId2.toString()), ActorRef.noSender());
    verifyRaftState(leader, state -> assertEquals("getRaftState", RaftState.IsolatedLeader.toString(), state.getRaftState()));
    // Expect inJeopardy notification on the isolated leader.
    verify(leaderListener, timeout(5000).times(3)).ownershipChanged(or(or(ownershipChange(entity1, true, true, true, true), ownershipChange(entity2, false, false, true, true)), ownershipChange(entity3, false, false, true, true)));
    reset(leaderListener);
    verifyRaftState(peer1, state -> assertEquals("getRaftState", RaftState.Leader.toString(), state.getRaftState()));
    // Send PeerDown to the new leader peer1 so it re-assigns ownership for the entities owned by the
    // isolated leader.
    peer1.tell(new PeerDown(leaderId.getMemberName(), leaderId.toString()), ActorRef.noSender());
    verifyOwner(peer1, entity1.getType(), entity1.getIdentifier(), PEER_MEMBER_1_NAME);
    verify(peer1Listener, timeout(5000)).ownershipChanged(ownershipChange(entity1, false, true, true));
    reset(peer1Listener);
    verify(peer2Listener, timeout(5000)).ownershipChanged(ownershipChange(entity1, false, false, true));
    reset(peer2Listener);
    // Remove the isolation.
    leader.underlyingActor().stopDroppingMessagesOfType(RequestVote.class);
    leader.underlyingActor().stopDroppingMessagesOfType(AppendEntries.class);
    peer2.underlyingActor().stopDroppingMessagesOfType(AppendEntries.class);
    peer1.underlyingActor().stopDroppingMessagesOfType(AppendEntries.class);
    // Previous leader should switch to Follower and send inJeopardy cleared notifications for all entities.
    verifyRaftState(leader, state -> assertEquals("getRaftState", RaftState.Follower.toString(), state.getRaftState()));
    verify(leaderListener, timeout(5000).times(3)).ownershipChanged(or(or(ownershipChange(entity1, true, true, true), ownershipChange(entity2, false, false, true)), ownershipChange(entity3, false, false, true)));
    verifyOwner(leader, entity1.getType(), entity1.getIdentifier(), PEER_MEMBER_1_NAME);
    verify(leaderListener, timeout(5000)).ownershipChanged(ownershipChange(entity1, true, false, true));
    Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
    verifyOwner(leader, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
    verifyOwner(leader, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_2_NAME);
    verifyNoMoreInteractions(leaderListener);
    verifyNoMoreInteractions(peer1Listener);
    verifyNoMoreInteractions(peer2Listener);
    testLog.info("testLeaderIsolation ending");
}