Examples with ShardIdentifier org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier used on opensource projects

Example 26 with ShardIdentifier

use of org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier in project controller by opendaylight.

the class ShardTest method testClusteredDataChangeListenerRegistration.

@Test
public void testClusteredDataChangeListenerRegistration() throws Exception {
    new ShardTestKit(getSystem()) {

        {
            final String testName = "testClusteredDataChangeListenerRegistration";
            final ShardIdentifier followerShardID = ShardIdentifier.create("inventory", MemberName.forName(actorFactory.generateActorId(testName + "-follower")), "config");
            final ShardIdentifier leaderShardID = ShardIdentifier.create("inventory", MemberName.forName(actorFactory.generateActorId(testName + "-leader")), "config");
            final TestActorRef<Shard> followerShard = actorFactory.createTestActor(Shard.builder().id(followerShardID).datastoreContext(dataStoreContextBuilder.shardElectionTimeoutFactor(1000).build()).peerAddresses(Collections.singletonMap(leaderShardID.toString(), "akka://test/user/" + leaderShardID.toString())).schemaContextProvider(() -> SCHEMA_CONTEXT).props().withDispatcher(Dispatchers.DefaultDispatcherId()), followerShardID.toString());
            final TestActorRef<Shard> leaderShard = actorFactory.createTestActor(Shard.builder().id(leaderShardID).datastoreContext(newDatastoreContext()).peerAddresses(Collections.singletonMap(followerShardID.toString(), "akka://test/user/" + followerShardID.toString())).schemaContextProvider(() -> SCHEMA_CONTEXT).props().withDispatcher(Dispatchers.DefaultDispatcherId()), leaderShardID.toString());
            leaderShard.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
            final String leaderPath = waitUntilLeader(followerShard);
            assertEquals("Shard leader path", leaderShard.path().toString(), leaderPath);
            final YangInstanceIdentifier path = TestModel.TEST_PATH;
            final MockDataChangeListener listener = new MockDataChangeListener(1);
            final ActorRef dclActor = actorFactory.createActor(DataChangeListener.props(listener, path), actorFactory.generateActorId(testName + "-DataChangeListener"));
            followerShard.tell(new RegisterChangeListener(path, dclActor, AsyncDataBroker.DataChangeScope.BASE, true), getRef());
            final RegisterDataTreeNotificationListenerReply reply = expectMsgClass(duration("5 seconds"), RegisterDataTreeNotificationListenerReply.class);
            assertNotNull("getListenerRegistrationPath", reply.getListenerRegistrationPath());
            writeToStore(followerShard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME));
            listener.waitForChangeEvents();
        }
    };
}

Example 27 with ShardIdentifier

use of org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier in project controller by opendaylight.

the class EntityOwnershipShardTest method testDelayedEntityOwnerSelection.

@Test
public void testDelayedEntityOwnerSelection() throws Exception {
    testLog.info("testDelayedEntityOwnerSelection starting");
    final ShardTestKit kit = new ShardTestKit(getSystem());
    EntityOwnerSelectionStrategyConfig.Builder builder = EntityOwnerSelectionStrategyConfig.newBuilder().addStrategy(ENTITY_TYPE, LastCandidateSelectionStrategy.class, 500);
    dataStoreContextBuilder.shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(2);
    ShardIdentifier leaderId = newShardId(LOCAL_MEMBER_NAME);
    ShardIdentifier peerId1 = newShardId(PEER_MEMBER_1_NAME);
    ShardIdentifier peerId2 = newShardId(PEER_MEMBER_2_NAME);
    TestActorRef<TestEntityOwnershipShard> peer1 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId1, peerMap(leaderId.toString(), peerId2.toString()), PEER_MEMBER_1_NAME)), peerId1.toString());
    peer1.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
    TestActorRef<TestEntityOwnershipShard> peer2 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId2, peerMap(leaderId.toString(), peerId1.toString()), PEER_MEMBER_2_NAME)), peerId2.toString());
    peer2.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
    TestActorRef<EntityOwnershipShard> leader = actorFactory.createTestActor(newShardProps(leaderId, peerMap(peerId1.toString(), peerId2.toString()), LOCAL_MEMBER_NAME, builder.build()), leaderId.toString());
    ShardTestKit.waitUntilLeader(leader);
    DOMEntity entity = new DOMEntity(ENTITY_TYPE, ENTITY_ID1);
    // Add a remote candidate
    peer1.tell(new RegisterCandidateLocal(entity), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    // Register local
    leader.tell(new RegisterCandidateLocal(entity), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    // Verify the local candidate becomes owner
    verifyCommittedEntityCandidate(leader, entity.getType(), entity.getIdentifier(), PEER_MEMBER_1_NAME);
    verifyCommittedEntityCandidate(leader, entity.getType(), entity.getIdentifier(), LOCAL_MEMBER_NAME);
    verifyOwner(leader, entity.getType(), entity.getIdentifier(), LOCAL_MEMBER_NAME);
    testLog.info("testDelayedEntityOwnerSelection ending");
}

Example 28 with ShardIdentifier

use of org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier in project controller by opendaylight.

the class EntityOwnershipShardTest method testOnRegisterCandidateLocalWithRemoteLeader.

@Test
public void testOnRegisterCandidateLocalWithRemoteLeader() throws Exception {
    testLog.info("testOnRegisterCandidateLocalWithRemoteLeader starting");
    ShardTestKit kit = new ShardTestKit(getSystem());
    dataStoreContextBuilder.shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(2).shardBatchedModificationCount(5);
    ShardIdentifier leaderId = newShardId(PEER_MEMBER_1_NAME);
    ShardIdentifier localId = newShardId(LOCAL_MEMBER_NAME);
    TestActorRef<TestEntityOwnershipShard> leader = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(leaderId, peerMap(localId.toString()), PEER_MEMBER_1_NAME), actorFactory.createActor(MessageCollectorActor.props())), leaderId.toString());
    final TestEntityOwnershipShard leaderShard = leader.underlyingActor();
    TestActorRef<TestEntityOwnershipShard> local = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(localId, peerMap(leaderId.toString()), LOCAL_MEMBER_NAME)), localId.toString());
    local.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
    local.tell(new RegisterCandidateLocal(new DOMEntity(ENTITY_TYPE, ENTITY_ID1)), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, ENTITY_TYPE, ENTITY_ID1, LOCAL_MEMBER_NAME);
    verifyOwner(leader, ENTITY_TYPE, ENTITY_ID1, LOCAL_MEMBER_NAME);
    // Test with initial commit timeout and subsequent retry.
    local.tell(dataStoreContextBuilder.shardTransactionCommitTimeoutInSeconds(1).build(), ActorRef.noSender());
    leaderShard.startDroppingMessagesOfType(BatchedModifications.class);
    local.tell(new RegisterCandidateLocal(new DOMEntity(ENTITY_TYPE, ENTITY_ID2)), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    expectFirstMatching(leaderShard.collectorActor(), BatchedModifications.class);
    // Send a bunch of registration messages quickly and verify.
    leaderShard.stopDroppingMessagesOfType(BatchedModifications.class);
    clearMessages(leaderShard.collectorActor());
    int max = 100;
    List<YangInstanceIdentifier> entityIds = new ArrayList<>();
    for (int i = 1; i <= max; i++) {
        YangInstanceIdentifier id = YangInstanceIdentifier.of(QName.create("test", "2015-08-14", "test" + i));
        entityIds.add(id);
        local.tell(new RegisterCandidateLocal(new DOMEntity(ENTITY_TYPE, id)), kit.getRef());
    }
    for (int i = 0; i < max; i++) {
        verifyCommittedEntityCandidate(local, ENTITY_TYPE, entityIds.get(i), LOCAL_MEMBER_NAME);
    }
    testLog.info("testOnRegisterCandidateLocalWithRemoteLeader ending");
}

Example 29 with ShardIdentifier

use of org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier in project controller by opendaylight.

the class EntityOwnershipShardTest method testOwnershipChanges.

@Test
public void testOwnershipChanges() throws Exception {
    testLog.info("testOwnershipChanges starting");
    final ShardTestKit kit = new ShardTestKit(getSystem());
    dataStoreContextBuilder.shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(2);
    ShardIdentifier leaderId = newShardId(LOCAL_MEMBER_NAME);
    ShardIdentifier peerId1 = newShardId(PEER_MEMBER_1_NAME);
    ShardIdentifier peerId2 = newShardId(PEER_MEMBER_2_NAME);
    TestActorRef<TestEntityOwnershipShard> peer1 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId1, peerMap(leaderId.toString(), peerId2.toString()), PEER_MEMBER_1_NAME)), peerId1.toString());
    peer1.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
    TestActorRef<TestEntityOwnershipShard> peer2 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId2, peerMap(leaderId.toString(), peerId1.toString()), PEER_MEMBER_2_NAME)), peerId2.toString());
    peer2.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
    TestActorRef<EntityOwnershipShard> leader = actorFactory.createTestActor(newShardProps(leaderId, peerMap(peerId1.toString(), peerId2.toString()), LOCAL_MEMBER_NAME), leaderId.toString());
    ShardTestKit.waitUntilLeader(leader);
    DOMEntity entity = new DOMEntity(ENTITY_TYPE, ENTITY_ID1);
    // Add a remote candidate
    peer1.tell(new RegisterCandidateLocal(entity), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity.getType(), entity.getIdentifier(), PEER_MEMBER_1_NAME);
    // Register local
    leader.tell(new RegisterCandidateLocal(entity), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    // Verify the remote candidate becomes owner
    verifyCommittedEntityCandidate(leader, entity.getType(), entity.getIdentifier(), LOCAL_MEMBER_NAME);
    verifyOwner(leader, entity.getType(), entity.getIdentifier(), PEER_MEMBER_1_NAME);
    // Add another remote candidate and verify ownership doesn't change
    peer2.tell(new RegisterCandidateLocal(entity), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity.getType(), entity.getIdentifier(), PEER_MEMBER_2_NAME);
    Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
    verifyOwner(leader, entity.getType(), entity.getIdentifier(), PEER_MEMBER_1_NAME);
    // Remove the second remote candidate and verify ownership doesn't change
    peer2.tell(new UnregisterCandidateLocal(entity), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyEntityCandidateRemoved(leader, entity.getType(), entity.getIdentifier(), PEER_MEMBER_2_NAME);
    Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
    verifyOwner(leader, entity.getType(), entity.getIdentifier(), PEER_MEMBER_1_NAME);
    // Remove the first remote candidate and verify the local candidate becomes owner
    peer1.tell(new UnregisterCandidateLocal(entity), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyEntityCandidateRemoved(leader, entity.getType(), entity.getIdentifier(), PEER_MEMBER_1_NAME);
    verifyOwner(leader, entity.getType(), entity.getIdentifier(), LOCAL_MEMBER_NAME);
    // Add the second remote candidate back and verify ownership doesn't change
    peer2.tell(new RegisterCandidateLocal(entity), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity.getType(), entity.getIdentifier(), PEER_MEMBER_2_NAME);
    Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
    verifyOwner(leader, entity.getType(), entity.getIdentifier(), LOCAL_MEMBER_NAME);
    // Unregister the local candidate and verify the second remote candidate becomes owner
    leader.tell(new UnregisterCandidateLocal(entity), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyEntityCandidateRemoved(leader, entity.getType(), entity.getIdentifier(), LOCAL_MEMBER_NAME);
    verifyOwner(leader, entity.getType(), entity.getIdentifier(), PEER_MEMBER_2_NAME);
    testLog.info("testOwnershipChanges ending");
}

Example 30 with ShardIdentifier

use of org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier in project controller by opendaylight.

the class EntityOwnershipShardTest method testLeaderIsolation.

@Test
public void testLeaderIsolation() throws Exception {
    testLog.info("testLeaderIsolation starting");
    final ShardTestKit kit = new ShardTestKit(getSystem());
    ShardIdentifier leaderId = newShardId(LOCAL_MEMBER_NAME);
    ShardIdentifier peerId1 = newShardId(PEER_MEMBER_1_NAME);
    ShardIdentifier peerId2 = newShardId(PEER_MEMBER_2_NAME);
    dataStoreContextBuilder.shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(4).shardIsolatedLeaderCheckIntervalInMillis(100000);
    TestActorRef<TestEntityOwnershipShard> peer1 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId1, peerMap(leaderId.toString(), peerId2.toString()), PEER_MEMBER_1_NAME)), peerId1.toString());
    peer1.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
    TestActorRef<TestEntityOwnershipShard> peer2 = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(peerId2, peerMap(leaderId.toString(), peerId1.toString()), PEER_MEMBER_2_NAME)), peerId2.toString());
    peer2.underlyingActor().startDroppingMessagesOfType(ElectionTimeout.class);
    dataStoreContextBuilder = DatastoreContext.newBuilderFrom(dataStoreContextBuilder.build()).shardIsolatedLeaderCheckIntervalInMillis(500);
    TestActorRef<TestEntityOwnershipShard> leader = actorFactory.createTestActor(TestEntityOwnershipShard.props(newShardBuilder(leaderId, peerMap(peerId1.toString(), peerId2.toString()), LOCAL_MEMBER_NAME)), leaderId.toString());
    ShardTestKit.waitUntilLeader(leader);
    // Add entity1 candidates for all members with the leader as the owner
    DOMEntity entity1 = new DOMEntity(ENTITY_TYPE, ENTITY_ID1);
    leader.tell(new RegisterCandidateLocal(entity1), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
    peer1.tell(new RegisterCandidateLocal(entity1), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity1.getType(), entity1.getIdentifier(), PEER_MEMBER_1_NAME);
    peer2.tell(new RegisterCandidateLocal(entity1), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity1.getType(), entity1.getIdentifier(), PEER_MEMBER_2_NAME);
    verifyOwner(leader, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
    verifyOwner(peer1, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
    verifyOwner(peer2, entity1.getType(), entity1.getIdentifier(), LOCAL_MEMBER_NAME);
    // Add entity2 candidates for all members with peer1 as the owner
    DOMEntity entity2 = new DOMEntity(ENTITY_TYPE, ENTITY_ID2);
    peer1.tell(new RegisterCandidateLocal(entity2), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
    peer2.tell(new RegisterCandidateLocal(entity2), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_2_NAME);
    leader.tell(new RegisterCandidateLocal(entity2), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity2.getType(), entity2.getIdentifier(), LOCAL_MEMBER_NAME);
    verifyOwner(leader, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
    verifyOwner(peer1, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
    verifyOwner(peer2, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
    // Add entity3 candidates for all members with peer2 as the owner
    DOMEntity entity3 = new DOMEntity(ENTITY_TYPE, ENTITY_ID3);
    peer2.tell(new RegisterCandidateLocal(entity3), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_2_NAME);
    leader.tell(new RegisterCandidateLocal(entity3), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity3.getType(), entity3.getIdentifier(), LOCAL_MEMBER_NAME);
    peer1.tell(new RegisterCandidateLocal(entity3), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verifyCommittedEntityCandidate(leader, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_1_NAME);
    verifyOwner(leader, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_2_NAME);
    verifyOwner(peer1, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_2_NAME);
    verifyOwner(peer2, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_2_NAME);
    // Add listeners on all members
    DOMEntityOwnershipListener leaderListener = mock(DOMEntityOwnershipListener.class);
    leader.tell(new RegisterListenerLocal(leaderListener, ENTITY_TYPE), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verify(leaderListener, timeout(5000).times(3)).ownershipChanged(or(or(ownershipChange(entity1, false, true, true), ownershipChange(entity2, false, false, true)), ownershipChange(entity3, false, false, true)));
    reset(leaderListener);
    DOMEntityOwnershipListener peer1Listener = mock(DOMEntityOwnershipListener.class);
    peer1.tell(new RegisterListenerLocal(peer1Listener, ENTITY_TYPE), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verify(peer1Listener, timeout(5000).times(3)).ownershipChanged(or(or(ownershipChange(entity1, false, false, true), ownershipChange(entity2, false, true, true)), ownershipChange(entity3, false, false, true)));
    reset(peer1Listener);
    DOMEntityOwnershipListener peer2Listener = mock(DOMEntityOwnershipListener.class);
    peer2.tell(new RegisterListenerLocal(peer2Listener, ENTITY_TYPE), kit.getRef());
    kit.expectMsgClass(SuccessReply.class);
    verify(peer2Listener, timeout(5000).times(3)).ownershipChanged(or(or(ownershipChange(entity1, false, false, true), ownershipChange(entity2, false, false, true)), ownershipChange(entity3, false, true, true)));
    reset(peer2Listener);
    // Isolate the leader by dropping AppendEntries to the followers and incoming messages from the followers.
    leader.underlyingActor().startDroppingMessagesOfType(RequestVote.class);
    leader.underlyingActor().startDroppingMessagesOfType(AppendEntries.class);
    peer2.underlyingActor().startDroppingMessagesOfType(AppendEntries.class, ae -> ae.getLeaderId().equals(leaderId.toString()));
    peer1.underlyingActor().startDroppingMessagesOfType(AppendEntries.class);
    // Make peer1 start an election and become leader by enabling the ElectionTimeout message.
    peer1.underlyingActor().stopDroppingMessagesOfType(ElectionTimeout.class);
    // Send PeerDown to the isolated leader so it tries to re-assign ownership for the entities owned by the
    // isolated peers.
    leader.tell(new PeerDown(peerId1.getMemberName(), peerId1.toString()), ActorRef.noSender());
    leader.tell(new PeerDown(peerId2.getMemberName(), peerId2.toString()), ActorRef.noSender());
    verifyRaftState(leader, state -> assertEquals("getRaftState", RaftState.IsolatedLeader.toString(), state.getRaftState()));
    // Expect inJeopardy notification on the isolated leader.
    verify(leaderListener, timeout(5000).times(3)).ownershipChanged(or(or(ownershipChange(entity1, true, true, true, true), ownershipChange(entity2, false, false, true, true)), ownershipChange(entity3, false, false, true, true)));
    reset(leaderListener);
    verifyRaftState(peer1, state -> assertEquals("getRaftState", RaftState.Leader.toString(), state.getRaftState()));
    // Send PeerDown to the new leader peer1 so it re-assigns ownership for the entities owned by the
    // isolated leader.
    peer1.tell(new PeerDown(leaderId.getMemberName(), leaderId.toString()), ActorRef.noSender());
    verifyOwner(peer1, entity1.getType(), entity1.getIdentifier(), PEER_MEMBER_1_NAME);
    verify(peer1Listener, timeout(5000)).ownershipChanged(ownershipChange(entity1, false, true, true));
    reset(peer1Listener);
    verify(peer2Listener, timeout(5000)).ownershipChanged(ownershipChange(entity1, false, false, true));
    reset(peer2Listener);
    // Remove the isolation.
    leader.underlyingActor().stopDroppingMessagesOfType(RequestVote.class);
    leader.underlyingActor().stopDroppingMessagesOfType(AppendEntries.class);
    peer2.underlyingActor().stopDroppingMessagesOfType(AppendEntries.class);
    peer1.underlyingActor().stopDroppingMessagesOfType(AppendEntries.class);
    // Previous leader should switch to Follower and send inJeopardy cleared notifications for all entities.
    verifyRaftState(leader, state -> assertEquals("getRaftState", RaftState.Follower.toString(), state.getRaftState()));
    verify(leaderListener, timeout(5000).times(3)).ownershipChanged(or(or(ownershipChange(entity1, true, true, true), ownershipChange(entity2, false, false, true)), ownershipChange(entity3, false, false, true)));
    verifyOwner(leader, entity1.getType(), entity1.getIdentifier(), PEER_MEMBER_1_NAME);
    verify(leaderListener, timeout(5000)).ownershipChanged(ownershipChange(entity1, true, false, true));
    Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
    verifyOwner(leader, entity2.getType(), entity2.getIdentifier(), PEER_MEMBER_1_NAME);
    verifyOwner(leader, entity3.getType(), entity3.getIdentifier(), PEER_MEMBER_2_NAME);
    verifyNoMoreInteractions(leaderListener);
    verifyNoMoreInteractions(peer1Listener);
    verifyNoMoreInteractions(peer2Listener);
    testLog.info("testLeaderIsolation ending");
}