Examples with MemberNode org.opendaylight.controller.cluster.datastore.MemberNode used on opensource projects

Example 21 with MemberNode

use of org.opendaylight.controller.cluster.datastore.MemberNode in project controller by opendaylight.

the class ClusterAdminRpcServiceTest method testFlipMemberVotingStatesWithVotingMembersDown.

@Test
public void testFlipMemberVotingStatesWithVotingMembersDown() throws Exception {
    String name = "testFlipMemberVotingStatesWithVotingMembersDown";
    // Members 4, 5, and 6 are initially non-voting and simulated as down by not starting them up.
    ServerConfigurationPayload persistedServerConfig = new ServerConfigurationPayload(Arrays.asList(new ServerInfo("member-1", true), new ServerInfo("member-2", true), new ServerInfo("member-3", true), new ServerInfo("member-4", false), new ServerInfo("member-5", false), new ServerInfo("member-6", false)));
    setupPersistedServerConfigPayload(persistedServerConfig, "member-1", name, "cars", "people");
    setupPersistedServerConfigPayload(persistedServerConfig, "member-2", name, "cars", "people");
    setupPersistedServerConfigPayload(persistedServerConfig, "member-3", name, "cars", "people");
    String moduleShardsConfig = "module-shards-member1-and-2-and-3.conf";
    final MemberNode leaderNode1 = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name).moduleShardsConfig(moduleShardsConfig).datastoreContextBuilder(DatastoreContext.newBuilder().shardHeartbeatIntervalInMillis(300).shardElectionTimeoutFactor(1)).build();
    final MemberNode replicaNode2 = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name).moduleShardsConfig(moduleShardsConfig).build();
    final MemberNode replicaNode3 = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name).moduleShardsConfig(moduleShardsConfig).build();
    leaderNode1.configDataStore().waitTillReady();
    leaderNode1.operDataStore().waitTillReady();
    verifyVotingStates(leaderNode1.configDataStore(), "cars", new SimpleEntry<>("member-1", TRUE), new SimpleEntry<>("member-2", TRUE), new SimpleEntry<>("member-3", TRUE), new SimpleEntry<>("member-4", FALSE), new SimpleEntry<>("member-5", FALSE), new SimpleEntry<>("member-6", FALSE));
    ClusterAdminRpcService service1 = new ClusterAdminRpcService(leaderNode1.configDataStore(), leaderNode1.operDataStore(), null);
    RpcResult<FlipMemberVotingStatesForAllShardsOutput> rpcResult = service1.flipMemberVotingStatesForAllShards().get(10, TimeUnit.SECONDS);
    FlipMemberVotingStatesForAllShardsOutput result = verifySuccessfulRpcResult(rpcResult);
    verifyShardResults(result.getShardResult(), successShardResult("cars", DataStoreType.Config), successShardResult("people", DataStoreType.Config), successShardResult("cars", DataStoreType.Operational), successShardResult("people", DataStoreType.Operational));
    // Members 2 and 3 are now non-voting but should get replicated with the new new server config.
    verifyVotingStates(new AbstractDataStore[] { leaderNode1.configDataStore(), leaderNode1.operDataStore(), replicaNode2.configDataStore(), replicaNode2.operDataStore(), replicaNode3.configDataStore(), replicaNode3.operDataStore() }, new String[] { "cars", "people" }, new SimpleEntry<>("member-1", FALSE), new SimpleEntry<>("member-2", FALSE), new SimpleEntry<>("member-3", FALSE), new SimpleEntry<>("member-4", TRUE), new SimpleEntry<>("member-5", TRUE), new SimpleEntry<>("member-6", TRUE));
    // The leader (member 1) was changed to non-voting but it shouldn't be able to step down as leader yet
    // b/c it can't get a majority consensus with all voting members down. So verify it remains the leader.
    verifyRaftState(leaderNode1.configDataStore(), "cars", raftState -> {
        assertNotNull("Expected non-null leader Id", raftState.getLeader());
        assertTrue("Expected leader member-1", raftState.getLeader().contains("member-1"));
    });
}

Example 22 with MemberNode

use of org.opendaylight.controller.cluster.datastore.MemberNode in project controller by opendaylight.

the class ClusterAdminRpcServiceTest method testFlipMemberVotingStatesWithNoInitialLeader.

@Test
public void testFlipMemberVotingStatesWithNoInitialLeader() throws Exception {
    String name = "testFlipMemberVotingStatesWithNoInitialLeader";
    // Members 1, 2, and 3 are initially started up as non-voting. Members 4, 5, and 6 are initially
    // non-voting and simulated as down by not starting them up.
    ServerConfigurationPayload persistedServerConfig = new ServerConfigurationPayload(Arrays.asList(new ServerInfo("member-1", false), new ServerInfo("member-2", false), new ServerInfo("member-3", false), new ServerInfo("member-4", true), new ServerInfo("member-5", true), new ServerInfo("member-6", true)));
    setupPersistedServerConfigPayload(persistedServerConfig, "member-1", name, "cars", "people");
    setupPersistedServerConfigPayload(persistedServerConfig, "member-2", name, "cars", "people");
    setupPersistedServerConfigPayload(persistedServerConfig, "member-3", name, "cars", "people");
    String moduleShardsConfig = "module-shards-member1-and-2-and-3.conf";
    final MemberNode replicaNode1 = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name).moduleShardsConfig(moduleShardsConfig).datastoreContextBuilder(DatastoreContext.newBuilder().shardHeartbeatIntervalInMillis(300).shardElectionTimeoutFactor(1)).build();
    final MemberNode replicaNode2 = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name).moduleShardsConfig(moduleShardsConfig).build();
    final MemberNode replicaNode3 = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name).moduleShardsConfig(moduleShardsConfig).build();
    // Initially there won't be a leader b/c all the up nodes are non-voting.
    replicaNode1.waitForMembersUp("member-2", "member-3");
    verifyVotingStates(replicaNode1.configDataStore(), "cars", new SimpleEntry<>("member-1", FALSE), new SimpleEntry<>("member-2", FALSE), new SimpleEntry<>("member-3", FALSE), new SimpleEntry<>("member-4", TRUE), new SimpleEntry<>("member-5", TRUE), new SimpleEntry<>("member-6", TRUE));
    verifyRaftState(replicaNode1.configDataStore(), "cars", raftState -> assertEquals("Expected raft state", RaftState.Follower.toString(), raftState.getRaftState()));
    ClusterAdminRpcService service1 = new ClusterAdminRpcService(replicaNode1.configDataStore(), replicaNode1.operDataStore(), null);
    RpcResult<FlipMemberVotingStatesForAllShardsOutput> rpcResult = service1.flipMemberVotingStatesForAllShards().get(10, TimeUnit.SECONDS);
    FlipMemberVotingStatesForAllShardsOutput result = verifySuccessfulRpcResult(rpcResult);
    verifyShardResults(result.getShardResult(), successShardResult("cars", DataStoreType.Config), successShardResult("people", DataStoreType.Config), successShardResult("cars", DataStoreType.Operational), successShardResult("people", DataStoreType.Operational));
    verifyVotingStates(new AbstractDataStore[] { replicaNode1.configDataStore(), replicaNode1.operDataStore(), replicaNode2.configDataStore(), replicaNode2.operDataStore(), replicaNode3.configDataStore(), replicaNode3.operDataStore() }, new String[] { "cars", "people" }, new SimpleEntry<>("member-1", TRUE), new SimpleEntry<>("member-2", TRUE), new SimpleEntry<>("member-3", TRUE), new SimpleEntry<>("member-4", FALSE), new SimpleEntry<>("member-5", FALSE), new SimpleEntry<>("member-6", FALSE));
    // Since member 1 was changed to voting and there was no leader, it should've started and election
    // and become leader
    verifyRaftState(replicaNode1.configDataStore(), "cars", raftState -> {
        assertNotNull("Expected non-null leader Id", raftState.getLeader());
        assertTrue("Expected leader member-1. Actual: " + raftState.getLeader(), raftState.getLeader().contains("member-1"));
    });
    verifyRaftState(replicaNode1.operDataStore(), "cars", raftState -> {
        assertNotNull("Expected non-null leader Id", raftState.getLeader());
        assertTrue("Expected leader member-1. Actual: " + raftState.getLeader(), raftState.getLeader().contains("member-1"));
    });
}

Example 23 with MemberNode

use of org.opendaylight.controller.cluster.datastore.MemberNode in project controller by opendaylight.

the class ClusterAdminRpcServiceTest method testChangeMemberVotingStatesForShard.

@Test
public void testChangeMemberVotingStatesForShard() throws Exception {
    String name = "testChangeMemberVotingStatusForShard";
    String moduleShardsConfig = "module-shards-member1-and-2-and-3.conf";
    final MemberNode leaderNode1 = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name).moduleShardsConfig(moduleShardsConfig).datastoreContextBuilder(DatastoreContext.newBuilder().shardHeartbeatIntervalInMillis(300).shardElectionTimeoutFactor(1)).build();
    final MemberNode replicaNode2 = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name).moduleShardsConfig(moduleShardsConfig).build();
    final MemberNode replicaNode3 = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name).moduleShardsConfig(moduleShardsConfig).build();
    leaderNode1.configDataStore().waitTillReady();
    replicaNode3.configDataStore().waitTillReady();
    verifyRaftPeersPresent(leaderNode1.configDataStore(), "cars", "member-2", "member-3");
    verifyRaftPeersPresent(replicaNode2.configDataStore(), "cars", "member-1", "member-3");
    verifyRaftPeersPresent(replicaNode3.configDataStore(), "cars", "member-1", "member-2");
    // Invoke RPC service on member-3 to change voting status
    ClusterAdminRpcService service3 = new ClusterAdminRpcService(replicaNode3.configDataStore(), replicaNode3.operDataStore(), null);
    RpcResult<Void> rpcResult = service3.changeMemberVotingStatesForShard(new ChangeMemberVotingStatesForShardInputBuilder().setShardName("cars").setDataStoreType(DataStoreType.Config).setMemberVotingState(ImmutableList.of(new MemberVotingStateBuilder().setMemberName("member-2").setVoting(FALSE).build(), new MemberVotingStateBuilder().setMemberName("member-3").setVoting(FALSE).build())).build()).get(10, TimeUnit.SECONDS);
    verifySuccessfulRpcResult(rpcResult);
    verifyVotingStates(leaderNode1.configDataStore(), "cars", new SimpleEntry<>("member-1", TRUE), new SimpleEntry<>("member-2", FALSE), new SimpleEntry<>("member-3", FALSE));
    verifyVotingStates(replicaNode2.configDataStore(), "cars", new SimpleEntry<>("member-1", TRUE), new SimpleEntry<>("member-2", FALSE), new SimpleEntry<>("member-3", FALSE));
    verifyVotingStates(replicaNode3.configDataStore(), "cars", new SimpleEntry<>("member-1", TRUE), new SimpleEntry<>("member-2", FALSE), new SimpleEntry<>("member-3", FALSE));
}

Example 24 with MemberNode

use of org.opendaylight.controller.cluster.datastore.MemberNode in project controller by opendaylight.

the class DistributedEntityOwnershipIntegrationTest method testEntityOwnershipShardBootstrapping.

/**
 * Tests bootstrapping the entity-ownership shard when there's no shards initially configured for local
 * member. The entity-ownership shard is initially created as inactive (ie remains a follower), requiring
 * an AddShardReplica request to join it to an existing leader.
 */
@Test
public void testEntityOwnershipShardBootstrapping() throws Exception {
    String name = "testEntityOwnershipShardBootstrapping";
    String moduleShardsConfig = MODULE_SHARDS_MEMBER_1_CONFIG;
    MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name).moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(leaderDatastoreContextBuilder).build();
    AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
    final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
    leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
    MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name).moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
    AbstractDataStore follower1DistributedDataStore = follower1Node.configDataStore();
    follower1DistributedDataStore.waitTillReady();
    leaderNode.waitForMembersUp("member-2");
    follower1Node.waitForMembersUp("member-1");
    DOMEntityOwnershipService follower1EntityOwnershipService = newOwnershipService(follower1DistributedDataStore);
    leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
    // Register a candidate for follower1 - should get queued since follower1 has no leader
    final DOMEntityOwnershipCandidateRegistration candidateReg = follower1EntityOwnershipService.registerCandidate(ENTITY1);
    Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
    verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
    // Add replica in follower1
    AddShardReplica addReplica = new AddShardReplica(ENTITY_OWNERSHIP_SHARD_NAME);
    follower1DistributedDataStore.getActorContext().getShardManager().tell(addReplica, follower1Node.kit().getRef());
    Object reply = follower1Node.kit().expectMsgAnyClassOf(follower1Node.kit().duration("5 sec"), Success.class, Failure.class);
    if (reply instanceof Failure) {
        throw new AssertionError("AddShardReplica failed", ((Failure) reply).cause());
    }
    // The queued candidate registration should proceed
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
    reset(leaderMockListener);
    candidateReg.close();
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, false));
    reset(leaderMockListener);
    // Restart follower1 and verify the entity ownership shard is re-instated by registering.
    Cluster.get(leaderNode.kit().getSystem()).down(Cluster.get(follower1Node.kit().getSystem()).selfAddress());
    follower1Node.cleanup();
    follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name).moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
    follower1EntityOwnershipService = newOwnershipService(follower1Node.configDataStore());
    follower1EntityOwnershipService.registerCandidate(ENTITY1);
    verify(leaderMockListener, timeout(20000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
    verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME, raftState -> {
        assertNull("Custom RaftPolicy class name", raftState.getCustomRaftPolicyClassName());
        assertEquals("Peer count", 1, raftState.getPeerAddresses().keySet().size());
        assertThat("Peer Id", Iterables.<String>getLast(raftState.getPeerAddresses().keySet()), org.hamcrest.CoreMatchers.containsString("member-1"));
    });
}

Example 25 with MemberNode

use of org.opendaylight.controller.cluster.datastore.MemberNode in project controller by opendaylight.

the class DistributedEntityOwnershipIntegrationTest method testFunctionalityWithThreeNodes.

@Test
public void testFunctionalityWithThreeNodes() throws Exception {
    String name = "testFunctionalityWithThreeNodes";
    MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name).moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(leaderDatastoreContextBuilder).build();
    MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name).moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
    MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name).moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
    AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
    leaderDistributedDataStore.waitTillReady();
    follower1Node.configDataStore().waitTillReady();
    follower2Node.configDataStore().waitTillReady();
    final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
    final DOMEntityOwnershipService follower1EntityOwnershipService = newOwnershipService(follower1Node.configDataStore());
    final DOMEntityOwnershipService follower2EntityOwnershipService = newOwnershipService(follower2Node.configDataStore());
    leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
    leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
    leaderEntityOwnershipService.registerListener(ENTITY_TYPE2, leaderMockListener2);
    follower1EntityOwnershipService.registerListener(ENTITY_TYPE1, follower1MockListener);
    // Register leader candidate for entity1 and verify it becomes owner
    leaderEntityOwnershipService.registerCandidate(ENTITY1);
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, true, true));
    verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
    reset(leaderMockListener, follower1MockListener);
    verifyGetOwnershipState(leaderEntityOwnershipService, ENTITY1, EntityOwnershipState.IS_OWNER);
    verifyGetOwnershipState(follower1EntityOwnershipService, ENTITY1, EntityOwnershipState.OWNED_BY_OTHER);
    // Register leader candidate for entity1_2 (same id, different type) and verify it becomes owner
    leaderEntityOwnershipService.registerCandidate(ENTITY1_2);
    verify(leaderMockListener2, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1_2, false, true, true));
    Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
    verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1_2));
    reset(leaderMockListener2);
    // Register follower1 candidate for entity1 and verify it gets added but doesn't become owner
    follower1EntityOwnershipService.registerCandidate(ENTITY1);
    verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-1", "member-2");
    verifyOwner(leaderDistributedDataStore, ENTITY1, "member-1");
    verifyOwner(follower2Node.configDataStore(), ENTITY1, "member-1");
    Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
    verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
    verify(follower1MockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
    // Register follower1 candidate for entity2 and verify it becomes owner
    final DOMEntityOwnershipCandidateRegistration follower1Entity2Reg = follower1EntityOwnershipService.registerCandidate(ENTITY2);
    verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, true));
    verifyOwner(follower2Node.configDataStore(), ENTITY2, "member-2");
    reset(leaderMockListener, follower1MockListener);
    // Register follower2 candidate for entity2 and verify it gets added but doesn't become owner
    follower2EntityOwnershipService.registerListener(ENTITY_TYPE1, follower2MockListener);
    verify(follower2MockListener, timeout(5000).times(2)).ownershipChanged(or(ownershipChange(ENTITY1, false, false, true), ownershipChange(ENTITY2, false, false, true)));
    follower2EntityOwnershipService.registerCandidate(ENTITY2);
    verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-2", "member-3");
    verifyOwner(leaderDistributedDataStore, ENTITY2, "member-2");
    // Unregister follower1 candidate for entity2 and verify follower2 becomes owner
    follower1Entity2Reg.close();
    verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-3");
    verifyOwner(leaderDistributedDataStore, ENTITY2, "member-3");
    verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, true, false, true));
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, true));
    // Depending on timing, follower2MockListener could get ownershipChanged with "false, false, true" if
    // if the original ownership change with "member-2 is replicated to follower2 after the listener is
    // registered.
    Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
    verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
    // Register follower1 candidate for entity3 and verify it becomes owner
    follower1EntityOwnershipService.registerCandidate(ENTITY3);
    verifyOwner(leaderDistributedDataStore, ENTITY3, "member-2");
    verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, true, true));
    verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, false, true));
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, false, true));
    // Register follower2 candidate for entity4 and verify it becomes owner
    follower2EntityOwnershipService.registerCandidate(ENTITY4);
    verifyOwner(leaderDistributedDataStore, ENTITY4, "member-3");
    verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, true, true));
    verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
    reset(follower1MockListener, follower2MockListener);
    // Register follower1 candidate for entity4 and verify it gets added but doesn't become owner
    follower1EntityOwnershipService.registerCandidate(ENTITY4);
    verifyCandidates(leaderDistributedDataStore, ENTITY4, "member-3", "member-2");
    verifyOwner(leaderDistributedDataStore, ENTITY4, "member-3");
    // Shutdown follower2 and verify it's owned entities (entity 4) get re-assigned
    reset(leaderMockListener, follower1MockListener);
    follower2Node.cleanup();
    verify(follower1MockListener, timeout(15000)).ownershipChanged(ownershipChange(ENTITY4, false, true, true));
    verify(leaderMockListener, timeout(15000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
    // Register leader candidate for entity2 and verify it becomes owner
    DOMEntityOwnershipCandidateRegistration leaderEntity2Reg = leaderEntityOwnershipService.registerCandidate(ENTITY2);
    verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
    // Unregister leader candidate for entity2 and verify the owner is cleared
    leaderEntity2Reg.close();
    verifyOwner(leaderDistributedDataStore, ENTITY2, "");
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, true, false, false));
    verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, false));
}