Examples with AbstractDataStore org.opendaylight.controller.cluster.datastore.AbstractDataStore used on opensource projects

Example 6 with AbstractDataStore

use of org.opendaylight.controller.cluster.datastore.AbstractDataStore in project controller by opendaylight.

the class DistributedEntityOwnershipIntegrationTest method testLeaderAndFollowerEntityOwnersReassignedAfterShutdown.

@Test
public void testLeaderAndFollowerEntityOwnersReassignedAfterShutdown() throws Exception {
    followerDatastoreContextBuilder.shardElectionTimeoutFactor(5).customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName());
    String name = "testLeaderAndFollowerEntityOwnersReassignedAfterShutdown";
    final MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").useAkkaArtery(false).testName(name).moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(leaderDatastoreContextBuilder).build();
    final MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").useAkkaArtery(false).testName(name).moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
    final MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").useAkkaArtery(false).testName(name).moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
    final MemberNode follower3Node = MemberNode.builder(memberNodes).akkaConfig("Member4").useAkkaArtery(false).testName(name).moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
    final MemberNode follower4Node = MemberNode.builder(memberNodes).akkaConfig("Member5").useAkkaArtery(false).testName(name).moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
    AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
    leaderDistributedDataStore.waitTillReady();
    follower1Node.configDataStore().waitTillReady();
    follower2Node.configDataStore().waitTillReady();
    follower3Node.configDataStore().waitTillReady();
    follower4Node.configDataStore().waitTillReady();
    leaderNode.waitForMembersUp("member-2", "member-3", "member-4", "member-5");
    follower1Node.waitForMembersUp("member-1", "member-3", "member-4", "member-5");
    final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
    final DOMEntityOwnershipService follower1EntityOwnershipService = newOwnershipService(follower1Node.configDataStore());
    final DOMEntityOwnershipService follower2EntityOwnershipService = newOwnershipService(follower2Node.configDataStore());
    final DOMEntityOwnershipService follower3EntityOwnershipService = newOwnershipService(follower3Node.configDataStore());
    newOwnershipService(follower4Node.configDataStore());
    leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
    // Register follower1 candidate for entity1 and verify it becomes owner
    follower1EntityOwnershipService.registerCandidate(ENTITY1);
    verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
    // Register leader candidate for entity1
    leaderEntityOwnershipService.registerCandidate(ENTITY1);
    verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2", "member-1");
    verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
    // Register leader candidate for entity2 and verify it becomes owner
    leaderEntityOwnershipService.registerCandidate(ENTITY2);
    verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
    // Register follower2 candidate for entity2
    follower2EntityOwnershipService.registerCandidate(ENTITY2);
    verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3");
    verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
    // Register follower3 as a candidate for entity2 as well
    follower3EntityOwnershipService.registerCandidate(ENTITY2);
    verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3", "member-4");
    verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
    // Re-enable elections on all remaining followers so one becomes the new leader
    ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
    follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build()).customRaftPolicyImplementation(null).build(), ActorRef.noSender());
    ActorRef follower2Shard = IntegrationTestKit.findLocalShard(follower2Node.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
    follower2Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build()).customRaftPolicyImplementation(null).build(), ActorRef.noSender());
    ActorRef follower4Shard = IntegrationTestKit.findLocalShard(follower4Node.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
    follower4Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build()).customRaftPolicyImplementation(null).build(), ActorRef.noSender());
    // Shutdown the leader and follower3
    leaderNode.cleanup();
    follower3Node.cleanup();
    follower1Node.waitForMemberDown("member-1");
    follower1Node.waitForMemberDown("member-4");
    follower2Node.waitForMemberDown("member-1");
    follower2Node.waitForMemberDown("member-4");
    follower4Node.waitForMemberDown("member-1");
    follower4Node.waitForMemberDown("member-4");
    // Verify the prior leader's and follower3 entity owners are re-assigned.
    verifyCandidates(follower1Node.configDataStore(), ENTITY1, "member-2", "member-1");
    verifyCandidates(follower1Node.configDataStore(), ENTITY2, "member-1", "member-3", "member-4");
    verifyOwner(follower1Node.configDataStore(), ENTITY1, "member-2");
    verifyOwner(follower1Node.configDataStore(), ENTITY2, "member-3");
}

Example 7 with AbstractDataStore

use of org.opendaylight.controller.cluster.datastore.AbstractDataStore in project controller by opendaylight.

the class DistributedEntityOwnershipIntegrationTest method testCloseCandidateRegistrationInQuickSuccession.

/**
 * Reproduces bug <a href="https://bugs.opendaylight.org/show_bug.cgi?id=4554">4554</a>.
 */
@Test
public void testCloseCandidateRegistrationInQuickSuccession() throws Exception {
    String name = "testCloseCandidateRegistrationInQuickSuccession";
    MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name).moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(leaderDatastoreContextBuilder).build();
    MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name).moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
    MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name).moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
    AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
    leaderDistributedDataStore.waitTillReady();
    follower1Node.configDataStore().waitTillReady();
    follower2Node.configDataStore().waitTillReady();
    final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
    final DOMEntityOwnershipService follower1EntityOwnershipService = newOwnershipService(follower1Node.configDataStore());
    final DOMEntityOwnershipService follower2EntityOwnershipService = newOwnershipService(follower2Node.configDataStore());
    leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
    leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
    follower1EntityOwnershipService.registerListener(ENTITY_TYPE1, follower1MockListener);
    follower2EntityOwnershipService.registerListener(ENTITY_TYPE1, follower2MockListener);
    final DOMEntityOwnershipCandidateRegistration candidate1 = leaderEntityOwnershipService.registerCandidate(ENTITY1);
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, true, true));
    final DOMEntityOwnershipCandidateRegistration candidate2 = follower1EntityOwnershipService.registerCandidate(ENTITY1);
    verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
    final DOMEntityOwnershipCandidateRegistration candidate3 = follower2EntityOwnershipService.registerCandidate(ENTITY1);
    verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
    Mockito.reset(leaderMockListener, follower1MockListener, follower2MockListener);
    ArgumentCaptor<DOMEntityOwnershipChange> leaderChangeCaptor = ArgumentCaptor.forClass(DOMEntityOwnershipChange.class);
    ArgumentCaptor<DOMEntityOwnershipChange> follower1ChangeCaptor = ArgumentCaptor.forClass(DOMEntityOwnershipChange.class);
    ArgumentCaptor<DOMEntityOwnershipChange> follower2ChangeCaptor = ArgumentCaptor.forClass(DOMEntityOwnershipChange.class);
    doNothing().when(leaderMockListener).ownershipChanged(leaderChangeCaptor.capture());
    doNothing().when(follower1MockListener).ownershipChanged(follower1ChangeCaptor.capture());
    doNothing().when(follower2MockListener).ownershipChanged(follower2ChangeCaptor.capture());
    candidate1.close();
    candidate2.close();
    candidate3.close();
    boolean passed = false;
    for (int i = 0; i < 100; i++) {
        Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
        final Optional<EntityOwnershipState> leaderState = leaderEntityOwnershipService.getOwnershipState(ENTITY1);
        final Optional<EntityOwnershipState> follower1State = follower1EntityOwnershipService.getOwnershipState(ENTITY1);
        final Optional<EntityOwnershipState> follower2State = follower2EntityOwnershipService.getOwnershipState(ENTITY1);
        final Optional<DOMEntityOwnershipChange> leaderChange = getValueSafely(leaderChangeCaptor);
        final Optional<DOMEntityOwnershipChange> follower1Change = getValueSafely(follower1ChangeCaptor);
        final Optional<DOMEntityOwnershipChange> follower2Change = getValueSafely(follower2ChangeCaptor);
        if (!leaderState.isPresent() || leaderState.get() == EntityOwnershipState.NO_OWNER && follower1State.isPresent() && follower1State.get() == EntityOwnershipState.NO_OWNER && follower2State.isPresent() && follower2State.get() == EntityOwnershipState.NO_OWNER && leaderChange.isPresent() && !leaderChange.get().getState().hasOwner() && follower1Change.isPresent() && !follower1Change.get().getState().hasOwner() && follower2Change.isPresent() && !follower2Change.get().getState().hasOwner()) {
            passed = true;
            break;
        }
    }
    assertTrue("No ownership change message was sent with hasOwner=false", passed);
}

Example 8 with AbstractDataStore

use of org.opendaylight.controller.cluster.datastore.AbstractDataStore in project controller by opendaylight.

the class DistributedEntityOwnershipIntegrationTest method testEntityOwnershipShardBootstrapping.

/**
 * Tests bootstrapping the entity-ownership shard when there's no shards initially configured for local
 * member. The entity-ownership shard is initially created as inactive (ie remains a follower), requiring
 * an AddShardReplica request to join it to an existing leader.
 */
@Test
public void testEntityOwnershipShardBootstrapping() throws Exception {
    String name = "testEntityOwnershipShardBootstrapping";
    String moduleShardsConfig = MODULE_SHARDS_MEMBER_1_CONFIG;
    MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name).moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(leaderDatastoreContextBuilder).build();
    AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
    final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
    leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
    MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name).moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
    AbstractDataStore follower1DistributedDataStore = follower1Node.configDataStore();
    follower1DistributedDataStore.waitTillReady();
    leaderNode.waitForMembersUp("member-2");
    follower1Node.waitForMembersUp("member-1");
    DOMEntityOwnershipService follower1EntityOwnershipService = newOwnershipService(follower1DistributedDataStore);
    leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
    // Register a candidate for follower1 - should get queued since follower1 has no leader
    final DOMEntityOwnershipCandidateRegistration candidateReg = follower1EntityOwnershipService.registerCandidate(ENTITY1);
    Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
    verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
    // Add replica in follower1
    AddShardReplica addReplica = new AddShardReplica(ENTITY_OWNERSHIP_SHARD_NAME);
    follower1DistributedDataStore.getActorContext().getShardManager().tell(addReplica, follower1Node.kit().getRef());
    Object reply = follower1Node.kit().expectMsgAnyClassOf(follower1Node.kit().duration("5 sec"), Success.class, Failure.class);
    if (reply instanceof Failure) {
        throw new AssertionError("AddShardReplica failed", ((Failure) reply).cause());
    }
    // The queued candidate registration should proceed
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
    reset(leaderMockListener);
    candidateReg.close();
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, false));
    reset(leaderMockListener);
    // Restart follower1 and verify the entity ownership shard is re-instated by registering.
    Cluster.get(leaderNode.kit().getSystem()).down(Cluster.get(follower1Node.kit().getSystem()).selfAddress());
    follower1Node.cleanup();
    follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name).moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
    follower1EntityOwnershipService = newOwnershipService(follower1Node.configDataStore());
    follower1EntityOwnershipService.registerCandidate(ENTITY1);
    verify(leaderMockListener, timeout(20000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
    verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME, raftState -> {
        assertNull("Custom RaftPolicy class name", raftState.getCustomRaftPolicyClassName());
        assertEquals("Peer count", 1, raftState.getPeerAddresses().keySet().size());
        assertThat("Peer Id", Iterables.<String>getLast(raftState.getPeerAddresses().keySet()), org.hamcrest.CoreMatchers.containsString("member-1"));
    });
}

Example 9 with AbstractDataStore

use of org.opendaylight.controller.cluster.datastore.AbstractDataStore in project controller by opendaylight.

the class DistributedEntityOwnershipIntegrationTest method testFunctionalityWithThreeNodes.

@Test
public void testFunctionalityWithThreeNodes() throws Exception {
    String name = "testFunctionalityWithThreeNodes";
    MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name).moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(leaderDatastoreContextBuilder).build();
    MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name).moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
    MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name).moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
    AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
    leaderDistributedDataStore.waitTillReady();
    follower1Node.configDataStore().waitTillReady();
    follower2Node.configDataStore().waitTillReady();
    final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
    final DOMEntityOwnershipService follower1EntityOwnershipService = newOwnershipService(follower1Node.configDataStore());
    final DOMEntityOwnershipService follower2EntityOwnershipService = newOwnershipService(follower2Node.configDataStore());
    leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
    leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
    leaderEntityOwnershipService.registerListener(ENTITY_TYPE2, leaderMockListener2);
    follower1EntityOwnershipService.registerListener(ENTITY_TYPE1, follower1MockListener);
    // Register leader candidate for entity1 and verify it becomes owner
    leaderEntityOwnershipService.registerCandidate(ENTITY1);
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, true, true));
    verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
    reset(leaderMockListener, follower1MockListener);
    verifyGetOwnershipState(leaderEntityOwnershipService, ENTITY1, EntityOwnershipState.IS_OWNER);
    verifyGetOwnershipState(follower1EntityOwnershipService, ENTITY1, EntityOwnershipState.OWNED_BY_OTHER);
    // Register leader candidate for entity1_2 (same id, different type) and verify it becomes owner
    leaderEntityOwnershipService.registerCandidate(ENTITY1_2);
    verify(leaderMockListener2, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1_2, false, true, true));
    Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
    verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1_2));
    reset(leaderMockListener2);
    // Register follower1 candidate for entity1 and verify it gets added but doesn't become owner
    follower1EntityOwnershipService.registerCandidate(ENTITY1);
    verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-1", "member-2");
    verifyOwner(leaderDistributedDataStore, ENTITY1, "member-1");
    verifyOwner(follower2Node.configDataStore(), ENTITY1, "member-1");
    Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
    verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
    verify(follower1MockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
    // Register follower1 candidate for entity2 and verify it becomes owner
    final DOMEntityOwnershipCandidateRegistration follower1Entity2Reg = follower1EntityOwnershipService.registerCandidate(ENTITY2);
    verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, true));
    verifyOwner(follower2Node.configDataStore(), ENTITY2, "member-2");
    reset(leaderMockListener, follower1MockListener);
    // Register follower2 candidate for entity2 and verify it gets added but doesn't become owner
    follower2EntityOwnershipService.registerListener(ENTITY_TYPE1, follower2MockListener);
    verify(follower2MockListener, timeout(5000).times(2)).ownershipChanged(or(ownershipChange(ENTITY1, false, false, true), ownershipChange(ENTITY2, false, false, true)));
    follower2EntityOwnershipService.registerCandidate(ENTITY2);
    verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-2", "member-3");
    verifyOwner(leaderDistributedDataStore, ENTITY2, "member-2");
    // Unregister follower1 candidate for entity2 and verify follower2 becomes owner
    follower1Entity2Reg.close();
    verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-3");
    verifyOwner(leaderDistributedDataStore, ENTITY2, "member-3");
    verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, true, false, true));
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, true));
    // Depending on timing, follower2MockListener could get ownershipChanged with "false, false, true" if
    // if the original ownership change with "member-2 is replicated to follower2 after the listener is
    // registered.
    Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
    verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
    // Register follower1 candidate for entity3 and verify it becomes owner
    follower1EntityOwnershipService.registerCandidate(ENTITY3);
    verifyOwner(leaderDistributedDataStore, ENTITY3, "member-2");
    verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, true, true));
    verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, false, true));
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, false, true));
    // Register follower2 candidate for entity4 and verify it becomes owner
    follower2EntityOwnershipService.registerCandidate(ENTITY4);
    verifyOwner(leaderDistributedDataStore, ENTITY4, "member-3");
    verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, true, true));
    verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
    reset(follower1MockListener, follower2MockListener);
    // Register follower1 candidate for entity4 and verify it gets added but doesn't become owner
    follower1EntityOwnershipService.registerCandidate(ENTITY4);
    verifyCandidates(leaderDistributedDataStore, ENTITY4, "member-3", "member-2");
    verifyOwner(leaderDistributedDataStore, ENTITY4, "member-3");
    // Shutdown follower2 and verify it's owned entities (entity 4) get re-assigned
    reset(leaderMockListener, follower1MockListener);
    follower2Node.cleanup();
    verify(follower1MockListener, timeout(15000)).ownershipChanged(ownershipChange(ENTITY4, false, true, true));
    verify(leaderMockListener, timeout(15000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
    // Register leader candidate for entity2 and verify it becomes owner
    DOMEntityOwnershipCandidateRegistration leaderEntity2Reg = leaderEntityOwnershipService.registerCandidate(ENTITY2);
    verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
    // Unregister leader candidate for entity2 and verify the owner is cleared
    leaderEntity2Reg.close();
    verifyOwner(leaderDistributedDataStore, ENTITY2, "");
    verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, true, false, false));
    verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, false));
}