Examples with ServerConfigurationPayload org.opendaylight.controller.cluster.raft.persisted.ServerConfigurationPayload used on opensource projects

Example 16 with ServerConfigurationPayload

use of org.opendaylight.controller.cluster.raft.persisted.ServerConfigurationPayload in project controller by opendaylight.

the class RaftActorServerConfigurationSupportTest method testChangeToVotingWithNoLeader.

@Test
public void testChangeToVotingWithNoLeader() {
    LOG.info("testChangeToVotingWithNoLeader starting");
    DefaultConfigParamsImpl configParams = new DefaultConfigParamsImpl();
    configParams.setHeartBeatInterval(new FiniteDuration(100, TimeUnit.MILLISECONDS));
    configParams.setElectionTimeoutFactor(5);
    final String node1ID = "node1";
    final String node2ID = "node2";
    // Set up a persisted ServerConfigurationPayload. Initially node1 and node2 will come up as non-voting.
    // via the server config. The server config will also contain 2 voting peers that are down (ie no
    // actors created).
    ServerConfigurationPayload persistedServerConfig = new ServerConfigurationPayload(Arrays.asList(new ServerInfo(node1ID, false), new ServerInfo(node2ID, false), new ServerInfo("downNode1", true), new ServerInfo("downNode2", true)));
    SimpleReplicatedLogEntry persistedServerConfigEntry = new SimpleReplicatedLogEntry(0, 1, persistedServerConfig);
    InMemoryJournal.addEntry(node1ID, 1, new UpdateElectionTerm(1, "downNode1"));
    InMemoryJournal.addEntry(node1ID, 2, persistedServerConfigEntry);
    InMemoryJournal.addEntry(node1ID, 3, new ApplyJournalEntries(0));
    InMemoryJournal.addEntry(node2ID, 1, new UpdateElectionTerm(1, "downNode2"));
    InMemoryJournal.addEntry(node2ID, 2, persistedServerConfigEntry);
    InMemoryJournal.addEntry(node2ID, 3, new ApplyJournalEntries(0));
    ActorRef node1Collector = actorFactory.createActor(MessageCollectorActor.props(), actorFactory.generateActorId("collector"));
    TestActorRef<CollectingMockRaftActor> node1RaftActorRef = actorFactory.createTestActor(CollectingMockRaftActor.props(node1ID, ImmutableMap.<String, String>of(), configParams, PERSISTENT, node1Collector).withDispatcher(Dispatchers.DefaultDispatcherId()), node1ID);
    CollectingMockRaftActor node1RaftActor = node1RaftActorRef.underlyingActor();
    ActorRef node2Collector = actorFactory.createActor(MessageCollectorActor.props(), actorFactory.generateActorId("collector"));
    TestActorRef<CollectingMockRaftActor> node2RaftActorRef = actorFactory.createTestActor(CollectingMockRaftActor.props(node2ID, ImmutableMap.<String, String>of(), configParams, PERSISTENT, node2Collector).withDispatcher(Dispatchers.DefaultDispatcherId()), node2ID);
    CollectingMockRaftActor node2RaftActor = node2RaftActorRef.underlyingActor();
    node1RaftActor.waitForInitializeBehaviorComplete();
    node2RaftActor.waitForInitializeBehaviorComplete();
    // Verify the intended server config was loaded and applied.
    verifyServerConfigurationPayloadEntry(node1RaftActor.getRaftActorContext().getReplicatedLog(), nonVotingServer(node1ID), nonVotingServer(node2ID), votingServer("downNode1"), votingServer("downNode2"));
    assertEquals("isVotingMember", false, node1RaftActor.getRaftActorContext().isVotingMember());
    assertEquals("getRaftState", RaftState.Follower, node1RaftActor.getRaftState());
    assertEquals("getLeaderId", null, node1RaftActor.getLeaderId());
    verifyServerConfigurationPayloadEntry(node2RaftActor.getRaftActorContext().getReplicatedLog(), nonVotingServer(node1ID), nonVotingServer(node2ID), votingServer("downNode1"), votingServer("downNode2"));
    assertEquals("isVotingMember", false, node2RaftActor.getRaftActorContext().isVotingMember());
    // For the test, we send a ChangeServersVotingStatus message to node1 to flip the voting states for
    // each server, ie node1 and node2 to voting and the 2 down nodes to non-voting. This should cause
    // node1 to try to elect itself as leader in order to apply the new server config. Since the 2
    // down nodes are switched to non-voting, node1 should only need a vote from node2.
    // First send the message such that node1 has no peer address for node2 - should fail.
    ChangeServersVotingStatus changeServers = new ChangeServersVotingStatus(ImmutableMap.of(node1ID, true, node2ID, true, "downNode1", false, "downNode2", false));
    node1RaftActorRef.tell(changeServers, testKit.getRef());
    ServerChangeReply reply = testKit.expectMsgClass(testKit.duration("5 seconds"), ServerChangeReply.class);
    assertEquals("getStatus", ServerChangeStatus.NO_LEADER, reply.getStatus());
    assertEquals("getRaftState", RaftState.Follower, node1RaftActor.getRaftState());
    // Send an AppendEntries so node1 has a leaderId
    long term = node1RaftActor.getRaftActorContext().getTermInformation().getCurrentTerm();
    node1RaftActorRef.tell(new AppendEntries(term, "downNode1", -1L, -1L, Collections.<ReplicatedLogEntry>emptyList(), 0, -1, (short) 1), ActorRef.noSender());
    // Wait for the ElectionTimeout to clear the leaderId. The leaderId must be null so on the next
    // ChangeServersVotingStatus message, it will try to elect a leader.
    AbstractRaftActorIntegrationTest.verifyRaftState(node1RaftActorRef, rs -> assertEquals("getLeader", null, rs.getLeader()));
    // Update node2's peer address and send the message again
    node1RaftActor.setPeerAddress(node2ID, node2RaftActorRef.path().toString());
    node1RaftActorRef.tell(changeServers, testKit.getRef());
    reply = testKit.expectMsgClass(testKit.duration("5 seconds"), ServerChangeReply.class);
    assertEquals("getStatus", ServerChangeStatus.OK, reply.getStatus());
    ApplyJournalEntries apply = MessageCollectorActor.expectFirstMatching(node1Collector, ApplyJournalEntries.class);
    assertEquals("getToIndex", 1, apply.getToIndex());
    verifyServerConfigurationPayloadEntry(node1RaftActor.getRaftActorContext().getReplicatedLog(), votingServer(node1ID), votingServer(node2ID), nonVotingServer("downNode1"), nonVotingServer("downNode2"));
    assertEquals("isVotingMember", true, node1RaftActor.getRaftActorContext().isVotingMember());
    assertEquals("getRaftState", RaftState.Leader, node1RaftActor.getRaftState());
    apply = MessageCollectorActor.expectFirstMatching(node2Collector, ApplyJournalEntries.class);
    assertEquals("getToIndex", 1, apply.getToIndex());
    verifyServerConfigurationPayloadEntry(node2RaftActor.getRaftActorContext().getReplicatedLog(), votingServer(node1ID), votingServer(node2ID), nonVotingServer("downNode1"), nonVotingServer("downNode2"));
    assertEquals("isVotingMember", true, node2RaftActor.getRaftActorContext().isVotingMember());
    assertEquals("getRaftState", RaftState.Follower, node2RaftActor.getRaftState());
    LOG.info("testChangeToVotingWithNoLeader ending");
}

Example 17 with ServerConfigurationPayload

use of org.opendaylight.controller.cluster.raft.persisted.ServerConfigurationPayload in project controller by opendaylight.

the class ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest method verifyInstallSnapshotToLaggingFollower.

/**
 * Resume the lagging follower 2 and verify it receives an install snapshot from the leader.
 */
private void verifyInstallSnapshotToLaggingFollower(long lastAppliedIndex, @Nullable ServerConfigurationPayload expServerConfig) throws Exception {
    testLog.info("verifyInstallSnapshotToLaggingFollower starting");
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    // Now stop dropping AppendEntries in follower 2.
    follower2Actor.underlyingActor().stopDropMessages(AppendEntries.class);
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, SaveSnapshotSuccess.class);
    // Verify the leader's persisted snapshot. The previous snapshot (currently) won't be deleted from
    // the snapshot store because the second snapshot was initiated by the follower install snapshot and
    // not because the batch count was reached so the persisted journal sequence number wasn't advanced
    // far enough to cause the previous snapshot to be deleted. This is because
    // RaftActor#trimPersistentData subtracts the snapshotBatchCount from the snapshot's sequence number.
    // This is OK - the next snapshot should delete it. In production, even if the system restarted
    // before another snapshot, they would both get applied which wouldn't hurt anything.
    List<Snapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(leaderId, Snapshot.class);
    Assert.assertTrue("Expected at least 1 persisted snapshots", persistedSnapshots.size() > 0);
    Snapshot persistedSnapshot = persistedSnapshots.get(persistedSnapshots.size() - 1);
    verifySnapshot("Persisted", persistedSnapshot, currentTerm, lastAppliedIndex, currentTerm, lastAppliedIndex);
    List<ReplicatedLogEntry> unAppliedEntry = persistedSnapshot.getUnAppliedEntries();
    assertEquals("Persisted Snapshot getUnAppliedEntries size", 0, unAppliedEntry.size());
    int snapshotSize = SerializationUtils.serialize(persistedSnapshot.getState()).length;
    final int expTotalChunks = snapshotSize / SNAPSHOT_CHUNK_SIZE + (snapshotSize % SNAPSHOT_CHUNK_SIZE > 0 ? 1 : 0);
    InstallSnapshot installSnapshot = MessageCollectorActor.expectFirstMatching(follower2CollectorActor, InstallSnapshot.class);
    assertEquals("InstallSnapshot getTerm", currentTerm, installSnapshot.getTerm());
    assertEquals("InstallSnapshot getLeaderId", leaderId, installSnapshot.getLeaderId());
    assertEquals("InstallSnapshot getChunkIndex", 1, installSnapshot.getChunkIndex());
    assertEquals("InstallSnapshot getTotalChunks", expTotalChunks, installSnapshot.getTotalChunks());
    assertEquals("InstallSnapshot getLastIncludedTerm", currentTerm, installSnapshot.getLastIncludedTerm());
    assertEquals("InstallSnapshot getLastIncludedIndex", lastAppliedIndex, installSnapshot.getLastIncludedIndex());
    // assertArrayEquals("InstallSnapshot getData", snapshot, installSnapshot.getData().toByteArray());
    List<InstallSnapshotReply> installSnapshotReplies = MessageCollectorActor.expectMatching(leaderCollectorActor, InstallSnapshotReply.class, expTotalChunks);
    int index = 1;
    for (InstallSnapshotReply installSnapshotReply : installSnapshotReplies) {
        assertEquals("InstallSnapshotReply getTerm", currentTerm, installSnapshotReply.getTerm());
        assertEquals("InstallSnapshotReply getChunkIndex", index++, installSnapshotReply.getChunkIndex());
        assertEquals("InstallSnapshotReply getFollowerId", follower2Id, installSnapshotReply.getFollowerId());
        assertEquals("InstallSnapshotReply isSuccess", true, installSnapshotReply.isSuccess());
    }
    // Verify follower 2 applies the snapshot.
    ApplySnapshot applySnapshot = MessageCollectorActor.expectFirstMatching(follower2CollectorActor, ApplySnapshot.class);
    verifySnapshot("Follower 2", applySnapshot.getSnapshot(), currentTerm, lastAppliedIndex, currentTerm, lastAppliedIndex);
    assertEquals("Persisted Snapshot getUnAppliedEntries size", 0, applySnapshot.getSnapshot().getUnAppliedEntries().size());
    // Wait for the snapshot to complete.
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, SaveSnapshotSuccess.class);
    // Ensure there's at least 1 more heartbeat.
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, AppendEntriesReply.class);
    // The leader should now have performed fake snapshots to advance the snapshot index and to trim
    // the log. In addition replicatedToAllIndex should've advanced.
    verifyLeadersTrimmedLog(lastAppliedIndex);
    if (expServerConfig != null) {
        Set<ServerInfo> expServerInfo = new HashSet<>(expServerConfig.getServerConfig());
        assertEquals("Leader snapshot server config", expServerInfo, new HashSet<>(persistedSnapshot.getServerConfiguration().getServerConfig()));
        assertEquals("Follower 2 snapshot server config", expServerInfo, new HashSet<>(applySnapshot.getSnapshot().getServerConfiguration().getServerConfig()));
        ServerConfigurationPayload follower2ServerConfig = follower2Context.getPeerServerInfo(true);
        assertNotNull("Follower 2 server config is null", follower2ServerConfig);
        assertEquals("Follower 2 server config", expServerInfo, new HashSet<>(follower2ServerConfig.getServerConfig()));
    }
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.clearMessages(follower1CollectorActor);
    MessageCollectorActor.clearMessages(follower2CollectorActor);
    testLog.info("verifyInstallSnapshotToLaggingFollower complete");
}

Example 18 with ServerConfigurationPayload

use of org.opendaylight.controller.cluster.raft.persisted.ServerConfigurationPayload in project controller by opendaylight.

the class FollowerTest method testFollowerSchedulesElectionIfNonVoting.

@Test
public void testFollowerSchedulesElectionIfNonVoting() {
    MockRaftActorContext context = createActorContext();
    context.updatePeerIds(new ServerConfigurationPayload(Arrays.asList(new ServerInfo(context.getId(), false))));
    ((DefaultConfigParamsImpl) context.getConfigParams()).setHeartBeatInterval(FiniteDuration.apply(100, TimeUnit.MILLISECONDS));
    ((DefaultConfigParamsImpl) context.getConfigParams()).setElectionTimeoutFactor(1);
    follower = new Follower(context, "leader", (short) 1);
    ElectionTimeout electionTimeout = MessageCollectorActor.expectFirstMatching(followerActor, ElectionTimeout.class);
    RaftActorBehavior newBehavior = follower.handleMessage(ActorRef.noSender(), electionTimeout);
    assertSame("handleMessage result", follower, newBehavior);
    assertNull("Expected null leaderId", follower.getLeaderId());
}

Example 19 with ServerConfigurationPayload

use of org.opendaylight.controller.cluster.raft.persisted.ServerConfigurationPayload in project controller by opendaylight.

the class Follower method processNewEntries.

private boolean processNewEntries(final AppendEntries appendEntries, final ActorRef sender) {
    int numLogEntries = appendEntries.getEntries().size();
    if (numLogEntries == 0) {
        return true;
    }
    log.debug("{}: Number of entries to be appended = {}", logName(), numLogEntries);
    long lastIndex = lastIndex();
    int addEntriesFrom = 0;
    // term), delete the existing entry and all that follow it (§5.3)
    if (context.getReplicatedLog().size() > 0) {
        // Find the entry up until the one that is not in the follower's log
        for (int i = 0; i < numLogEntries; i++, addEntriesFrom++) {
            ReplicatedLogEntry matchEntry = appendEntries.getEntries().get(i);
            if (!isLogEntryPresent(matchEntry.getIndex())) {
                // newEntry not found in the log
                break;
            }
            long existingEntryTerm = getLogEntryTerm(matchEntry.getIndex());
            log.debug("{}: matchEntry {} is present: existingEntryTerm: {}", logName(), matchEntry, existingEntryTerm);
            // what the term was so we'll assume it matches.
            if (existingEntryTerm == -1 || existingEntryTerm == matchEntry.getTerm()) {
                continue;
            }
            if (!context.getRaftPolicy().applyModificationToStateBeforeConsensus()) {
                log.info("{}: Removing entries from log starting at {}, commitIndex: {}, lastApplied: {}", logName(), matchEntry.getIndex(), context.getCommitIndex(), context.getLastApplied());
                // been applied to the state yet.
                if (matchEntry.getIndex() <= context.getLastApplied() || !context.getReplicatedLog().removeFromAndPersist(matchEntry.getIndex())) {
                    // Could not remove the entries - this means the matchEntry index must be in the
                    // snapshot and not the log. In this case the prior entries are part of the state
                    // so we must send back a reply to force a snapshot to completely re-sync the
                    // follower's log and state.
                    log.info("{}: Could not remove entries - sending reply to force snapshot", logName());
                    sender.tell(new AppendEntriesReply(context.getId(), currentTerm(), false, lastIndex, lastTerm(), context.getPayloadVersion(), true), actor());
                    return false;
                }
                break;
            } else {
                sender.tell(new AppendEntriesReply(context.getId(), currentTerm(), false, lastIndex, lastTerm(), context.getPayloadVersion(), true), actor());
                return false;
            }
        }
    }
    lastIndex = lastIndex();
    log.debug("{}: After cleanup, lastIndex: {}, entries to be added from: {}", logName(), lastIndex, addEntriesFrom);
    // When persistence successfully completes for each new log entry appended, we need to determine if we
    // should capture a snapshot to compact the persisted log. shouldCaptureSnapshot tracks whether or not
    // one of the log entries has exceeded the log size threshold whereby a snapshot should be taken. However
    // we don't initiate the snapshot at that log entry but rather after the last log entry has been persisted.
    // This is done because subsequent log entries after the one that tripped the threshold may have been
    // applied to the state already, as the persistence callback occurs async, and we want those entries
    // purged from the persisted log as well.
    final AtomicBoolean shouldCaptureSnapshot = new AtomicBoolean(false);
    final Procedure<ReplicatedLogEntry> appendAndPersistCallback = logEntry -> {
        final List<ReplicatedLogEntry> entries = appendEntries.getEntries();
        final ReplicatedLogEntry lastEntryToAppend = entries.get(entries.size() - 1);
        if (shouldCaptureSnapshot.get() && logEntry == lastEntryToAppend) {
            context.getSnapshotManager().capture(context.getReplicatedLog().last(), getReplicatedToAllIndex());
        }
    };
    // Append any new entries not already in the log
    for (int i = addEntriesFrom; i < numLogEntries; i++) {
        ReplicatedLogEntry entry = appendEntries.getEntries().get(i);
        log.debug("{}: Append entry to log {}", logName(), entry.getData());
        context.getReplicatedLog().appendAndPersist(entry, appendAndPersistCallback, false);
        shouldCaptureSnapshot.compareAndSet(false, context.getReplicatedLog().shouldCaptureSnapshot(entry.getIndex()));
        if (entry.getData() instanceof ServerConfigurationPayload) {
            context.updatePeerIds((ServerConfigurationPayload) entry.getData());
        }
    }
    log.debug("{}: Log size is now {}", logName(), context.getReplicatedLog().size());
    return true;
}

Example 20 with ServerConfigurationPayload

use of org.opendaylight.controller.cluster.raft.persisted.ServerConfigurationPayload in project controller by opendaylight.

the class ClusterAdminRpcServiceTest method testAddShardReplica.

@Test
public void testAddShardReplica() throws Exception {
    String name = "testAddShardReplica";
    String moduleShardsConfig = "module-shards-cars-member-1.conf";
    MemberNode leaderNode1 = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name).moduleShardsConfig(moduleShardsConfig).waitForShardLeader("cars").build();
    MemberNode newReplicaNode2 = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name).moduleShardsConfig(moduleShardsConfig).build();
    leaderNode1.waitForMembersUp("member-2");
    doAddShardReplica(newReplicaNode2, "cars", "member-1");
    MemberNode newReplicaNode3 = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name).moduleShardsConfig(moduleShardsConfig).build();
    leaderNode1.waitForMembersUp("member-3");
    newReplicaNode2.waitForMembersUp("member-3");
    doAddShardReplica(newReplicaNode3, "cars", "member-1", "member-2");
    verifyRaftPeersPresent(newReplicaNode2.configDataStore(), "cars", "member-1", "member-3");
    verifyRaftPeersPresent(newReplicaNode2.operDataStore(), "cars", "member-1", "member-3");
    // Write data to member-2's config datastore and read/verify via member-3
    final NormalizedNode<?, ?> configCarsNode = writeCarsNodeAndVerify(newReplicaNode2.configDataStore(), newReplicaNode3.configDataStore());
    // Write data to member-3's oper datastore and read/verify via member-2
    writeCarsNodeAndVerify(newReplicaNode3.operDataStore(), newReplicaNode2.operDataStore());
    // Verify all data has been replicated. We expect 4 log entries and thus last applied index of 3 -
    // 2 ServerConfigurationPayload entries,  the transaction payload entry plus a purge payload.
    RaftStateVerifier verifier = raftState -> {
        assertEquals("Commit index", 3, raftState.getCommitIndex());
        assertEquals("Last applied index", 3, raftState.getLastApplied());
    };
    verifyRaftState(leaderNode1.configDataStore(), "cars", verifier);
    verifyRaftState(leaderNode1.operDataStore(), "cars", verifier);
    verifyRaftState(newReplicaNode2.configDataStore(), "cars", verifier);
    verifyRaftState(newReplicaNode2.operDataStore(), "cars", verifier);
    verifyRaftState(newReplicaNode3.configDataStore(), "cars", verifier);
    verifyRaftState(newReplicaNode3.operDataStore(), "cars", verifier);
    // Restart member-3 and verify the cars config shard is re-instated.
    Cluster.get(leaderNode1.kit().getSystem()).down(Cluster.get(newReplicaNode3.kit().getSystem()).selfAddress());
    newReplicaNode3.cleanup();
    newReplicaNode3 = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name).moduleShardsConfig(moduleShardsConfig).createOperDatastore(false).build();
    verifyRaftState(newReplicaNode3.configDataStore(), "cars", verifier);
    readCarsNodeAndVerify(newReplicaNode3.configDataStore(), configCarsNode);
}