Examples with AppendEntries org.opendaylight.controller.cluster.raft.messages.AppendEntries used on opensource projects

Example 21 with AppendEntries

use of org.opendaylight.controller.cluster.raft.messages.AppendEntries in project controller by opendaylight.

the class LeaderTest method testHandleReplicateMessageWithHigherTermThanPreviousEntry.

@Test
public void testHandleReplicateMessageWithHigherTermThanPreviousEntry() throws Exception {
    logStart("testHandleReplicateMessageWithHigherTermThanPreviousEntry");
    MockRaftActorContext actorContext = createActorContextWithFollower();
    actorContext.setCommitIndex(-1);
    actorContext.setLastApplied(-1);
    // The raft context is initialized with a couple log entries. However the commitIndex
    // is -1, simulating that the leader previously didn't get consensus and thus the log entries weren't
    // committed and applied. Now it regains leadership with a higher term (2).
    long prevTerm = actorContext.getTermInformation().getCurrentTerm();
    long newTerm = prevTerm + 1;
    actorContext.getTermInformation().update(newTerm, "");
    leader = new Leader(actorContext);
    actorContext.setCurrentBehavior(leader);
    // Leader will send an immediate heartbeat - ignore it.
    MessageCollectorActor.expectFirstMatching(followerActor, AppendEntries.class);
    // The follower replies with the leader's current last index and term, simulating that it is
    // up to date with the leader.
    long lastIndex = actorContext.getReplicatedLog().lastIndex();
    leader.handleMessage(followerActor, new AppendEntriesReply(FOLLOWER_ID, newTerm, true, lastIndex, prevTerm, (short) 0));
    // The commit index should not get updated even though consensus was reached. This is b/c the
    // last entry's term does match the current term. As per §5.4.1, "Raft never commits log entries
    // from previous terms by counting replicas".
    assertEquals("Commit Index", -1, actorContext.getCommitIndex());
    followerActor.underlyingActor().clear();
    // Now replicate a new entry with the new term 2.
    long newIndex = lastIndex + 1;
    sendReplicate(actorContext, newTerm, newIndex);
    AppendEntries appendEntries = MessageCollectorActor.expectFirstMatching(followerActor, AppendEntries.class);
    assertEquals("getPrevLogIndex", lastIndex, appendEntries.getPrevLogIndex());
    assertEquals("getPrevLogTerm", prevTerm, appendEntries.getPrevLogTerm());
    assertEquals("Entries size", 1, appendEntries.getEntries().size());
    assertEquals("Entry getIndex", newIndex, appendEntries.getEntries().get(0).getIndex());
    assertEquals("Entry getTerm", newTerm, appendEntries.getEntries().get(0).getTerm());
    assertEquals("Entry payload", "foo", appendEntries.getEntries().get(0).getData().toString());
    // The follower replies with success. The leader should now update the commit index to the new index
    // as per §5.4.1 "once an entry from the current term is committed by counting replicas, then all
    // prior entries are committed indirectly".
    leader.handleMessage(followerActor, new AppendEntriesReply(FOLLOWER_ID, newTerm, true, newIndex, newTerm, (short) 0));
    assertEquals("Commit Index", newIndex, actorContext.getCommitIndex());
}

Example 22 with AppendEntries

use of org.opendaylight.controller.cluster.raft.messages.AppendEntries in project controller by opendaylight.

the class LeaderTest method testReplicationConsensusWithNonVotingFollower.

@Test
public void testReplicationConsensusWithNonVotingFollower() {
    logStart("testReplicationConsensusWithNonVotingFollower");
    MockRaftActorContext leaderActorContext = createActorContextWithFollower();
    ((DefaultConfigParamsImpl) leaderActorContext.getConfigParams()).setHeartBeatInterval(new FiniteDuration(1000, TimeUnit.SECONDS));
    leaderActorContext.setReplicatedLog(new MockRaftActorContext.MockReplicatedLogBuilder().build());
    leaderActorContext.setCommitIndex(-1);
    leaderActorContext.setLastApplied(-1);
    String nonVotingFollowerId = "nonvoting-follower";
    ActorRef nonVotingFollowerActor = actorFactory.createActor(MessageCollectorActor.props(), actorFactory.generateActorId(nonVotingFollowerId));
    leaderActorContext.addToPeers(nonVotingFollowerId, nonVotingFollowerActor.path().toString(), VotingState.NON_VOTING);
    leader = new Leader(leaderActorContext);
    leaderActorContext.setCurrentBehavior(leader);
    // Ignore initial heartbeats
    MessageCollectorActor.expectFirstMatching(followerActor, AppendEntries.class);
    MessageCollectorActor.expectFirstMatching(nonVotingFollowerActor, AppendEntries.class);
    MessageCollectorActor.clearMessages(followerActor);
    MessageCollectorActor.clearMessages(nonVotingFollowerActor);
    MessageCollectorActor.clearMessages(leaderActor);
    // Send a Replicate message and wait for AppendEntries.
    sendReplicate(leaderActorContext, 0);
    MessageCollectorActor.expectFirstMatching(followerActor, AppendEntries.class);
    MessageCollectorActor.expectFirstMatching(nonVotingFollowerActor, AppendEntries.class);
    // Send reply only from the voting follower and verify consensus via ApplyState.
    leader.handleMessage(leaderActor, new AppendEntriesReply(FOLLOWER_ID, 1, true, 0, 1, (short) 0));
    MessageCollectorActor.expectFirstMatching(leaderActor, ApplyState.class);
    leader.handleMessage(leaderActor, new AppendEntriesReply(nonVotingFollowerId, 1, true, 0, 1, (short) 0));
    MessageCollectorActor.clearMessages(followerActor);
    MessageCollectorActor.clearMessages(nonVotingFollowerActor);
    MessageCollectorActor.clearMessages(leaderActor);
    // Send another Replicate message
    sendReplicate(leaderActorContext, 1);
    MessageCollectorActor.expectFirstMatching(followerActor, AppendEntries.class);
    AppendEntries appendEntries = MessageCollectorActor.expectFirstMatching(nonVotingFollowerActor, AppendEntries.class);
    assertEquals("Log entries size", 1, appendEntries.getEntries().size());
    assertEquals("Log entry index", 1, appendEntries.getEntries().get(0).getIndex());
    // Send reply only from the non-voting follower and verify no consensus via no ApplyState.
    leader.handleMessage(leaderActor, new AppendEntriesReply(nonVotingFollowerId, 1, true, 1, 1, (short) 0));
    MessageCollectorActor.assertNoneMatching(leaderActor, ApplyState.class, 500);
    // Send reply from the voting follower and verify consensus.
    leader.handleMessage(leaderActor, new AppendEntriesReply(FOLLOWER_ID, 1, true, 1, 1, (short) 0));
    MessageCollectorActor.expectFirstMatching(leaderActor, ApplyState.class);
}

Example 23 with AppendEntries

use of org.opendaylight.controller.cluster.raft.messages.AppendEntries in project controller by opendaylight.

the class PreLeaderScenarioTest method testUnComittedEntryOnLeaderChange.

@Test
public void testUnComittedEntryOnLeaderChange() throws Exception {
    testLog.info("testUnComittedEntryOnLeaderChange starting");
    createRaftActors();
    // Drop AppendEntriesReply to the leader so it doesn't commit the payload entry.
    leaderActor.underlyingActor().startDropMessages(AppendEntriesReply.class);
    follower2Actor.underlyingActor().startDropMessages(AppendEntries.class);
    // Send a payload and verify AppendEntries is received in follower1.
    MockPayload payload0 = sendPayloadData(leaderActor, "zero");
    AppendEntries appendEntries = expectFirstMatching(follower1CollectorActor, AppendEntries.class);
    assertEquals("AppendEntries - # entries", 1, appendEntries.getEntries().size());
    verifyReplicatedLogEntry(appendEntries.getEntries().get(0), currentTerm, 0, payload0);
    // Kill the leader actor.
    killActor(leaderActor);
    // At this point, the payload entry is in follower1's log but is uncommitted. follower2 has not
    // received the payload entry yet.
    assertEquals("Follower 1 journal log size", 1, follower1Context.getReplicatedLog().size());
    assertEquals("Follower 1 journal last index", 0, follower1Context.getReplicatedLog().lastIndex());
    assertEquals("Follower 1 commit index", -1, follower1Context.getCommitIndex());
    assertEquals("Follower 1 last applied index", -1, follower1Context.getLastApplied());
    assertEquals("Follower 2 journal log size", 0, follower2Context.getReplicatedLog().size());
    follower2Actor.underlyingActor().stopDropMessages(AppendEntries.class);
    clearMessages(follower1NotifierActor);
    // Force follower1 to start an election. It should win since it's journal is more up-to-date than
    // follower2's journal.
    follower1Actor.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
    // Verify the expected raft state changes. It should go to PreLeader since it has an uncommitted entry.
    List<RoleChanged> roleChange = expectMatching(follower1NotifierActor, RoleChanged.class, 3);
    assertEquals("Role change 1", RaftState.Candidate.name(), roleChange.get(0).getNewRole());
    assertEquals("Role change 2", RaftState.PreLeader.name(), roleChange.get(1).getNewRole());
    assertEquals("Role change 3", RaftState.Leader.name(), roleChange.get(2).getNewRole());
    final long previousTerm = currentTerm;
    currentTerm = follower1Context.getTermInformation().getCurrentTerm();
    // Since it went to Leader, it should've appended and successfully replicated a NoopPaylod with the
    // new term to follower2 and committed both entries, including the first payload from the previous term.
    assertEquals("Follower 1 journal log size", 2, follower1Context.getReplicatedLog().size());
    assertEquals("Follower 1 journal last index", 1, follower1Context.getReplicatedLog().lastIndex());
    assertEquals("Follower 1 commit index", 1, follower1Context.getCommitIndex());
    verifyReplicatedLogEntry(follower1Context.getReplicatedLog().get(0), previousTerm, 0, payload0);
    verifyReplicatedLogEntry(follower1Context.getReplicatedLog().get(1), currentTerm, 1, NoopPayload.INSTANCE);
    // Both entries should be applied to the state.
    expectMatching(follower1CollectorActor, ApplyState.class, 2);
    expectMatching(follower2CollectorActor, ApplyState.class, 2);
    assertEquals("Follower 1 last applied index", 1, follower1Context.getLastApplied());
    // Verify follower2's journal matches follower1's.
    assertEquals("Follower 2 journal log size", 2, follower2Context.getReplicatedLog().size());
    assertEquals("Follower 2 journal last index", 1, follower2Context.getReplicatedLog().lastIndex());
    assertEquals("Follower 2 commit index", 1, follower2Context.getCommitIndex());
    assertEquals("Follower 2 last applied index", 1, follower2Context.getLastApplied());
    verifyReplicatedLogEntry(follower2Context.getReplicatedLog().get(0), previousTerm, 0, payload0);
    verifyReplicatedLogEntry(follower2Context.getReplicatedLog().get(1), currentTerm, 1, NoopPayload.INSTANCE);
    // Reinstate follower1.
    killActor(follower1Actor);
    follower1Actor = newTestRaftActor(follower1Id, TestRaftActor.newBuilder().peerAddresses(ImmutableMap.of(leaderId, testActorPath(leaderId), follower2Id, testActorPath(follower2Id))).config(followerConfigParams));
    follower1Actor.underlyingActor().waitForRecoveryComplete();
    follower1Context = follower1Actor.underlyingActor().getRaftActorContext();
    // Verify follower1's journal was persisted and recovered correctly.
    assertEquals("Follower 1 journal log size", 2, follower1Context.getReplicatedLog().size());
    assertEquals("Follower 1 journal last index", 1, follower1Context.getReplicatedLog().lastIndex());
    assertEquals("Follower 1 commit index", 1, follower1Context.getCommitIndex());
    assertEquals("Follower 1 last applied index", 1, follower1Context.getLastApplied());
    verifyReplicatedLogEntry(follower1Context.getReplicatedLog().get(0), previousTerm, 0, payload0);
    verifyReplicatedLogEntry(follower1Context.getReplicatedLog().get(1), currentTerm, 1, NoopPayload.INSTANCE);
    testLog.info("testUnComittedEntryOnLeaderChange ending");
}

Example 24 with AppendEntries

use of org.opendaylight.controller.cluster.raft.messages.AppendEntries in project controller by opendaylight.

the class RaftActorServerConfigurationSupportTest method testRemoveServerForwardToLeader.

@Test
public void testRemoveServerForwardToLeader() {
    LOG.info("testRemoveServerForwardToLeader starting");
    DefaultConfigParamsImpl configParams = new DefaultConfigParamsImpl();
    configParams.setHeartBeatInterval(new FiniteDuration(1, TimeUnit.DAYS));
    ActorRef leaderActor = actorFactory.createTestActor(MessageCollectorActor.props(), actorFactory.generateActorId(LEADER_ID));
    TestActorRef<MockRaftActor> followerRaftActor = actorFactory.createTestActor(MockRaftActor.builder().id(FOLLOWER_ID).peerAddresses(ImmutableMap.of(LEADER_ID, leaderActor.path().toString())).config(configParams).persistent(Optional.of(false)).props().withDispatcher(Dispatchers.DefaultDispatcherId()), actorFactory.generateActorId(FOLLOWER_ID));
    followerRaftActor.underlyingActor().waitForInitializeBehaviorComplete();
    followerRaftActor.tell(new AppendEntries(1, LEADER_ID, 0, 1, Collections.<ReplicatedLogEntry>emptyList(), -1, -1, (short) 0), leaderActor);
    followerRaftActor.tell(new RemoveServer(FOLLOWER_ID), testKit.getRef());
    expectFirstMatching(leaderActor, RemoveServer.class);
    LOG.info("testRemoveServerForwardToLeader ending");
}

Example 25 with AppendEntries

use of org.opendaylight.controller.cluster.raft.messages.AppendEntries in project controller by opendaylight.

the class IsolationScenarioTest method testLeaderIsolationWithPriorUncommittedEntryAndOneConflictingEntry.

/**
 * Isolates the leader with a payload entry that's replicated to all followers and committed on the leader but
 * uncommitted on the followers. While isolated, the majority partition elects a new leader and both sides of the
 * partition attempt to commit one entry independently. After isolation is removed, the entry will conflict and both
 * sides should reconcile their logs appropriately.
 */
@Test
public void testLeaderIsolationWithPriorUncommittedEntryAndOneConflictingEntry() throws Exception {
    testLog.info("testLeaderIsolationWithPriorUncommittedEntryAndOneConflictingEntry starting");
    createRaftActors();
    // Submit an initial payload that is committed/applied on all nodes.
    final MockPayload payload0 = sendPayloadData(leaderActor, "zero");
    verifyApplyJournalEntries(leaderCollectorActor, 0);
    verifyApplyJournalEntries(follower1CollectorActor, 0);
    verifyApplyJournalEntries(follower2CollectorActor, 0);
    // Submit another payload that is replicated to all followers and committed on the leader but the leader is
    // isolated before the entry is committed on the followers. To accomplish this we drop the AppendEntries
    // with the updated leader commit index.
    follower1Actor.underlyingActor().startDropMessages(AppendEntries.class, ae -> ae.getLeaderCommit() == 1);
    follower2Actor.underlyingActor().startDropMessages(AppendEntries.class, ae -> ae.getLeaderCommit() == 1);
    MockPayload payload1 = sendPayloadData(leaderActor, "one");
    // Wait for the isolated leader to send AppendEntries to the followers with the new entry with index 1. This
    // message is forwarded to the followers.
    expectFirstMatching(follower1CollectorActor, AppendEntries.class, ae -> ae.getEntries().size() == 1 && ae.getEntries().get(0).getIndex() == 1 && ae.getEntries().get(0).getData().equals(payload1));
    expectFirstMatching(follower2CollectorActor, AppendEntries.class, ae -> ae.getEntries().size() == 1 && ae.getEntries().get(0).getIndex() == 1 && ae.getEntries().get(0).getData().equals(payload1));
    verifyApplyJournalEntries(leaderCollectorActor, 1);
    isolateLeader();
    // Send a payload to the isolated leader so it has an uncommitted log entry with index 2.
    testLog.info("Sending payload to isolated leader");
    final MockPayload isolatedLeaderPayload2 = sendPayloadData(leaderActor, "two");
    // Wait for the isolated leader to send AppendEntries to follower1 with the entry at index 2. Note the message
    // is collected but not forwarded to the follower RaftActor.
    AppendEntries appendEntries = expectFirstMatching(follower1CollectorActor, AppendEntries.class);
    assertEquals("getTerm", currentTerm, appendEntries.getTerm());
    assertEquals("getLeaderId", leaderId, appendEntries.getLeaderId());
    assertEquals("getEntries().size()", 1, appendEntries.getEntries().size());
    verifyReplicatedLogEntry(appendEntries.getEntries().get(0), currentTerm, 2, isolatedLeaderPayload2);
    // The leader should transition to IsolatedLeader.
    expectFirstMatching(leaderNotifierActor, RoleChanged.class, rc -> rc.getNewRole().equals(RaftState.IsolatedLeader.name()));
    forceElectionOnFollower1();
    // Send a payload to the new leader follower1 and verify it's replicated to follower2 and committed. Since the
    // entry with index 1 from the previous term was uncommitted, the new leader should've also committed a
    // NoopPayload entry with index 2 in the PreLeader state. Thus the new payload will have index 3.
    testLog.info("Sending payload to new leader");
    final MockPayload newLeaderPayload2 = sendPayloadData(follower1Actor, "two-new");
    verifyApplyJournalEntries(follower1CollectorActor, 3);
    verifyApplyJournalEntries(follower2CollectorActor, 3);
    assertEquals("Follower 1 journal last term", currentTerm, follower1Context.getReplicatedLog().lastTerm());
    assertEquals("Follower 1 journal last index", 3, follower1Context.getReplicatedLog().lastIndex());
    assertEquals("Follower 1 commit index", 3, follower1Context.getCommitIndex());
    verifyReplicatedLogEntry(follower1Context.getReplicatedLog().get(3), currentTerm, 3, newLeaderPayload2);
    assertEquals("Follower 1 state", Lists.newArrayList(payload0, payload1, newLeaderPayload2), follower1Actor.underlyingActor().getState());
    removeIsolation();
    // Previous leader should switch to follower b/c it will receive either an AppendEntries or AppendEntriesReply
    // with a higher term.
    expectFirstMatching(leaderNotifierActor, RoleChanged.class, rc -> rc.getNewRole().equals(RaftState.Follower.name()));
    // The previous leader has a conflicting log entry at index 2 with a different term which should get
    // replaced by the new leader's entry.
    verifyApplyJournalEntries(leaderCollectorActor, 3);
    verifyRaftState(leaderActor, raftState -> {
        assertEquals("Prior leader journal last term", currentTerm, leaderContext.getReplicatedLog().lastTerm());
        assertEquals("Prior leader journal last index", 3, leaderContext.getReplicatedLog().lastIndex());
        assertEquals("Prior leader commit index", 3, leaderContext.getCommitIndex());
    });
    assertEquals("Prior leader state", Lists.newArrayList(payload0, payload1, newLeaderPayload2), leaderActor.underlyingActor().getState());
    // Ensure the prior leader didn't apply its conflicting entry with index 2, term 1.
    List<ApplyState> applyState = getAllMatching(leaderCollectorActor, ApplyState.class);
    for (ApplyState as : applyState) {
        if (as.getReplicatedLogEntry().getIndex() == 2 && as.getReplicatedLogEntry().getTerm() == 1) {
            fail("Got unexpected ApplyState: " + as);
        }
    }
    // The prior leader should not have needed a snapshot installed in order to get it synced.
    assertNoneMatching(leaderCollectorActor, InstallSnapshot.class);
    testLog.info("testLeaderIsolationWithPriorUncommittedEntryAndOneConflictingEntry ending");
}