Examples with FollowerLogInformation org.opendaylight.controller.cluster.raft.FollowerLogInformation used on opensource projects

Example 11 with FollowerLogInformation

use of org.opendaylight.controller.cluster.raft.FollowerLogInformation in project controller by opendaylight.

the class LeaderTest method testHandleAppendEntriesReplyFailureWithFollowersLogEmpty.

@Test
public void testHandleAppendEntriesReplyFailureWithFollowersLogEmpty() {
    logStart("testHandleAppendEntriesReplyFailureWithFollowersLogEmpty");
    MockRaftActorContext leaderActorContext = createActorContextWithFollower();
    ((DefaultConfigParamsImpl) leaderActorContext.getConfigParams()).setHeartBeatInterval(new FiniteDuration(1000, TimeUnit.SECONDS));
    leaderActorContext.setReplicatedLog(new MockRaftActorContext.MockReplicatedLogBuilder().createEntries(0, 2, 1).build());
    long leaderCommitIndex = 1;
    leaderActorContext.setCommitIndex(leaderCommitIndex);
    leaderActorContext.setLastApplied(leaderCommitIndex);
    final ReplicatedLogEntry leadersFirstLogEntry = leaderActorContext.getReplicatedLog().get(0);
    final ReplicatedLogEntry leadersSecondLogEntry = leaderActorContext.getReplicatedLog().get(1);
    MockRaftActorContext followerActorContext = createFollowerActorContextWithLeader();
    followerActorContext.setReplicatedLog(new MockRaftActorContext.MockReplicatedLogBuilder().build());
    followerActorContext.setCommitIndex(-1);
    followerActorContext.setLastApplied(-1);
    Follower follower = new Follower(followerActorContext);
    followerActor.underlyingActor().setBehavior(follower);
    followerActorContext.setCurrentBehavior(follower);
    leader = new Leader(leaderActorContext);
    AppendEntries appendEntries = MessageCollectorActor.expectFirstMatching(followerActor, AppendEntries.class);
    final AppendEntriesReply appendEntriesReply = MessageCollectorActor.expectFirstMatching(leaderActor, AppendEntriesReply.class);
    MessageCollectorActor.clearMessages(followerActor);
    MessageCollectorActor.clearMessages(leaderActor);
    // Verify initial AppendEntries sent with the leader's current commit index.
    assertEquals("getLeaderCommit", -1, appendEntries.getLeaderCommit());
    assertEquals("Log entries size", 0, appendEntries.getEntries().size());
    assertEquals("getPrevLogIndex", 0, appendEntries.getPrevLogIndex());
    leaderActor.underlyingActor().setBehavior(leader);
    leaderActorContext.setCurrentBehavior(leader);
    leader.handleMessage(followerActor, appendEntriesReply);
    MessageCollectorActor.expectMatching(leaderActor, AppendEntriesReply.class, 1);
    appendEntries = MessageCollectorActor.expectFirstMatching(followerActor, AppendEntries.class);
    assertEquals("getLeaderCommit", leaderCommitIndex, appendEntries.getLeaderCommit());
    assertEquals("getPrevLogIndex", -1, appendEntries.getPrevLogIndex());
    assertEquals("Log entries size", 2, appendEntries.getEntries().size());
    assertEquals("First entry index", 0, appendEntries.getEntries().get(0).getIndex());
    assertEquals("First entry data", leadersFirstLogEntry.getData(), appendEntries.getEntries().get(0).getData());
    assertEquals("Second entry index", 1, appendEntries.getEntries().get(1).getIndex());
    assertEquals("Second entry data", leadersSecondLogEntry.getData(), appendEntries.getEntries().get(1).getData());
    FollowerLogInformation followerInfo = leader.getFollower(FOLLOWER_ID);
    assertEquals("getNextIndex", 2, followerInfo.getNextIndex());
    List<ApplyState> applyStateList = MessageCollectorActor.expectMatching(followerActor, ApplyState.class, 2);
    ApplyState applyState = applyStateList.get(0);
    assertEquals("Follower's first ApplyState index", 0, applyState.getReplicatedLogEntry().getIndex());
    assertEquals("Follower's first ApplyState term", 1, applyState.getReplicatedLogEntry().getTerm());
    assertEquals("Follower's first ApplyState data", leadersFirstLogEntry.getData(), applyState.getReplicatedLogEntry().getData());
    applyState = applyStateList.get(1);
    assertEquals("Follower's second ApplyState index", 1, applyState.getReplicatedLogEntry().getIndex());
    assertEquals("Follower's second ApplyState term", 1, applyState.getReplicatedLogEntry().getTerm());
    assertEquals("Follower's second ApplyState data", leadersSecondLogEntry.getData(), applyState.getReplicatedLogEntry().getData());
    assertEquals("Follower's commit index", 1, followerActorContext.getCommitIndex());
    assertEquals("Follower's lastIndex", 1, followerActorContext.getReplicatedLog().lastIndex());
}

Example 12 with FollowerLogInformation

use of org.opendaylight.controller.cluster.raft.FollowerLogInformation in project controller by opendaylight.

the class AbstractLeader method purgeInMemoryLog.

private void purgeInMemoryLog() {
    // find the lowest index across followers which has been replicated to all.
    // lastApplied if there are no followers, so that we keep clearing the log for single-node
    // we would delete the in-mem log from that index on, in-order to minimize mem usage
    // we would also share this info thru AE with the followers so that they can delete their log entries as well.
    long minReplicatedToAllIndex = followerToLog.isEmpty() ? context.getLastApplied() : Long.MAX_VALUE;
    for (FollowerLogInformation info : followerToLog.values()) {
        minReplicatedToAllIndex = Math.min(minReplicatedToAllIndex, info.getMatchIndex());
    }
    super.performSnapshotWithoutCapture(minReplicatedToAllIndex);
}

Example 13 with FollowerLogInformation

use of org.opendaylight.controller.cluster.raft.FollowerLogInformation in project controller by opendaylight.

the class AbstractLeader method possiblyUpdateCommitIndex.

private void possiblyUpdateCommitIndex() {
    // set commitIndex = N (§5.3, §5.4).
    for (long index = context.getCommitIndex() + 1; ; index++) {
        ReplicatedLogEntry replicatedLogEntry = context.getReplicatedLog().get(index);
        if (replicatedLogEntry == null) {
            log.trace("{}: ReplicatedLogEntry not found for index {} - snapshotIndex: {}, journal size: {}", logName(), index, context.getReplicatedLog().getSnapshotIndex(), context.getReplicatedLog().size());
            break;
        }
        // Count our entry if it has been persisted.
        int replicatedCount = replicatedLogEntry.isPersistencePending() ? 0 : 1;
        if (replicatedCount == 0) {
            // amongst the followers w/o the local persistence ack.
            break;
        }
        log.trace("{}: checking Nth index {}", logName(), index);
        for (FollowerLogInformation info : followerToLog.values()) {
            final PeerInfo peerInfo = context.getPeerInfo(info.getId());
            if (info.getMatchIndex() >= index && peerInfo != null && peerInfo.isVoting()) {
                replicatedCount++;
            } else if (log.isTraceEnabled()) {
                log.trace("{}: Not counting follower {} - matchIndex: {}, {}", logName(), info.getId(), info.getMatchIndex(), peerInfo);
            }
        }
        if (log.isTraceEnabled()) {
            log.trace("{}: replicatedCount {}, minReplicationCount: {}", logName(), replicatedCount, minReplicationCount);
        }
        if (replicatedCount >= minReplicationCount) {
            // counting replicas, then all prior entries are committed indirectly".
            if (replicatedLogEntry.getTerm() == currentTerm()) {
                log.trace("{}: Setting commit index to {}", logName(), index);
                context.setCommitIndex(index);
            } else {
                log.debug("{}: Not updating commit index to {} - retrieved log entry with index {}, " + "term {} does not match the current term {}", logName(), index, replicatedLogEntry.getIndex(), replicatedLogEntry.getTerm(), currentTerm());
            }
        } else {
            log.trace("{}: minReplicationCount not reached, actual {} - breaking", logName(), replicatedCount);
            break;
        }
    }
    // Apply the change to the state machine
    if (context.getCommitIndex() > context.getLastApplied()) {
        log.debug("{}: Applying to log - commitIndex: {}, lastAppliedIndex: {}", logName(), context.getCommitIndex(), context.getLastApplied());
        applyLogToStateMachine(context.getCommitIndex());
    }
    if (!context.getSnapshotManager().isCapturing()) {
        purgeInMemoryLog();
    }
}

Example 14 with FollowerLogInformation

use of org.opendaylight.controller.cluster.raft.FollowerLogInformation in project controller by opendaylight.

the class AbstractLeader method handleAppendEntriesReply.

@Override
protected RaftActorBehavior handleAppendEntriesReply(final ActorRef sender, final AppendEntriesReply appendEntriesReply) {
    log.trace("{}: handleAppendEntriesReply: {}", logName(), appendEntriesReply);
    // Update the FollowerLogInformation
    String followerId = appendEntriesReply.getFollowerId();
    FollowerLogInformation followerLogInformation = followerToLog.get(followerId);
    if (followerLogInformation == null) {
        log.error("{}: handleAppendEntriesReply - unknown follower {}", logName(), followerId);
        return this;
    }
    final long lastActivityNanos = followerLogInformation.nanosSinceLastActivity();
    if (lastActivityNanos > context.getConfigParams().getElectionTimeOutInterval().toNanos()) {
        log.warn("{} : handleAppendEntriesReply delayed beyond election timeout, " + "appendEntriesReply : {}, timeSinceLastActivity : {}, lastApplied : {}, commitIndex : {}", logName(), appendEntriesReply, TimeUnit.NANOSECONDS.toMillis(lastActivityNanos), context.getLastApplied(), context.getCommitIndex());
    }
    followerLogInformation.markFollowerActive();
    followerLogInformation.setPayloadVersion(appendEntriesReply.getPayloadVersion());
    followerLogInformation.setRaftVersion(appendEntriesReply.getRaftVersion());
    long followerLastLogIndex = appendEntriesReply.getLogLastIndex();
    long followersLastLogTermInLeadersLog = getLogEntryTerm(followerLastLogIndex);
    boolean updated = false;
    if (appendEntriesReply.getLogLastIndex() > context.getReplicatedLog().lastIndex()) {
        // The follower's log is actually ahead of the leader's log. Normally this doesn't happen
        // in raft as a node cannot become leader if it's log is behind another's. However, the
        // non-voting semantics deviate a bit from raft. Only voting members participate in
        // elections and can become leader so it's possible for a non-voting follower to be ahead
        // of the leader. This can happen if persistence is disabled and all voting members are
        // restarted. In this case, the voting leader will start out with an empty log however
        // the non-voting followers still retain the previous data in memory. On the first
        // AppendEntries, the non-voting follower returns a successful reply b/c the prevLogIndex
        // sent by the leader is -1 and thus the integrity checks pass. However the follower's returned
        // lastLogIndex may be higher in which case we want to reset the follower by installing a
        // snapshot. It's also possible that the follower's last log index is behind the leader's.
        // However in this case the log terms won't match and the logs will conflict - this is handled
        // elsewhere.
        log.info("{}: handleAppendEntriesReply: follower {} lastIndex {} is ahead of our lastIndex {} " + "(snapshotIndex {}) - forcing install snaphot", logName(), followerLogInformation.getId(), appendEntriesReply.getLogLastIndex(), context.getReplicatedLog().lastIndex(), context.getReplicatedLog().getSnapshotIndex());
        followerLogInformation.setMatchIndex(-1);
        followerLogInformation.setNextIndex(-1);
        initiateCaptureSnapshot(followerId);
        updated = true;
    } else if (appendEntriesReply.isSuccess()) {
        if (followerLastLogIndex >= 0 && followersLastLogTermInLeadersLog >= 0 && followersLastLogTermInLeadersLog != appendEntriesReply.getLogLastTerm()) {
            // The follower's last entry is present in the leader's journal but the terms don't match so the
            // follower has a conflicting entry. Since the follower didn't report that it's out of sync, this means
            // either the previous leader entry sent didn't conflict or the previous leader entry is in the snapshot
            // and no longer in the journal. Either way, we set the follower's next index to 1 less than the last
            // index reported by the follower. For the former case, the leader will send all entries starting with
            // the previous follower's index and the follower will remove and replace the conflicting entries as
            // needed. For the latter, the leader will initiate an install snapshot.
            followerLogInformation.setNextIndex(followerLastLogIndex - 1);
            updated = true;
            log.info("{}: handleAppendEntriesReply: follower {} last log term {} for index {} conflicts with the " + "leader's {} - set the follower's next index to {}", logName(), followerId, appendEntriesReply.getLogLastTerm(), appendEntriesReply.getLogLastIndex(), followersLastLogTermInLeadersLog, followerLogInformation.getNextIndex());
        } else {
            updated = updateFollowerLogInformation(followerLogInformation, appendEntriesReply);
        }
    } else {
        log.info("{}: handleAppendEntriesReply - received unsuccessful reply: {}, leader snapshotIndex: {}", logName(), appendEntriesReply, context.getReplicatedLog().getSnapshotIndex());
        if (appendEntriesReply.isForceInstallSnapshot()) {
            // Reset the followers match and next index. This is to signal that this follower has nothing
            // in common with this Leader and so would require a snapshot to be installed
            followerLogInformation.setMatchIndex(-1);
            followerLogInformation.setNextIndex(-1);
            // Force initiate a snapshot capture
            initiateCaptureSnapshot(followerId);
        } else if (followerLastLogIndex < 0 || followersLastLogTermInLeadersLog >= 0 && followersLastLogTermInLeadersLog == appendEntriesReply.getLogLastTerm()) {
            // The follower's log is empty or the last entry is present in the leader's journal
            // and the terms match so the follower is just behind the leader's journal from
            // the last snapshot, if any. We'll catch up the follower quickly by starting at the
            // follower's last log index.
            updated = updateFollowerLogInformation(followerLogInformation, appendEntriesReply);
            log.info("{}: follower {} appears to be behind the leader from the last snapshot - " + "updated: matchIndex: {}, nextIndex: {}", logName(), followerId, followerLogInformation.getMatchIndex(), followerLogInformation.getNextIndex());
        } else {
            if (followerLogInformation.decrNextIndex()) {
                updated = true;
                log.info("{}: follower {} last log term {} conflicts with the leader's {} - dec next index to {}", logName(), followerId, appendEntriesReply.getLogLastTerm(), followersLastLogTermInLeadersLog, followerLogInformation.getNextIndex());
            }
        }
    }
    if (log.isTraceEnabled()) {
        log.trace("{}: handleAppendEntriesReply from {}: commitIndex: {}, lastAppliedIndex: {}, currentTerm: {}", logName(), followerId, context.getCommitIndex(), context.getLastApplied(), currentTerm());
    }
    possiblyUpdateCommitIndex();
    // Send the next log entry immediately, if possible, no need to wait for heartbeat to trigger that event
    sendUpdatesToFollower(followerId, followerLogInformation, false, !updated);
    return this;
}

Example 15 with FollowerLogInformation

use of org.opendaylight.controller.cluster.raft.FollowerLogInformation in project controller by opendaylight.

the class AbstractLeader method sendAppendEntries.

protected void sendAppendEntries(final long timeSinceLastActivityIntervalNanos, final boolean isHeartbeat) {
    // Send an AppendEntries to all followers
    for (Entry<String, FollowerLogInformation> e : followerToLog.entrySet()) {
        final String followerId = e.getKey();
        final FollowerLogInformation followerLogInformation = e.getValue();
        // This checks helps not to send a repeat message to the follower
        if (!followerLogInformation.isFollowerActive() || followerLogInformation.nanosSinceLastActivity() >= timeSinceLastActivityIntervalNanos) {
            sendUpdatesToFollower(followerId, followerLogInformation, true, isHeartbeat);
        }
    }
}