Examples with Snapshot org.opendaylight.controller.cluster.raft.persisted.Snapshot used on opensource projects

Example 31 with Snapshot

use of org.opendaylight.controller.cluster.raft.persisted.Snapshot in project controller by opendaylight.

the class ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest method verifyReplicationsAndSnapshotWithNoLaggingAfterInstallSnapshot.

/**
 * Do another round of payloads and snapshot to verify replicatedToAllIndex gets back on track and
 * snapshots works as expected after doing a follower snapshot. In this step we don't lag a follower.
 */
private long verifyReplicationsAndSnapshotWithNoLaggingAfterInstallSnapshot() throws Exception {
    testLog.info("verifyReplicationsAndSnapshotWithNoLaggingAfterInstallSnapshot starting: replicatedToAllIndex: {}", leader.getReplicatedToAllIndex());
    // Send another payload - a snapshot should occur.
    MockPayload payload4 = sendPayloadData(leaderActor, "four");
    // Wait for the snapshot to complete.
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, SaveSnapshotSuccess.class);
    ApplyState applyState = MessageCollectorActor.expectFirstMatching(leaderCollectorActor, ApplyState.class);
    verifyApplyState(applyState, leaderCollectorActor, payload4.toString(), currentTerm, 4, payload4);
    // Verify the leader's last persisted snapshot (previous ones may not be purged yet).
    List<Snapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(leaderId, Snapshot.class);
    Snapshot persistedSnapshot = persistedSnapshots.get(persistedSnapshots.size() - 1);
    // The last (fourth) payload may or may not have been applied when the snapshot is captured depending on the
    // timing when the async persistence completes.
    List<ReplicatedLogEntry> unAppliedEntry = persistedSnapshot.getUnAppliedEntries();
    long leadersSnapshotIndex;
    if (unAppliedEntry.isEmpty()) {
        leadersSnapshotIndex = 4;
        expSnapshotState.add(payload4);
        verifySnapshot("Persisted", persistedSnapshot, currentTerm, 4, currentTerm, 4);
    } else {
        leadersSnapshotIndex = 3;
        verifySnapshot("Persisted", persistedSnapshot, currentTerm, 3, currentTerm, 4);
        assertEquals("Persisted Snapshot getUnAppliedEntries size", 1, unAppliedEntry.size());
        verifyReplicatedLogEntry(unAppliedEntry.get(0), currentTerm, 4, payload4);
        expSnapshotState.add(payload4);
    }
    // Send a couple more payloads.
    MockPayload payload5 = sendPayloadData(leaderActor, "five");
    MockPayload payload6 = sendPayloadData(leaderActor, "six");
    // Verify the leader applies the 2 log entries.
    List<ApplyState> applyStates = MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 3);
    verifyApplyState(applyStates.get(1), leaderCollectorActor, payload5.toString(), currentTerm, 5, payload5);
    verifyApplyState(applyStates.get(2), leaderCollectorActor, payload6.toString(), currentTerm, 6, payload6);
    // Verify the leader applies a log entry for at least the last entry index.
    verifyApplyJournalEntries(leaderCollectorActor, 6);
    // Ensure there's at least 1 more heartbeat to trim the log.
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, AppendEntriesReply.class);
    // Verify the leader's final state.
    verifyLeadersTrimmedLog(6);
    InMemoryJournal.dumpJournal(leaderId);
    // Verify the leaders's persisted journal log - it should only contain the last 2 ReplicatedLogEntries
    // added after the snapshot as the persisted journal should've been purged to the snapshot
    // sequence number.
    verifyPersistedJournal(leaderId, Arrays.asList(new SimpleReplicatedLogEntry(5, currentTerm, payload5), new SimpleReplicatedLogEntry(6, currentTerm, payload6)));
    // Verify the leaders's persisted journal contains an ApplyJournalEntries for at least the last entry index.
    List<ApplyJournalEntries> persistedApplyJournalEntries = InMemoryJournal.get(leaderId, ApplyJournalEntries.class);
    boolean found = false;
    for (ApplyJournalEntries entry : persistedApplyJournalEntries) {
        if (entry.getToIndex() == 6) {
            found = true;
            break;
        }
    }
    Assert.assertTrue(String.format("ApplyJournalEntries with index %d not found in leader's persisted journal", 6), found);
    // Verify follower 1 applies the 3 log entries.
    applyStates = MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, 3);
    verifyApplyState(applyStates.get(0), null, null, currentTerm, 4, payload4);
    verifyApplyState(applyStates.get(1), null, null, currentTerm, 5, payload5);
    verifyApplyState(applyStates.get(2), null, null, currentTerm, 6, payload6);
    // Verify follower 1's log state.
    verifyFollowersTrimmedLog(1, follower1Actor, 6);
    // Verify follower 2 applies the 3 log entries.
    applyStates = MessageCollectorActor.expectMatching(follower2CollectorActor, ApplyState.class, 3);
    verifyApplyState(applyStates.get(0), null, null, currentTerm, 4, payload4);
    verifyApplyState(applyStates.get(1), null, null, currentTerm, 5, payload5);
    verifyApplyState(applyStates.get(2), null, null, currentTerm, 6, payload6);
    // Verify follower 2's log state.
    verifyFollowersTrimmedLog(2, follower2Actor, 6);
    expSnapshotState.add(payload5);
    expSnapshotState.add(payload6);
    testLog.info("verifyReplicationsAndSnapshotWithNoLaggingAfterInstallSnapshot ending");
    return leadersSnapshotIndex;
}

Example 32 with Snapshot

use of org.opendaylight.controller.cluster.raft.persisted.Snapshot in project controller by opendaylight.

the class ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest method testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries.

/**
 * Send payloads to trigger a leader snapshot due to snapshotBatchCount reached with follower 2
 * lagging but not enough for the leader to trim its log from the last applied index. Follower 2's log
 * will be behind by several entries and, when it is resumed, it should be caught up via AppendEntries
 * sent by the leader.
 */
@Test
public void testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries() throws Exception {
    testLog.info("testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries starting");
    setup();
    sendInitialPayloadsReplicatedToAllFollowers("zero", "one");
    // Configure follower 2 to drop messages and lag.
    follower2Actor.underlyingActor().startDropMessages(AppendEntries.class);
    // Send the first payload and verify it gets applied by the leader and follower 1.
    MockPayload payload2 = sendPayloadData(leaderActor, "two");
    ApplyState applyState = MessageCollectorActor.expectFirstMatching(leaderCollectorActor, ApplyState.class);
    verifyApplyState(applyState, leaderCollectorActor, payload2.toString(), currentTerm, 2, payload2);
    applyState = MessageCollectorActor.expectFirstMatching(follower1CollectorActor, ApplyState.class);
    verifyApplyState(applyState, null, null, currentTerm, 2, payload2);
    expSnapshotState.add(payload2);
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.clearMessages(follower1CollectorActor);
    // Send another payload - this should cause a snapshot due to snapshotBatchCount reached.
    MockPayload payload3 = sendPayloadData(leaderActor, "three");
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, SaveSnapshotSuccess.class);
    testLog.info("testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries: sending 2 more payloads");
    // Send 2 more payloads - not enough to trigger another snapshot.
    MockPayload payload4 = sendPayloadData(leaderActor, "four");
    MockPayload payload5 = sendPayloadData(leaderActor, "five");
    // Verify the leader got consensus and applies each log entry even though follower 2 didn't respond.
    List<ApplyState> applyStates = MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 3);
    verifyApplyState(applyStates.get(0), leaderCollectorActor, payload3.toString(), currentTerm, 3, payload3);
    verifyApplyState(applyStates.get(1), leaderCollectorActor, payload4.toString(), currentTerm, 4, payload4);
    verifyApplyState(applyStates.get(2), leaderCollectorActor, payload5.toString(), currentTerm, 5, payload5);
    // Verify follower 1 applies each log entry.
    applyStates = MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, 3);
    verifyApplyState(applyStates.get(0), null, null, currentTerm, 3, payload3);
    verifyApplyState(applyStates.get(1), null, null, currentTerm, 4, payload4);
    verifyApplyState(applyStates.get(2), null, null, currentTerm, 5, payload5);
    // The snapshot should have caused the leader to advanced the snapshot index to the
    // last previously applied index (1) that was replicated to all followers at the time of capture.
    // Note: since the log size (3) did not exceed the snapshot batch count (4), the leader should not
    // have trimmed the log to the last index actually applied (5).
    assertEquals("Leader snapshot term", currentTerm, leaderContext.getReplicatedLog().getSnapshotTerm());
    assertEquals("Leader snapshot index", 1, leaderContext.getReplicatedLog().getSnapshotIndex());
    assertEquals("Leader journal log size", 4, leaderContext.getReplicatedLog().size());
    assertEquals("Leader journal last index", 5, leaderContext.getReplicatedLog().lastIndex());
    assertEquals("Leader commit index", 5, leaderContext.getCommitIndex());
    assertEquals("Leader last applied", 5, leaderContext.getLastApplied());
    assertEquals("Leader replicatedToAllIndex", 1, leader.getReplicatedToAllIndex());
    // Now stop dropping AppendEntries in follower 2.
    follower2Actor.underlyingActor().stopDropMessages(AppendEntries.class);
    // Verify follower 2 applies each log entry. The leader should not install a snapshot b/c
    // follower 2's next index (3) is still present in the log.
    applyStates = MessageCollectorActor.expectMatching(follower2CollectorActor, ApplyState.class, 4);
    verifyApplyState(applyStates.get(0), null, null, currentTerm, 2, payload2);
    verifyApplyState(applyStates.get(1), null, null, currentTerm, 3, payload3);
    verifyApplyState(applyStates.get(2), null, null, currentTerm, 4, payload4);
    verifyApplyState(applyStates.get(3), null, null, currentTerm, 5, payload5);
    // Verify the leader did not try to install a snapshot to catch up follower 2.
    InstallSnapshot installSnapshot = MessageCollectorActor.getFirstMatching(follower2CollectorActor, InstallSnapshot.class);
    Assert.assertNull("Follower 2 received unexpected InstallSnapshot", installSnapshot);
    // Ensure there's at least 1 more heartbeat.
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, AppendEntriesReply.class);
    // The leader should now have performed fake snapshots to advance the snapshot index and to trim
    // the log. In addition replicatedToAllIndex should've advanced.
    verifyLeadersTrimmedLog(5);
    // Verify the leader's persisted snapshot.
    List<Snapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(leaderId, Snapshot.class);
    assertEquals("Persisted snapshots size", 1, persistedSnapshots.size());
    verifySnapshot("Persisted", persistedSnapshots.get(0), currentTerm, 2, currentTerm, 3);
    List<ReplicatedLogEntry> unAppliedEntry = persistedSnapshots.get(0).getUnAppliedEntries();
    assertEquals("Persisted Snapshot getUnAppliedEntries size", 1, unAppliedEntry.size());
    verifyReplicatedLogEntry(unAppliedEntry.get(0), currentTerm, 3, payload3);
    // Verify follower 1's log and snapshot indexes.
    MessageCollectorActor.clearMessages(follower1CollectorActor);
    MessageCollectorActor.expectFirstMatching(follower1CollectorActor, AppendEntries.class);
    verifyFollowersTrimmedLog(1, follower1Actor, 5);
    // Verify follower 2's log and snapshot indexes.
    MessageCollectorActor.clearMessages(follower2CollectorActor);
    MessageCollectorActor.expectFirstMatching(follower2CollectorActor, AppendEntries.class);
    verifyFollowersTrimmedLog(2, follower2Actor, 5);
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.clearMessages(follower1CollectorActor);
    MessageCollectorActor.clearMessages(follower2CollectorActor);
    expSnapshotState.add(payload3);
    expSnapshotState.add(payload4);
    expSnapshotState.add(payload5);
    testLog.info("testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries complete");
}

Example 33 with Snapshot

use of org.opendaylight.controller.cluster.raft.persisted.Snapshot in project controller by opendaylight.

the class Follower method processNewEntries.

private boolean processNewEntries(final AppendEntries appendEntries, final ActorRef sender) {
    int numLogEntries = appendEntries.getEntries().size();
    if (numLogEntries == 0) {
        return true;
    }
    log.debug("{}: Number of entries to be appended = {}", logName(), numLogEntries);
    long lastIndex = lastIndex();
    int addEntriesFrom = 0;
    // term), delete the existing entry and all that follow it (§5.3)
    if (context.getReplicatedLog().size() > 0) {
        // Find the entry up until the one that is not in the follower's log
        for (int i = 0; i < numLogEntries; i++, addEntriesFrom++) {
            ReplicatedLogEntry matchEntry = appendEntries.getEntries().get(i);
            if (!isLogEntryPresent(matchEntry.getIndex())) {
                // newEntry not found in the log
                break;
            }
            long existingEntryTerm = getLogEntryTerm(matchEntry.getIndex());
            log.debug("{}: matchEntry {} is present: existingEntryTerm: {}", logName(), matchEntry, existingEntryTerm);
            // what the term was so we'll assume it matches.
            if (existingEntryTerm == -1 || existingEntryTerm == matchEntry.getTerm()) {
                continue;
            }
            if (!context.getRaftPolicy().applyModificationToStateBeforeConsensus()) {
                log.info("{}: Removing entries from log starting at {}, commitIndex: {}, lastApplied: {}", logName(), matchEntry.getIndex(), context.getCommitIndex(), context.getLastApplied());
                // been applied to the state yet.
                if (matchEntry.getIndex() <= context.getLastApplied() || !context.getReplicatedLog().removeFromAndPersist(matchEntry.getIndex())) {
                    // Could not remove the entries - this means the matchEntry index must be in the
                    // snapshot and not the log. In this case the prior entries are part of the state
                    // so we must send back a reply to force a snapshot to completely re-sync the
                    // follower's log and state.
                    log.info("{}: Could not remove entries - sending reply to force snapshot", logName());
                    sender.tell(new AppendEntriesReply(context.getId(), currentTerm(), false, lastIndex, lastTerm(), context.getPayloadVersion(), true), actor());
                    return false;
                }
                break;
            } else {
                sender.tell(new AppendEntriesReply(context.getId(), currentTerm(), false, lastIndex, lastTerm(), context.getPayloadVersion(), true), actor());
                return false;
            }
        }
    }
    lastIndex = lastIndex();
    log.debug("{}: After cleanup, lastIndex: {}, entries to be added from: {}", logName(), lastIndex, addEntriesFrom);
    // When persistence successfully completes for each new log entry appended, we need to determine if we
    // should capture a snapshot to compact the persisted log. shouldCaptureSnapshot tracks whether or not
    // one of the log entries has exceeded the log size threshold whereby a snapshot should be taken. However
    // we don't initiate the snapshot at that log entry but rather after the last log entry has been persisted.
    // This is done because subsequent log entries after the one that tripped the threshold may have been
    // applied to the state already, as the persistence callback occurs async, and we want those entries
    // purged from the persisted log as well.
    final AtomicBoolean shouldCaptureSnapshot = new AtomicBoolean(false);
    final Procedure<ReplicatedLogEntry> appendAndPersistCallback = logEntry -> {
        final List<ReplicatedLogEntry> entries = appendEntries.getEntries();
        final ReplicatedLogEntry lastEntryToAppend = entries.get(entries.size() - 1);
        if (shouldCaptureSnapshot.get() && logEntry == lastEntryToAppend) {
            context.getSnapshotManager().capture(context.getReplicatedLog().last(), getReplicatedToAllIndex());
        }
    };
    // Append any new entries not already in the log
    for (int i = addEntriesFrom; i < numLogEntries; i++) {
        ReplicatedLogEntry entry = appendEntries.getEntries().get(i);
        log.debug("{}: Append entry to log {}", logName(), entry.getData());
        context.getReplicatedLog().appendAndPersist(entry, appendAndPersistCallback, false);
        shouldCaptureSnapshot.compareAndSet(false, context.getReplicatedLog().shouldCaptureSnapshot(entry.getIndex()));
        if (entry.getData() instanceof ServerConfigurationPayload) {
            context.updatePeerIds((ServerConfigurationPayload) entry.getData());
        }
    }
    log.debug("{}: Log size is now {}", logName(), context.getReplicatedLog().size());
    return true;
}

Example 34 with Snapshot

use of org.opendaylight.controller.cluster.raft.persisted.Snapshot in project controller by opendaylight.

the class MigratedMessagesTest method doTestSnapshotAfterStartupWithMigratedMessage.

@SuppressWarnings("checkstyle:IllegalCatch")
private TestActorRef<MockRaftActor> doTestSnapshotAfterStartupWithMigratedMessage(String id, boolean persistent, Consumer<Snapshot> snapshotVerifier, final State snapshotState) {
    InMemorySnapshotStore.addSnapshotSavedLatch(id);
    InMemoryJournal.addDeleteMessagesCompleteLatch(id);
    DefaultConfigParamsImpl config = new DefaultConfigParamsImpl();
    config.setCustomRaftPolicyImplementationClass(DisableElectionsRaftPolicy.class.getName());
    RaftActorSnapshotCohort snapshotCohort = new RaftActorSnapshotCohort() {

        @Override
        public void createSnapshot(ActorRef actorRef, java.util.Optional<OutputStream> installSnapshotStream) {
            actorRef.tell(new CaptureSnapshotReply(snapshotState, installSnapshotStream), actorRef);
        }

        @Override
        public void applySnapshot(State newState) {
        }

        @Override
        public State deserializeSnapshot(ByteSource snapshotBytes) {
            throw new UnsupportedOperationException();
        }
    };
    TestActorRef<MockRaftActor> raftActorRef = factory.createTestActor(MockRaftActor.builder().id(id).config(config).snapshotCohort(snapshotCohort).persistent(Optional.of(persistent)).peerAddresses(ImmutableMap.of("peer", "")).props().withDispatcher(Dispatchers.DefaultDispatcherId()), id);
    MockRaftActor mockRaftActor = raftActorRef.underlyingActor();
    mockRaftActor.waitForRecoveryComplete();
    Snapshot snapshot = InMemorySnapshotStore.waitForSavedSnapshot(id, Snapshot.class);
    snapshotVerifier.accept(snapshot);
    InMemoryJournal.waitForDeleteMessagesComplete(id);
    assertEquals("InMemoryJournal size", 0, InMemoryJournal.get(id).size());
    return raftActorRef;
}

Example 35 with Snapshot

use of org.opendaylight.controller.cluster.raft.persisted.Snapshot in project controller by opendaylight.

the class RaftActorRecoverySupport method possiblyRestoreFromSnapshot.

@SuppressWarnings("checkstyle:IllegalCatch")
private void possiblyRestoreFromSnapshot() {
    Snapshot restoreFromSnapshot = cohort.getRestoreFromSnapshot();
    if (restoreFromSnapshot == null) {
        return;
    }
    if (anyDataRecovered) {
        log.warn("{}: The provided restore snapshot was not applied because the persistence store is not empty", context.getId());
        return;
    }
    log.debug("{}: Restore snapshot: {}", context.getId(), restoreFromSnapshot);
    context.getSnapshotManager().apply(new ApplySnapshot(restoreFromSnapshot));
}