Examples with Snapshot org.opendaylight.controller.cluster.raft.persisted.Snapshot used on opensource projects

Example 26 with Snapshot

use of org.opendaylight.controller.cluster.raft.persisted.Snapshot in project controller by opendaylight.

the class LeaderTest method testHandleSnapshotSendsPreviousChunksHashCodeWhenSendingNextChunk.

@Test
public void testHandleSnapshotSendsPreviousChunksHashCodeWhenSendingNextChunk() throws Exception {
    logStart("testHandleSnapshotSendsPreviousChunksHashCodeWhenSendingNextChunk");
    MockRaftActorContext actorContext = createActorContextWithFollower();
    final int commitIndex = 3;
    final int snapshotIndex = 2;
    final int snapshotTerm = 1;
    final int currentTerm = 2;
    actorContext.setConfigParams(new DefaultConfigParamsImpl() {

        @Override
        public int getSnapshotChunkSize() {
            return 50;
        }
    });
    actorContext.setCommitIndex(commitIndex);
    leader = new Leader(actorContext);
    leader.getFollower(FOLLOWER_ID).setMatchIndex(-1);
    leader.getFollower(FOLLOWER_ID).setNextIndex(0);
    Map<String, String> leadersSnapshot = new HashMap<>();
    leadersSnapshot.put("1", "A");
    leadersSnapshot.put("2", "B");
    leadersSnapshot.put("3", "C");
    // set the snapshot variables in replicatedlog
    actorContext.getReplicatedLog().setSnapshotIndex(snapshotIndex);
    actorContext.getReplicatedLog().setSnapshotTerm(snapshotTerm);
    actorContext.getTermInformation().update(currentTerm, leaderActor.path().toString());
    ByteString bs = toByteString(leadersSnapshot);
    Snapshot snapshot = Snapshot.create(ByteState.of(bs.toByteArray()), Collections.<ReplicatedLogEntry>emptyList(), commitIndex, snapshotTerm, commitIndex, snapshotTerm, -1, null, null);
    leader.handleMessage(leaderActor, new SendInstallSnapshot(snapshot, ByteSource.wrap(bs.toByteArray())));
    InstallSnapshot installSnapshot = MessageCollectorActor.expectFirstMatching(followerActor, InstallSnapshot.class);
    assertEquals(1, installSnapshot.getChunkIndex());
    assertEquals(3, installSnapshot.getTotalChunks());
    assertEquals(LeaderInstallSnapshotState.INITIAL_LAST_CHUNK_HASH_CODE, installSnapshot.getLastChunkHashCode().get().intValue());
    final int hashCode = Arrays.hashCode(installSnapshot.getData());
    followerActor.underlyingActor().clear();
    leader.handleMessage(followerActor, new InstallSnapshotReply(installSnapshot.getTerm(), FOLLOWER_ID, 1, true));
    installSnapshot = MessageCollectorActor.expectFirstMatching(followerActor, InstallSnapshot.class);
    assertEquals(2, installSnapshot.getChunkIndex());
    assertEquals(3, installSnapshot.getTotalChunks());
    assertEquals(hashCode, installSnapshot.getLastChunkHashCode().get().intValue());
}

Example 27 with Snapshot

use of org.opendaylight.controller.cluster.raft.persisted.Snapshot in project controller by opendaylight.

the class RaftActorTest method testRaftActorRecoveryWithPersistenceEnabled.

@Test
public void testRaftActorRecoveryWithPersistenceEnabled() throws Exception {
    TEST_LOG.info("testRaftActorRecoveryWithPersistenceEnabled starting");
    TestKit kit = new TestKit(getSystem());
    String persistenceId = factory.generateActorId("follower-");
    DefaultConfigParamsImpl config = new DefaultConfigParamsImpl();
    // Set the heartbeat interval high to essentially disable election otherwise the test
    // may fail if the actor is switched to Leader and the commitIndex is set to the last
    // log entry.
    config.setHeartBeatInterval(new FiniteDuration(1, TimeUnit.DAYS));
    ImmutableMap<String, String> peerAddresses = ImmutableMap.<String, String>builder().put("member1", "address").build();
    ActorRef followerActor = factory.createActor(MockRaftActor.props(persistenceId, peerAddresses, config), persistenceId);
    kit.watch(followerActor);
    List<ReplicatedLogEntry> snapshotUnappliedEntries = new ArrayList<>();
    ReplicatedLogEntry entry1 = new SimpleReplicatedLogEntry(4, 1, new MockRaftActorContext.MockPayload("E"));
    snapshotUnappliedEntries.add(entry1);
    int lastAppliedDuringSnapshotCapture = 3;
    int lastIndexDuringSnapshotCapture = 4;
    // 4 messages as part of snapshot, which are applied to state
    MockSnapshotState snapshotState = new MockSnapshotState(Arrays.asList(new MockRaftActorContext.MockPayload("A"), new MockRaftActorContext.MockPayload("B"), new MockRaftActorContext.MockPayload("C"), new MockRaftActorContext.MockPayload("D")));
    Snapshot snapshot = Snapshot.create(snapshotState, snapshotUnappliedEntries, lastIndexDuringSnapshotCapture, 1, lastAppliedDuringSnapshotCapture, 1, -1, null, null);
    InMemorySnapshotStore.addSnapshot(persistenceId, snapshot);
    // add more entries after snapshot is taken
    List<ReplicatedLogEntry> entries = new ArrayList<>();
    ReplicatedLogEntry entry2 = new SimpleReplicatedLogEntry(5, 1, new MockRaftActorContext.MockPayload("F", 2));
    ReplicatedLogEntry entry3 = new SimpleReplicatedLogEntry(6, 1, new MockRaftActorContext.MockPayload("G", 3));
    ReplicatedLogEntry entry4 = new SimpleReplicatedLogEntry(7, 1, new MockRaftActorContext.MockPayload("H", 4));
    entries.add(entry2);
    entries.add(entry3);
    entries.add(entry4);
    final int lastAppliedToState = 5;
    final int lastIndex = 7;
    InMemoryJournal.addEntry(persistenceId, 5, entry2);
    // 2 entries are applied to state besides the 4 entries in snapshot
    InMemoryJournal.addEntry(persistenceId, 6, new ApplyJournalEntries(lastAppliedToState));
    InMemoryJournal.addEntry(persistenceId, 7, entry3);
    InMemoryJournal.addEntry(persistenceId, 8, entry4);
    // kill the actor
    followerActor.tell(PoisonPill.getInstance(), null);
    kit.expectMsgClass(kit.duration("5 seconds"), Terminated.class);
    kit.unwatch(followerActor);
    // reinstate the actor
    TestActorRef<MockRaftActor> ref = factory.createTestActor(MockRaftActor.props(persistenceId, peerAddresses, config));
    MockRaftActor mockRaftActor = ref.underlyingActor();
    mockRaftActor.waitForRecoveryComplete();
    RaftActorContext context = mockRaftActor.getRaftActorContext();
    assertEquals("Journal log size", snapshotUnappliedEntries.size() + entries.size(), context.getReplicatedLog().size());
    assertEquals("Journal data size", 10, context.getReplicatedLog().dataSize());
    assertEquals("Last index", lastIndex, context.getReplicatedLog().lastIndex());
    assertEquals("Last applied", lastAppliedToState, context.getLastApplied());
    assertEquals("Commit index", lastAppliedToState, context.getCommitIndex());
    assertEquals("Recovered state size", 6, mockRaftActor.getState().size());
    mockRaftActor.waitForInitializeBehaviorComplete();
    assertEquals("getRaftState", RaftState.Follower, mockRaftActor.getRaftState());
    TEST_LOG.info("testRaftActorRecoveryWithPersistenceEnabled ending");
}

Example 28 with Snapshot

use of org.opendaylight.controller.cluster.raft.persisted.Snapshot in project controller by opendaylight.

the class RaftActorTest method testRestoreFromSnapshotWithRecoveredData.

@Test
public void testRestoreFromSnapshotWithRecoveredData() throws Exception {
    TEST_LOG.info("testRestoreFromSnapshotWithRecoveredData starting");
    String persistenceId = factory.generateActorId("test-actor-");
    DefaultConfigParamsImpl config = new DefaultConfigParamsImpl();
    config.setCustomRaftPolicyImplementationClass(DisableElectionsRaftPolicy.class.getName());
    List<MockPayload> state = Arrays.asList(new MockRaftActorContext.MockPayload("A"));
    Snapshot snapshot = Snapshot.create(ByteState.of(fromObject(state).toByteArray()), Arrays.<ReplicatedLogEntry>asList(), 5, 2, 5, 2, 2, "member-1", null);
    InMemoryJournal.addEntry(persistenceId, 1, new SimpleReplicatedLogEntry(0, 1, new MockRaftActorContext.MockPayload("B")));
    TestActorRef<MockRaftActor> raftActorRef = factory.createTestActor(MockRaftActor.builder().id(persistenceId).config(config).restoreFromSnapshot(snapshot).props().withDispatcher(Dispatchers.DefaultDispatcherId()), persistenceId);
    MockRaftActor mockRaftActor = raftActorRef.underlyingActor();
    mockRaftActor.waitForRecoveryComplete();
    Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
    verify(mockRaftActor.snapshotCohortDelegate, never()).applySnapshot(any(Snapshot.State.class));
    RaftActorContext context = mockRaftActor.getRaftActorContext();
    assertEquals("Journal log size", 1, context.getReplicatedLog().size());
    assertEquals("Last index", 0, context.getReplicatedLog().lastIndex());
    assertEquals("Last applied", -1, context.getLastApplied());
    assertEquals("Commit index", -1, context.getCommitIndex());
    assertEquals("Current term", 0, context.getTermInformation().getCurrentTerm());
    assertEquals("Voted for", null, context.getTermInformation().getVotedFor());
    TEST_LOG.info("testRestoreFromSnapshotWithRecoveredData ending");
}

Example 29 with Snapshot

use of org.opendaylight.controller.cluster.raft.persisted.Snapshot in project controller by opendaylight.

the class RecoveryIntegrationSingleNodeTest method testJournalReplayAfterSnapshotWithSingleNode.

@Test
public void testJournalReplayAfterSnapshotWithSingleNode() throws Exception {
    String persistenceId = factory.generateActorId("singleNode");
    TestActorRef<AbstractRaftActorIntegrationTest.TestRaftActor> singleNodeActorRef = newTestRaftActor(persistenceId, ImmutableMap.<String, String>builder().build(), leaderConfigParams);
    waitUntilLeader(singleNodeActorRef);
    ActorRef singleNodeCollectorActor = singleNodeActorRef.underlyingActor().collectorActor();
    final RaftActorContext singleNodeContext = singleNodeActorRef.underlyingActor().getRaftActorContext();
    InMemoryJournal.addWriteMessagesCompleteLatch(persistenceId, 6, ApplyJournalEntries.class);
    final MockRaftActorContext.MockPayload payload0 = sendPayloadData(singleNodeActorRef, "zero");
    final MockRaftActorContext.MockPayload payload1 = sendPayloadData(singleNodeActorRef, "one");
    final MockRaftActorContext.MockPayload payload2 = sendPayloadData(singleNodeActorRef, "two");
    MessageCollectorActor.expectMatching(singleNodeCollectorActor, ApplyJournalEntries.class, 3);
    // this should trigger a snapshot
    final MockRaftActorContext.MockPayload payload3 = sendPayloadData(singleNodeActorRef, "three");
    MessageCollectorActor.expectMatching(singleNodeCollectorActor, ApplyJournalEntries.class, 4);
    // add 2 more
    final MockRaftActorContext.MockPayload payload4 = sendPayloadData(singleNodeActorRef, "four");
    final MockRaftActorContext.MockPayload payload5 = sendPayloadData(singleNodeActorRef, "five");
    // Wait for snapshot complete.
    MessageCollectorActor.expectFirstMatching(singleNodeCollectorActor, SaveSnapshotSuccess.class);
    MessageCollectorActor.expectMatching(singleNodeCollectorActor, ApplyJournalEntries.class, 6);
    assertEquals("Last applied", 5, singleNodeContext.getLastApplied());
    assertEquals("Incorrect State after snapshot success is received ", Lists.newArrayList(payload0, payload1, payload2, payload3, payload4, payload5), singleNodeActorRef.underlyingActor().getState());
    InMemoryJournal.waitForWriteMessagesComplete(persistenceId);
    // we get 2 log entries (4 and 5 indexes) and 3 ApplyJournalEntries (for 3, 4, and 5 indexes)
    assertEquals(5, InMemoryJournal.get(persistenceId).size());
    List<Snapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(persistenceId, Snapshot.class);
    assertEquals(1, persistedSnapshots.size());
    List<Object> snapshottedState = MockRaftActor.fromState(persistedSnapshots.get(0).getState());
    assertEquals("Incorrect Snapshot", Lists.newArrayList(payload0, payload1, payload2, payload3), snapshottedState);
    // recovery logic starts
    killActor(singleNodeActorRef);
    singleNodeActorRef = newTestRaftActor(persistenceId, ImmutableMap.<String, String>builder().build(), leaderConfigParams);
    singleNodeActorRef.underlyingActor().waitForRecoveryComplete();
    assertEquals("Incorrect State after Recovery ", Lists.newArrayList(payload0, payload1, payload2, payload3, payload4, payload5), singleNodeActorRef.underlyingActor().getState());
}

Example 30 with Snapshot

use of org.opendaylight.controller.cluster.raft.persisted.Snapshot in project controller by opendaylight.

the class ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest method verifyInstallSnapshotToLaggingFollower.

/**
 * Resume the lagging follower 2 and verify it receives an install snapshot from the leader.
 */
private void verifyInstallSnapshotToLaggingFollower(long lastAppliedIndex, @Nullable ServerConfigurationPayload expServerConfig) throws Exception {
    testLog.info("verifyInstallSnapshotToLaggingFollower starting");
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    // Now stop dropping AppendEntries in follower 2.
    follower2Actor.underlyingActor().stopDropMessages(AppendEntries.class);
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, SaveSnapshotSuccess.class);
    // Verify the leader's persisted snapshot. The previous snapshot (currently) won't be deleted from
    // the snapshot store because the second snapshot was initiated by the follower install snapshot and
    // not because the batch count was reached so the persisted journal sequence number wasn't advanced
    // far enough to cause the previous snapshot to be deleted. This is because
    // RaftActor#trimPersistentData subtracts the snapshotBatchCount from the snapshot's sequence number.
    // This is OK - the next snapshot should delete it. In production, even if the system restarted
    // before another snapshot, they would both get applied which wouldn't hurt anything.
    List<Snapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(leaderId, Snapshot.class);
    Assert.assertTrue("Expected at least 1 persisted snapshots", persistedSnapshots.size() > 0);
    Snapshot persistedSnapshot = persistedSnapshots.get(persistedSnapshots.size() - 1);
    verifySnapshot("Persisted", persistedSnapshot, currentTerm, lastAppliedIndex, currentTerm, lastAppliedIndex);
    List<ReplicatedLogEntry> unAppliedEntry = persistedSnapshot.getUnAppliedEntries();
    assertEquals("Persisted Snapshot getUnAppliedEntries size", 0, unAppliedEntry.size());
    int snapshotSize = SerializationUtils.serialize(persistedSnapshot.getState()).length;
    final int expTotalChunks = snapshotSize / SNAPSHOT_CHUNK_SIZE + (snapshotSize % SNAPSHOT_CHUNK_SIZE > 0 ? 1 : 0);
    InstallSnapshot installSnapshot = MessageCollectorActor.expectFirstMatching(follower2CollectorActor, InstallSnapshot.class);
    assertEquals("InstallSnapshot getTerm", currentTerm, installSnapshot.getTerm());
    assertEquals("InstallSnapshot getLeaderId", leaderId, installSnapshot.getLeaderId());
    assertEquals("InstallSnapshot getChunkIndex", 1, installSnapshot.getChunkIndex());
    assertEquals("InstallSnapshot getTotalChunks", expTotalChunks, installSnapshot.getTotalChunks());
    assertEquals("InstallSnapshot getLastIncludedTerm", currentTerm, installSnapshot.getLastIncludedTerm());
    assertEquals("InstallSnapshot getLastIncludedIndex", lastAppliedIndex, installSnapshot.getLastIncludedIndex());
    // assertArrayEquals("InstallSnapshot getData", snapshot, installSnapshot.getData().toByteArray());
    List<InstallSnapshotReply> installSnapshotReplies = MessageCollectorActor.expectMatching(leaderCollectorActor, InstallSnapshotReply.class, expTotalChunks);
    int index = 1;
    for (InstallSnapshotReply installSnapshotReply : installSnapshotReplies) {
        assertEquals("InstallSnapshotReply getTerm", currentTerm, installSnapshotReply.getTerm());
        assertEquals("InstallSnapshotReply getChunkIndex", index++, installSnapshotReply.getChunkIndex());
        assertEquals("InstallSnapshotReply getFollowerId", follower2Id, installSnapshotReply.getFollowerId());
        assertEquals("InstallSnapshotReply isSuccess", true, installSnapshotReply.isSuccess());
    }
    // Verify follower 2 applies the snapshot.
    ApplySnapshot applySnapshot = MessageCollectorActor.expectFirstMatching(follower2CollectorActor, ApplySnapshot.class);
    verifySnapshot("Follower 2", applySnapshot.getSnapshot(), currentTerm, lastAppliedIndex, currentTerm, lastAppliedIndex);
    assertEquals("Persisted Snapshot getUnAppliedEntries size", 0, applySnapshot.getSnapshot().getUnAppliedEntries().size());
    // Wait for the snapshot to complete.
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, SaveSnapshotSuccess.class);
    // Ensure there's at least 1 more heartbeat.
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, AppendEntriesReply.class);
    // The leader should now have performed fake snapshots to advance the snapshot index and to trim
    // the log. In addition replicatedToAllIndex should've advanced.
    verifyLeadersTrimmedLog(lastAppliedIndex);
    if (expServerConfig != null) {
        Set<ServerInfo> expServerInfo = new HashSet<>(expServerConfig.getServerConfig());
        assertEquals("Leader snapshot server config", expServerInfo, new HashSet<>(persistedSnapshot.getServerConfiguration().getServerConfig()));
        assertEquals("Follower 2 snapshot server config", expServerInfo, new HashSet<>(applySnapshot.getSnapshot().getServerConfiguration().getServerConfig()));
        ServerConfigurationPayload follower2ServerConfig = follower2Context.getPeerServerInfo(true);
        assertNotNull("Follower 2 server config is null", follower2ServerConfig);
        assertEquals("Follower 2 server config", expServerInfo, new HashSet<>(follower2ServerConfig.getServerConfig()));
    }
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.clearMessages(follower1CollectorActor);
    MessageCollectorActor.clearMessages(follower2CollectorActor);
    testLog.info("verifyInstallSnapshotToLaggingFollower complete");
}