Examples with InstallSnapshot org.opendaylight.controller.cluster.raft.messages.InstallSnapshot used on opensource projects

Example 11 with InstallSnapshot

use of org.opendaylight.controller.cluster.raft.messages.InstallSnapshot in project controller by opendaylight.

the class RaftActorServerConfigurationSupportTest method testAddServerWithOperationInProgress.

@Test
public void testAddServerWithOperationInProgress() throws Exception {
    LOG.info("testAddServerWithOperationInProgress starting");
    setupNewFollower();
    RaftActorContext initialActorContext = new MockRaftActorContext();
    TestActorRef<MockLeaderRaftActor> leaderActor = actorFactory.createTestActor(MockLeaderRaftActor.props(ImmutableMap.<String, String>of(), initialActorContext).withDispatcher(Dispatchers.DefaultDispatcherId()), actorFactory.generateActorId(LEADER_ID));
    MockLeaderRaftActor leaderRaftActor = leaderActor.underlyingActor();
    final RaftActorContext leaderActorContext = leaderRaftActor.getRaftActorContext();
    final ActorRef leaderCollectorActor = newLeaderCollectorActor(leaderRaftActor);
    RaftActorContext follower2ActorContext = newFollowerContext(NEW_SERVER_ID2, followerActor);
    Follower newFollower2 = new Follower(follower2ActorContext);
    followerActor.underlyingActor().setBehavior(newFollower2);
    MockNewFollowerRaftActor newFollowerRaftActorInstance = newFollowerRaftActor.underlyingActor();
    newFollowerRaftActorInstance.setDropMessageOfType(InstallSnapshot.class);
    leaderActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), true), testKit.getRef());
    // Wait for leader's install snapshot and capture it
    InstallSnapshot installSnapshot = expectFirstMatching(newFollowerCollectorActor, InstallSnapshot.class);
    // Send a second AddServer - should get queued
    TestKit testKit2 = new TestKit(getSystem());
    leaderActor.tell(new AddServer(NEW_SERVER_ID2, followerActor.path().toString(), false), testKit2.getRef());
    // Continue the first AddServer
    newFollowerRaftActorInstance.setDropMessageOfType(null);
    newFollowerRaftActor.tell(installSnapshot, leaderActor);
    // Verify both complete successfully
    AddServerReply addServerReply = testKit.expectMsgClass(testKit.duration("5 seconds"), AddServerReply.class);
    assertEquals("getStatus", ServerChangeStatus.OK, addServerReply.getStatus());
    addServerReply = testKit2.expectMsgClass(testKit2.duration("5 seconds"), AddServerReply.class);
    assertEquals("getStatus", ServerChangeStatus.OK, addServerReply.getStatus());
    // Verify ServerConfigurationPayload entries in leader's log
    expectMatching(leaderCollectorActor, ApplyState.class, 2);
    assertEquals("Leader journal last index", 1, leaderActorContext.getReplicatedLog().lastIndex());
    assertEquals("Leader commit index", 1, leaderActorContext.getCommitIndex());
    assertEquals("Leader last applied index", 1, leaderActorContext.getLastApplied());
    verifyServerConfigurationPayloadEntry(leaderActorContext.getReplicatedLog(), votingServer(LEADER_ID), votingServer(NEW_SERVER_ID), nonVotingServer(NEW_SERVER_ID2));
    // Verify ServerConfigurationPayload entry in the new follower
    expectMatching(newFollowerCollectorActor, ApplyState.class, 2);
    assertEquals("New follower peers", Sets.newHashSet(LEADER_ID, NEW_SERVER_ID2), newFollowerActorContext.getPeerIds());
    LOG.info("testAddServerWithOperationInProgress ending");
}

Example 12 with InstallSnapshot

use of org.opendaylight.controller.cluster.raft.messages.InstallSnapshot in project controller by opendaylight.

the class FollowerTest method testReceivingAppendEntriesDuringInstallSnapshot.

/**
 * Verify that when an AppendEntries is sent to a follower during a snapshot install
 * the Follower short-circuits the processing of the AppendEntries message.
 */
@Test
public void testReceivingAppendEntriesDuringInstallSnapshot() {
    logStart("testReceivingAppendEntriesDuringInstallSnapshot");
    MockRaftActorContext context = createActorContext();
    follower = createBehavior(context);
    ByteString bsSnapshot = createSnapshot();
    int snapshotLength = bsSnapshot.size();
    int chunkSize = 50;
    int totalChunks = snapshotLength / chunkSize + (snapshotLength % chunkSize > 0 ? 1 : 0);
    int lastIncludedIndex = 1;
    // Check that snapshot installation is not in progress
    assertNull(follower.getSnapshotTracker());
    // Make sure that we have more than 1 chunk to send
    assertTrue(totalChunks > 1);
    // Send an install snapshot with the first chunk to start the process of installing a snapshot
    byte[] chunkData = getNextChunk(bsSnapshot, 0, chunkSize);
    follower.handleMessage(leaderActor, new InstallSnapshot(1, "leader", lastIncludedIndex, 1, chunkData, 1, totalChunks));
    // Check if snapshot installation is in progress now
    assertNotNull(follower.getSnapshotTracker());
    // Send an append entry
    AppendEntries appendEntries = new AppendEntries(1, "leader", 1, 1, Arrays.asList(newReplicatedLogEntry(2, 1, "3")), 2, -1, (short) 1);
    follower.handleMessage(leaderActor, appendEntries);
    AppendEntriesReply reply = MessageCollectorActor.expectFirstMatching(leaderActor, AppendEntriesReply.class);
    assertEquals("isSuccess", true, reply.isSuccess());
    assertEquals("getLogLastIndex", context.getReplicatedLog().lastIndex(), reply.getLogLastIndex());
    assertEquals("getLogLastTerm", context.getReplicatedLog().lastTerm(), reply.getLogLastTerm());
    assertEquals("getTerm", context.getTermInformation().getCurrentTerm(), reply.getTerm());
    assertNotNull(follower.getSnapshotTracker());
}

Example 13 with InstallSnapshot

use of org.opendaylight.controller.cluster.raft.messages.InstallSnapshot in project controller by opendaylight.

the class LeaderTest method testHandleSnapshotSendsPreviousChunksHashCodeWhenSendingNextChunk.

@Test
public void testHandleSnapshotSendsPreviousChunksHashCodeWhenSendingNextChunk() throws Exception {
    logStart("testHandleSnapshotSendsPreviousChunksHashCodeWhenSendingNextChunk");
    MockRaftActorContext actorContext = createActorContextWithFollower();
    final int commitIndex = 3;
    final int snapshotIndex = 2;
    final int snapshotTerm = 1;
    final int currentTerm = 2;
    actorContext.setConfigParams(new DefaultConfigParamsImpl() {

        @Override
        public int getSnapshotChunkSize() {
            return 50;
        }
    });
    actorContext.setCommitIndex(commitIndex);
    leader = new Leader(actorContext);
    leader.getFollower(FOLLOWER_ID).setMatchIndex(-1);
    leader.getFollower(FOLLOWER_ID).setNextIndex(0);
    Map<String, String> leadersSnapshot = new HashMap<>();
    leadersSnapshot.put("1", "A");
    leadersSnapshot.put("2", "B");
    leadersSnapshot.put("3", "C");
    // set the snapshot variables in replicatedlog
    actorContext.getReplicatedLog().setSnapshotIndex(snapshotIndex);
    actorContext.getReplicatedLog().setSnapshotTerm(snapshotTerm);
    actorContext.getTermInformation().update(currentTerm, leaderActor.path().toString());
    ByteString bs = toByteString(leadersSnapshot);
    Snapshot snapshot = Snapshot.create(ByteState.of(bs.toByteArray()), Collections.<ReplicatedLogEntry>emptyList(), commitIndex, snapshotTerm, commitIndex, snapshotTerm, -1, null, null);
    leader.handleMessage(leaderActor, new SendInstallSnapshot(snapshot, ByteSource.wrap(bs.toByteArray())));
    InstallSnapshot installSnapshot = MessageCollectorActor.expectFirstMatching(followerActor, InstallSnapshot.class);
    assertEquals(1, installSnapshot.getChunkIndex());
    assertEquals(3, installSnapshot.getTotalChunks());
    assertEquals(LeaderInstallSnapshotState.INITIAL_LAST_CHUNK_HASH_CODE, installSnapshot.getLastChunkHashCode().get().intValue());
    final int hashCode = Arrays.hashCode(installSnapshot.getData());
    followerActor.underlyingActor().clear();
    leader.handleMessage(followerActor, new InstallSnapshotReply(installSnapshot.getTerm(), FOLLOWER_ID, 1, true));
    installSnapshot = MessageCollectorActor.expectFirstMatching(followerActor, InstallSnapshot.class);
    assertEquals(2, installSnapshot.getChunkIndex());
    assertEquals(3, installSnapshot.getTotalChunks());
    assertEquals(hashCode, installSnapshot.getLastChunkHashCode().get().intValue());
}

Example 14 with InstallSnapshot

use of org.opendaylight.controller.cluster.raft.messages.InstallSnapshot in project controller by opendaylight.

the class ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest method testReplicationsWithLaggingFollowerCaughtUpViaAppendEntries.

/**
 * Send 2 payload instances with follower 2 lagging then resume the follower and verifies it gets
 * caught up via AppendEntries.
 */
@Test
public void testReplicationsWithLaggingFollowerCaughtUpViaAppendEntries() throws Exception {
    testLog.info("testReplicationsWithLaggingFollowerCaughtUpViaAppendEntries starting: sending 2 new payloads");
    setup();
    // Simulate lagging by dropping AppendEntries messages in follower 2.
    follower2Actor.underlyingActor().startDropMessages(AppendEntries.class);
    // Send the payloads.
    MockPayload payload0 = sendPayloadData(leaderActor, "zero");
    MockPayload payload1 = sendPayloadData(leaderActor, "one");
    // Verify the leader got consensus and applies each log entry even though follower 2 didn't respond.
    List<ApplyState> applyStates = MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 2);
    verifyApplyState(applyStates.get(0), leaderCollectorActor, payload0.toString(), currentTerm, 0, payload0);
    verifyApplyState(applyStates.get(1), leaderCollectorActor, payload1.toString(), currentTerm, 1, payload1);
    // Verify follower 1 applies each log entry.
    applyStates = MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, 2);
    verifyApplyState(applyStates.get(0), null, null, currentTerm, 0, payload0);
    verifyApplyState(applyStates.get(1), null, null, currentTerm, 1, payload1);
    // Ensure there's at least 1 more heartbeat.
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, AppendEntriesReply.class);
    // The leader should not have performed fake snapshots to trim the log because the entries have not
    // been replicated to follower 2.
    assertEquals("Leader snapshot term", -1, leaderContext.getReplicatedLog().getSnapshotTerm());
    assertEquals("Leader snapshot index", -1, leaderContext.getReplicatedLog().getSnapshotIndex());
    assertEquals("Leader journal log size", 2, leaderContext.getReplicatedLog().size());
    assertEquals("Leader journal last index", 1, leaderContext.getReplicatedLog().lastIndex());
    assertEquals("Leader commit index", 1, leaderContext.getCommitIndex());
    assertEquals("Leader last applied", 1, leaderContext.getLastApplied());
    assertEquals("Leader replicatedToAllIndex", -1, leader.getReplicatedToAllIndex());
    testLog.info("testReplicationsWithLaggingFollowerCaughtUpViaAppendEntries: new entries applied - resuming follower {}", follower2Id);
    // Now stop dropping AppendEntries in follower 2.
    follower2Actor.underlyingActor().stopDropMessages(AppendEntries.class);
    // Verify follower 2 applies each log entry.
    applyStates = MessageCollectorActor.expectMatching(follower2CollectorActor, ApplyState.class, 2);
    verifyApplyState(applyStates.get(0), null, null, currentTerm, 0, payload0);
    verifyApplyState(applyStates.get(1), null, null, currentTerm, 1, payload1);
    // Ensure there's at least 1 more heartbeat.
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, AppendEntriesReply.class);
    // The leader should now have performed fake snapshots to trim the log.
    verifyLeadersTrimmedLog(1);
    // Even though follower 2 lagged behind, the leader should not have tried to install a snapshot
    // to catch it up because no snapshotting was done so the follower's next index was present in the log.
    InstallSnapshot installSnapshot = MessageCollectorActor.getFirstMatching(follower2CollectorActor, InstallSnapshot.class);
    Assert.assertNull("Follower 2 received unexpected InstallSnapshot", installSnapshot);
    testLog.info("testReplicationsWithLaggingFollowerCaughtUpViaAppendEntries complete");
}

Example 15 with InstallSnapshot

use of org.opendaylight.controller.cluster.raft.messages.InstallSnapshot in project controller by opendaylight.

the class ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest method verifyInstallSnapshotToLaggingFollower.

/**
 * Resume the lagging follower 2 and verify it receives an install snapshot from the leader.
 */
private void verifyInstallSnapshotToLaggingFollower(long lastAppliedIndex, @Nullable ServerConfigurationPayload expServerConfig) throws Exception {
    testLog.info("verifyInstallSnapshotToLaggingFollower starting");
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    // Now stop dropping AppendEntries in follower 2.
    follower2Actor.underlyingActor().stopDropMessages(AppendEntries.class);
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, SaveSnapshotSuccess.class);
    // Verify the leader's persisted snapshot. The previous snapshot (currently) won't be deleted from
    // the snapshot store because the second snapshot was initiated by the follower install snapshot and
    // not because the batch count was reached so the persisted journal sequence number wasn't advanced
    // far enough to cause the previous snapshot to be deleted. This is because
    // RaftActor#trimPersistentData subtracts the snapshotBatchCount from the snapshot's sequence number.
    // This is OK - the next snapshot should delete it. In production, even if the system restarted
    // before another snapshot, they would both get applied which wouldn't hurt anything.
    List<Snapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(leaderId, Snapshot.class);
    Assert.assertTrue("Expected at least 1 persisted snapshots", persistedSnapshots.size() > 0);
    Snapshot persistedSnapshot = persistedSnapshots.get(persistedSnapshots.size() - 1);
    verifySnapshot("Persisted", persistedSnapshot, currentTerm, lastAppliedIndex, currentTerm, lastAppliedIndex);
    List<ReplicatedLogEntry> unAppliedEntry = persistedSnapshot.getUnAppliedEntries();
    assertEquals("Persisted Snapshot getUnAppliedEntries size", 0, unAppliedEntry.size());
    int snapshotSize = SerializationUtils.serialize(persistedSnapshot.getState()).length;
    final int expTotalChunks = snapshotSize / SNAPSHOT_CHUNK_SIZE + (snapshotSize % SNAPSHOT_CHUNK_SIZE > 0 ? 1 : 0);
    InstallSnapshot installSnapshot = MessageCollectorActor.expectFirstMatching(follower2CollectorActor, InstallSnapshot.class);
    assertEquals("InstallSnapshot getTerm", currentTerm, installSnapshot.getTerm());
    assertEquals("InstallSnapshot getLeaderId", leaderId, installSnapshot.getLeaderId());
    assertEquals("InstallSnapshot getChunkIndex", 1, installSnapshot.getChunkIndex());
    assertEquals("InstallSnapshot getTotalChunks", expTotalChunks, installSnapshot.getTotalChunks());
    assertEquals("InstallSnapshot getLastIncludedTerm", currentTerm, installSnapshot.getLastIncludedTerm());
    assertEquals("InstallSnapshot getLastIncludedIndex", lastAppliedIndex, installSnapshot.getLastIncludedIndex());
    // assertArrayEquals("InstallSnapshot getData", snapshot, installSnapshot.getData().toByteArray());
    List<InstallSnapshotReply> installSnapshotReplies = MessageCollectorActor.expectMatching(leaderCollectorActor, InstallSnapshotReply.class, expTotalChunks);
    int index = 1;
    for (InstallSnapshotReply installSnapshotReply : installSnapshotReplies) {
        assertEquals("InstallSnapshotReply getTerm", currentTerm, installSnapshotReply.getTerm());
        assertEquals("InstallSnapshotReply getChunkIndex", index++, installSnapshotReply.getChunkIndex());
        assertEquals("InstallSnapshotReply getFollowerId", follower2Id, installSnapshotReply.getFollowerId());
        assertEquals("InstallSnapshotReply isSuccess", true, installSnapshotReply.isSuccess());
    }
    // Verify follower 2 applies the snapshot.
    ApplySnapshot applySnapshot = MessageCollectorActor.expectFirstMatching(follower2CollectorActor, ApplySnapshot.class);
    verifySnapshot("Follower 2", applySnapshot.getSnapshot(), currentTerm, lastAppliedIndex, currentTerm, lastAppliedIndex);
    assertEquals("Persisted Snapshot getUnAppliedEntries size", 0, applySnapshot.getSnapshot().getUnAppliedEntries().size());
    // Wait for the snapshot to complete.
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, SaveSnapshotSuccess.class);
    // Ensure there's at least 1 more heartbeat.
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, AppendEntriesReply.class);
    // The leader should now have performed fake snapshots to advance the snapshot index and to trim
    // the log. In addition replicatedToAllIndex should've advanced.
    verifyLeadersTrimmedLog(lastAppliedIndex);
    if (expServerConfig != null) {
        Set<ServerInfo> expServerInfo = new HashSet<>(expServerConfig.getServerConfig());
        assertEquals("Leader snapshot server config", expServerInfo, new HashSet<>(persistedSnapshot.getServerConfiguration().getServerConfig()));
        assertEquals("Follower 2 snapshot server config", expServerInfo, new HashSet<>(applySnapshot.getSnapshot().getServerConfiguration().getServerConfig()));
        ServerConfigurationPayload follower2ServerConfig = follower2Context.getPeerServerInfo(true);
        assertNotNull("Follower 2 server config is null", follower2ServerConfig);
        assertEquals("Follower 2 server config", expServerInfo, new HashSet<>(follower2ServerConfig.getServerConfig()));
    }
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.clearMessages(follower1CollectorActor);
    MessageCollectorActor.clearMessages(follower2CollectorActor);
    testLog.info("verifyInstallSnapshotToLaggingFollower complete");
}