Examples with MockPayload org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockPayload used on opensource projects

Example 31 with MockPayload

use of org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockPayload in project controller by opendaylight.

the class ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest method testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries.

/**
 * Send payloads to trigger a leader snapshot due to snapshotBatchCount reached with follower 2
 * lagging but not enough for the leader to trim its log from the last applied index. Follower 2's log
 * will be behind by several entries and, when it is resumed, it should be caught up via AppendEntries
 * sent by the leader.
 */
@Test
public void testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries() throws Exception {
    testLog.info("testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries starting");
    setup();
    sendInitialPayloadsReplicatedToAllFollowers("zero", "one");
    // Configure follower 2 to drop messages and lag.
    follower2Actor.underlyingActor().startDropMessages(AppendEntries.class);
    // Send the first payload and verify it gets applied by the leader and follower 1.
    MockPayload payload2 = sendPayloadData(leaderActor, "two");
    ApplyState applyState = MessageCollectorActor.expectFirstMatching(leaderCollectorActor, ApplyState.class);
    verifyApplyState(applyState, leaderCollectorActor, payload2.toString(), currentTerm, 2, payload2);
    applyState = MessageCollectorActor.expectFirstMatching(follower1CollectorActor, ApplyState.class);
    verifyApplyState(applyState, null, null, currentTerm, 2, payload2);
    expSnapshotState.add(payload2);
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.clearMessages(follower1CollectorActor);
    // Send another payload - this should cause a snapshot due to snapshotBatchCount reached.
    MockPayload payload3 = sendPayloadData(leaderActor, "three");
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, SaveSnapshotSuccess.class);
    testLog.info("testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries: sending 2 more payloads");
    // Send 2 more payloads - not enough to trigger another snapshot.
    MockPayload payload4 = sendPayloadData(leaderActor, "four");
    MockPayload payload5 = sendPayloadData(leaderActor, "five");
    // Verify the leader got consensus and applies each log entry even though follower 2 didn't respond.
    List<ApplyState> applyStates = MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 3);
    verifyApplyState(applyStates.get(0), leaderCollectorActor, payload3.toString(), currentTerm, 3, payload3);
    verifyApplyState(applyStates.get(1), leaderCollectorActor, payload4.toString(), currentTerm, 4, payload4);
    verifyApplyState(applyStates.get(2), leaderCollectorActor, payload5.toString(), currentTerm, 5, payload5);
    // Verify follower 1 applies each log entry.
    applyStates = MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, 3);
    verifyApplyState(applyStates.get(0), null, null, currentTerm, 3, payload3);
    verifyApplyState(applyStates.get(1), null, null, currentTerm, 4, payload4);
    verifyApplyState(applyStates.get(2), null, null, currentTerm, 5, payload5);
    // The snapshot should have caused the leader to advanced the snapshot index to the
    // last previously applied index (1) that was replicated to all followers at the time of capture.
    // Note: since the log size (3) did not exceed the snapshot batch count (4), the leader should not
    // have trimmed the log to the last index actually applied (5).
    assertEquals("Leader snapshot term", currentTerm, leaderContext.getReplicatedLog().getSnapshotTerm());
    assertEquals("Leader snapshot index", 1, leaderContext.getReplicatedLog().getSnapshotIndex());
    assertEquals("Leader journal log size", 4, leaderContext.getReplicatedLog().size());
    assertEquals("Leader journal last index", 5, leaderContext.getReplicatedLog().lastIndex());
    assertEquals("Leader commit index", 5, leaderContext.getCommitIndex());
    assertEquals("Leader last applied", 5, leaderContext.getLastApplied());
    assertEquals("Leader replicatedToAllIndex", 1, leader.getReplicatedToAllIndex());
    // Now stop dropping AppendEntries in follower 2.
    follower2Actor.underlyingActor().stopDropMessages(AppendEntries.class);
    // Verify follower 2 applies each log entry. The leader should not install a snapshot b/c
    // follower 2's next index (3) is still present in the log.
    applyStates = MessageCollectorActor.expectMatching(follower2CollectorActor, ApplyState.class, 4);
    verifyApplyState(applyStates.get(0), null, null, currentTerm, 2, payload2);
    verifyApplyState(applyStates.get(1), null, null, currentTerm, 3, payload3);
    verifyApplyState(applyStates.get(2), null, null, currentTerm, 4, payload4);
    verifyApplyState(applyStates.get(3), null, null, currentTerm, 5, payload5);
    // Verify the leader did not try to install a snapshot to catch up follower 2.
    InstallSnapshot installSnapshot = MessageCollectorActor.getFirstMatching(follower2CollectorActor, InstallSnapshot.class);
    Assert.assertNull("Follower 2 received unexpected InstallSnapshot", installSnapshot);
    // Ensure there's at least 1 more heartbeat.
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, AppendEntriesReply.class);
    // The leader should now have performed fake snapshots to advance the snapshot index and to trim
    // the log. In addition replicatedToAllIndex should've advanced.
    verifyLeadersTrimmedLog(5);
    // Verify the leader's persisted snapshot.
    List<Snapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(leaderId, Snapshot.class);
    assertEquals("Persisted snapshots size", 1, persistedSnapshots.size());
    verifySnapshot("Persisted", persistedSnapshots.get(0), currentTerm, 2, currentTerm, 3);
    List<ReplicatedLogEntry> unAppliedEntry = persistedSnapshots.get(0).getUnAppliedEntries();
    assertEquals("Persisted Snapshot getUnAppliedEntries size", 1, unAppliedEntry.size());
    verifyReplicatedLogEntry(unAppliedEntry.get(0), currentTerm, 3, payload3);
    // Verify follower 1's log and snapshot indexes.
    MessageCollectorActor.clearMessages(follower1CollectorActor);
    MessageCollectorActor.expectFirstMatching(follower1CollectorActor, AppendEntries.class);
    verifyFollowersTrimmedLog(1, follower1Actor, 5);
    // Verify follower 2's log and snapshot indexes.
    MessageCollectorActor.clearMessages(follower2CollectorActor);
    MessageCollectorActor.expectFirstMatching(follower2CollectorActor, AppendEntries.class);
    verifyFollowersTrimmedLog(2, follower2Actor, 5);
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.clearMessages(follower1CollectorActor);
    MessageCollectorActor.clearMessages(follower2CollectorActor);
    expSnapshotState.add(payload3);
    expSnapshotState.add(payload4);
    expSnapshotState.add(payload5);
    testLog.info("testLeaderSnapshotWithLaggingFollowerCaughtUpViaAppendEntries complete");
}

Example 32 with MockPayload

use of org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockPayload in project controller by opendaylight.

the class AbstractReplicatedLogImplTest method testGetFromWithMax.

@Test
public void testGetFromWithMax() {
    List<ReplicatedLogEntry> from = replicatedLogImpl.getFrom(0, 1, ReplicatedLog.NO_MAX_SIZE);
    Assert.assertEquals(1, from.size());
    Assert.assertEquals("A", from.get(0).getData().toString());
    from = replicatedLogImpl.getFrom(0, 20, ReplicatedLog.NO_MAX_SIZE);
    Assert.assertEquals(4, from.size());
    Assert.assertEquals("A", from.get(0).getData().toString());
    Assert.assertEquals("D", from.get(3).getData().toString());
    from = replicatedLogImpl.getFrom(1, 2, ReplicatedLog.NO_MAX_SIZE);
    Assert.assertEquals(2, from.size());
    Assert.assertEquals("B", from.get(0).getData().toString());
    Assert.assertEquals("C", from.get(1).getData().toString());
    from = replicatedLogImpl.getFrom(1, 3, 2);
    Assert.assertEquals(2, from.size());
    Assert.assertEquals("B", from.get(0).getData().toString());
    Assert.assertEquals("C", from.get(1).getData().toString());
    from = replicatedLogImpl.getFrom(1, 3, 3);
    Assert.assertEquals(3, from.size());
    Assert.assertEquals("B", from.get(0).getData().toString());
    Assert.assertEquals("C", from.get(1).getData().toString());
    Assert.assertEquals("D", from.get(2).getData().toString());
    from = replicatedLogImpl.getFrom(1, 2, 3);
    Assert.assertEquals(2, from.size());
    Assert.assertEquals("B", from.get(0).getData().toString());
    Assert.assertEquals("C", from.get(1).getData().toString());
    replicatedLogImpl.append(new SimpleReplicatedLogEntry(4, 2, new MockPayload("12345")));
    from = replicatedLogImpl.getFrom(4, 2, 2);
    Assert.assertEquals(1, from.size());
    Assert.assertEquals("12345", from.get(0).getData().toString());
}

Example 33 with MockPayload

use of org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockPayload in project controller by opendaylight.

the class AbstractReplicatedLogImplTest method testIsPresent.

@Test
public void testIsPresent() {
    assertTrue(replicatedLogImpl.isPresent(0));
    assertTrue(replicatedLogImpl.isPresent(1));
    assertTrue(replicatedLogImpl.isPresent(2));
    assertTrue(replicatedLogImpl.isPresent(3));
    replicatedLogImpl.append(new SimpleReplicatedLogEntry(4, 2, new MockPayload("D")));
    // snapshot on 3
    replicatedLogImpl.snapshotPreCommit(3, 2);
    replicatedLogImpl.snapshotCommit();
    assertFalse(replicatedLogImpl.isPresent(0));
    assertFalse(replicatedLogImpl.isPresent(1));
    assertFalse(replicatedLogImpl.isPresent(2));
    assertFalse(replicatedLogImpl.isPresent(3));
    assertTrue(replicatedLogImpl.isPresent(4));
    // snapshot on 4
    replicatedLogImpl.snapshotPreCommit(4, 2);
    replicatedLogImpl.snapshotCommit();
    assertFalse(replicatedLogImpl.isPresent(4));
    replicatedLogImpl.append(new SimpleReplicatedLogEntry(5, 2, new MockPayload("D")));
    assertTrue(replicatedLogImpl.isPresent(5));
}

Example 34 with MockPayload

use of org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockPayload in project controller by opendaylight.

the class AbstractReplicatedLogImplTest method testIndexOperations.

@Test
public void testIndexOperations() {
    // check if the values returned are correct, with snapshotIndex = -1
    assertEquals("B", replicatedLogImpl.get(1).getData().toString());
    assertEquals("D", replicatedLogImpl.last().getData().toString());
    assertEquals(3, replicatedLogImpl.lastIndex());
    assertEquals(2, replicatedLogImpl.lastTerm());
    assertEquals(2, replicatedLogImpl.getFrom(2).size());
    assertEquals(4, replicatedLogImpl.size());
    assertTrue(replicatedLogImpl.isPresent(2));
    assertFalse(replicatedLogImpl.isPresent(4));
    assertFalse(replicatedLogImpl.isInSnapshot(2));
    // now create a snapshot of 3 entries, with 1 unapplied entry left in the log
    // It removes the entries which have made it to snapshot
    // and updates the snapshot index and term
    takeSnapshot(3);
    // check the values after the snapshot.
    // each index value passed in the test is the logical index (log entry index)
    // which gets mapped to the list's physical index
    assertEquals("D", replicatedLogImpl.get(3).getData().toString());
    assertEquals("D", replicatedLogImpl.last().getData().toString());
    assertNull(replicatedLogImpl.get(1));
    assertEquals(3, replicatedLogImpl.lastIndex());
    assertEquals(2, replicatedLogImpl.lastTerm());
    assertEquals(0, replicatedLogImpl.getFrom(2).size());
    assertEquals(1, replicatedLogImpl.size());
    assertFalse(replicatedLogImpl.isPresent(2));
    assertTrue(replicatedLogImpl.isPresent(3));
    assertFalse(replicatedLogImpl.isPresent(4));
    assertTrue(replicatedLogImpl.isInSnapshot(2));
    // append few more entries
    replicatedLogImpl.append(new SimpleReplicatedLogEntry(4, 2, new MockPayload("E")));
    replicatedLogImpl.append(new SimpleReplicatedLogEntry(5, 2, new MockPayload("F")));
    replicatedLogImpl.append(new SimpleReplicatedLogEntry(6, 3, new MockPayload("G")));
    replicatedLogImpl.append(new SimpleReplicatedLogEntry(7, 3, new MockPayload("H")));
    // check their values as well
    assertEquals(5, replicatedLogImpl.size());
    assertEquals("D", replicatedLogImpl.get(3).getData().toString());
    assertEquals("E", replicatedLogImpl.get(4).getData().toString());
    assertEquals("H", replicatedLogImpl.last().getData().toString());
    assertEquals(3, replicatedLogImpl.lastTerm());
    assertEquals(7, replicatedLogImpl.lastIndex());
    assertTrue(replicatedLogImpl.isPresent(7));
    assertFalse(replicatedLogImpl.isInSnapshot(7));
    assertEquals(1, replicatedLogImpl.getFrom(7).size());
    assertEquals(2, replicatedLogImpl.getFrom(6).size());
    // take a second snapshot with 5 entries with 0 unapplied entries left in the log
    takeSnapshot(5);
    assertEquals(0, replicatedLogImpl.size());
    assertNull(replicatedLogImpl.last());
    assertNull(replicatedLogImpl.get(7));
    assertNull(replicatedLogImpl.get(1));
    assertFalse(replicatedLogImpl.isPresent(7));
    assertTrue(replicatedLogImpl.isInSnapshot(7));
    assertEquals(0, replicatedLogImpl.getFrom(7).size());
    assertEquals(0, replicatedLogImpl.getFrom(6).size());
}

Example 35 with MockPayload

use of org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockPayload in project controller by opendaylight.

the class IsolationScenarioTest method testLeaderIsolationWithAllPriorEntriesCommitted.

/**
 * Isolates the leader after all initial payload entries have been committed and applied on all nodes. While
 * isolated, the majority partition elects a new leader and both sides of the partition attempt to commit one entry
 * independently. After isolation is removed, the entry will conflict and both sides should reconcile their logs
 * appropriately.
 */
@Test
public void testLeaderIsolationWithAllPriorEntriesCommitted() throws Exception {
    testLog.info("testLeaderIsolationWithAllPriorEntriesCommitted starting");
    createRaftActors();
    // Send an initial payloads and verify replication.
    final MockPayload payload0 = sendPayloadData(leaderActor, "zero");
    final MockPayload payload1 = sendPayloadData(leaderActor, "one");
    verifyApplyJournalEntries(leaderCollectorActor, 1);
    verifyApplyJournalEntries(follower1CollectorActor, 1);
    verifyApplyJournalEntries(follower2CollectorActor, 1);
    isolateLeader();
    // Send a payload to the isolated leader so it has an uncommitted log entry with index 2.
    testLog.info("Sending payload to isolated leader");
    final MockPayload isolatedLeaderPayload2 = sendPayloadData(leaderActor, "two");
    // Wait for the isolated leader to send AppendEntries to follower1 with the entry at index 2. Note the message
    // is collected but not forwarded to the follower RaftActor.
    AppendEntries appendEntries = expectFirstMatching(follower1CollectorActor, AppendEntries.class);
    assertEquals("getTerm", currentTerm, appendEntries.getTerm());
    assertEquals("getLeaderId", leaderId, appendEntries.getLeaderId());
    assertEquals("getEntries().size()", 1, appendEntries.getEntries().size());
    verifyReplicatedLogEntry(appendEntries.getEntries().get(0), currentTerm, 2, isolatedLeaderPayload2);
    // The leader should transition to IsolatedLeader.
    expectFirstMatching(leaderNotifierActor, RoleChanged.class, rc -> rc.getNewRole().equals(RaftState.IsolatedLeader.name()));
    forceElectionOnFollower1();
    // Send a payload to the new leader follower1 with index 2 and verify it's replicated to follower2
    // and committed.
    testLog.info("Sending payload to new leader");
    final MockPayload newLeaderPayload2 = sendPayloadData(follower1Actor, "two-new");
    verifyApplyJournalEntries(follower1CollectorActor, 2);
    verifyApplyJournalEntries(follower2CollectorActor, 2);
    assertEquals("Follower 1 journal last term", currentTerm, follower1Context.getReplicatedLog().lastTerm());
    assertEquals("Follower 1 journal last index", 2, follower1Context.getReplicatedLog().lastIndex());
    assertEquals("Follower 1 commit index", 2, follower1Context.getCommitIndex());
    verifyReplicatedLogEntry(follower1Context.getReplicatedLog().get(2), currentTerm, 2, newLeaderPayload2);
    assertEquals("Follower 1 state", Lists.newArrayList(payload0, payload1, newLeaderPayload2), follower1Actor.underlyingActor().getState());
    removeIsolation();
    // Previous leader should switch to follower b/c it will receive either an AppendEntries or AppendEntriesReply
    // with a higher term.
    expectFirstMatching(leaderNotifierActor, RoleChanged.class, rc -> rc.getNewRole().equals(RaftState.Follower.name()));
    // The previous leader has a conflicting log entry at index 2 with a different term which should get
    // replaced by the new leader's index 1 entry.
    verifyApplyJournalEntries(leaderCollectorActor, 2);
    assertEquals("Prior leader journal last term", currentTerm, leaderContext.getReplicatedLog().lastTerm());
    assertEquals("Prior leader journal last index", 2, leaderContext.getReplicatedLog().lastIndex());
    assertEquals("Prior leader commit index", 2, leaderContext.getCommitIndex());
    verifyReplicatedLogEntry(leaderContext.getReplicatedLog().get(2), currentTerm, 2, newLeaderPayload2);
    assertEquals("Prior leader state", Lists.newArrayList(payload0, payload1, newLeaderPayload2), leaderActor.underlyingActor().getState());
    testLog.info("testLeaderIsolationWithAllPriorEntriesCommitted ending");
}