Examples with MockPayload org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockPayload used on opensource projects

Example 11 with MockPayload

use of org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockPayload in project controller by opendaylight.

the class ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest method testLeaderSnapshotTriggeredByMemoryThresholdExceededWithLaggingFollower.

/**
 * Send payloads with follower 2 lagging with the last payload having a large enough size to trigger a
 * leader snapshot such that the leader trims its log from the last applied index.. Follower 2's log will
 * be behind by several entries and, when it is resumed, it should be caught up via a snapshot installed
 * by the leader.
 */
@Test
public void testLeaderSnapshotTriggeredByMemoryThresholdExceededWithLaggingFollower() throws Exception {
    testLog.info("testLeaderSnapshotTriggeredByMemoryThresholdExceededWithLaggingFollower starting");
    snapshotBatchCount = 5;
    setup();
    sendInitialPayloadsReplicatedToAllFollowers("zero");
    leaderActor.underlyingActor().setMockTotalMemory(1000);
    // We'll expect a ReplicatedLogImplEntry message and an ApplyJournalEntries message added to the journal.
    InMemoryJournal.addWriteMessagesCompleteLatch(leaderId, 2);
    follower2Actor.underlyingActor().startDropMessages(AppendEntries.class);
    // Sleep for at least the election timeout interval so follower 2 is deemed inactive by the leader.
    Uninterruptibles.sleepUninterruptibly(leaderConfigParams.getElectionTimeOutInterval().toMillis() + 5, TimeUnit.MILLISECONDS);
    // Send a payload with a large relative size but not enough to trigger a snapshot.
    MockPayload payload1 = sendPayloadData(leaderActor, "one", 500);
    // Verify the leader got consensus and applies the first log entry even though follower 2 didn't respond.
    List<ApplyState> applyStates = MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 1);
    verifyApplyState(applyStates.get(0), leaderCollectorActor, payload1.toString(), currentTerm, 1, payload1);
    // Wait for all the ReplicatedLogImplEntry and ApplyJournalEntries messages to be added to the journal
    // before the snapshot so the snapshot sequence # will be higher to ensure the snapshot gets
    // purged from the snapshot store after subsequent snapshots.
    InMemoryJournal.waitForWriteMessagesComplete(leaderId);
    // Verify a snapshot is not triggered.
    CaptureSnapshot captureSnapshot = MessageCollectorActor.getFirstMatching(leaderCollectorActor, CaptureSnapshot.class);
    Assert.assertNull("Leader received unexpected CaptureSnapshot", captureSnapshot);
    expSnapshotState.add(payload1);
    // Sleep for at least the election timeout interval so follower 2 is deemed inactive by the leader.
    Uninterruptibles.sleepUninterruptibly(leaderConfigParams.getElectionTimeOutInterval().toMillis() + 5, TimeUnit.MILLISECONDS);
    // Send another payload with a large enough relative size in combination with the last payload
    // that exceeds the memory threshold (70% * 1000 = 700) - this should do a snapshot.
    MockPayload payload2 = sendPayloadData(leaderActor, "two", 201);
    // Verify the leader applies the last log entry.
    applyStates = MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 2);
    verifyApplyState(applyStates.get(1), leaderCollectorActor, payload2.toString(), currentTerm, 2, payload2);
    // Verify follower 1 applies each log entry.
    applyStates = MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, 2);
    verifyApplyState(applyStates.get(0), null, null, currentTerm, 1, payload1);
    verifyApplyState(applyStates.get(1), null, null, currentTerm, 2, payload2);
    // A snapshot should've occurred - wait for it to complete.
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, SaveSnapshotSuccess.class);
    // Because the snapshot was triggered by exceeding the memory threshold the leader should've advanced
    // the snapshot index to the last applied index and trimmed the log even though the entries weren't
    // replicated to all followers.
    verifyLeadersTrimmedLog(2, 0);
    // Verify the leader's persisted snapshot.
    List<Snapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(leaderId, Snapshot.class);
    assertEquals("Persisted snapshots size", 1, persistedSnapshots.size());
    verifySnapshot("Persisted", persistedSnapshots.get(0), currentTerm, 1, currentTerm, 2);
    List<ReplicatedLogEntry> unAppliedEntry = persistedSnapshots.get(0).getUnAppliedEntries();
    assertEquals("Persisted Snapshot getUnAppliedEntries size", 1, unAppliedEntry.size());
    verifyReplicatedLogEntry(unAppliedEntry.get(0), currentTerm, 2, payload2);
    expSnapshotState.add(payload2);
    verifyInstallSnapshotToLaggingFollower(2L, null);
    // Sends a payload with index 3.
    verifyNoSubsequentSnapshotAfterMemoryThresholdExceededSnapshot();
    // Sends 3 payloads with indexes 4, 5 and 6.
    long leadersSnapshotIndexOnRecovery = verifyReplicationsAndSnapshotWithNoLaggingAfterInstallSnapshot();
    // Recover the leader from persistence and verify.
    long leadersLastIndexOnRecovery = 6;
    long leadersFirstJournalEntryIndexOnRecovery = leadersSnapshotIndexOnRecovery + 1;
    verifyLeaderRecoveryAfterReinstatement(leadersLastIndexOnRecovery, leadersSnapshotIndexOnRecovery, leadersFirstJournalEntryIndexOnRecovery);
    testLog.info("testLeaderSnapshotTriggeredByMemoryThresholdExceeded ending");
}

Example 12 with MockPayload

use of org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockPayload in project controller by opendaylight.

the class ReplicationWithSlicedPayloadIntegrationTest method runTest.

@Test
public void runTest() throws Exception {
    testLog.info("ReplicationWithSlicedPayloadIntegrationTest starting");
    // Create the leader and 2 follower actors.
    snapshotChunkSize = 20;
    DefaultConfigParamsImpl followerConfigParams = newFollowerConfigParams();
    followerConfigParams.setSnapshotBatchCount(snapshotBatchCount);
    follower1Actor = newTestRaftActor(follower1Id, ImmutableMap.of(leaderId, testActorPath(leaderId), follower2Id, testActorPath(follower2Id)), followerConfigParams);
    follower2Actor = newTestRaftActor(follower2Id, ImmutableMap.of(leaderId, testActorPath(leaderId), follower1Id, testActorPath(follower1Id)), followerConfigParams);
    peerAddresses = ImmutableMap.<String, String>builder().put(follower1Id, follower1Actor.path().toString()).put(follower2Id, follower2Actor.path().toString()).build();
    leaderConfigParams = newLeaderConfigParams();
    leaderActor = newTestRaftActor(leaderId, peerAddresses, leaderConfigParams);
    follower1CollectorActor = follower1Actor.underlyingActor().collectorActor();
    follower2CollectorActor = follower2Actor.underlyingActor().collectorActor();
    leaderCollectorActor = leaderActor.underlyingActor().collectorActor();
    leaderContext = leaderActor.underlyingActor().getRaftActorContext();
    waitUntilLeader(leaderActor);
    currentTerm = leaderContext.getTermInformation().getCurrentTerm();
    // Send a large payload that exceeds the size threshold and needs to be sliced.
    MockPayload largePayload = sendPayloadData(leaderActor, "large", snapshotChunkSize + 1);
    // Then send a small payload that does not need to be sliced.
    MockPayload smallPayload = sendPayloadData(leaderActor, "normal", snapshotChunkSize - 1);
    final List<ApplyState> leaderApplyState = expectMatching(leaderCollectorActor, ApplyState.class, 2);
    verifyApplyState(leaderApplyState.get(0), leaderCollectorActor, largePayload.toString(), currentTerm, 0, largePayload);
    verifyApplyState(leaderApplyState.get(1), leaderCollectorActor, smallPayload.toString(), currentTerm, 1, smallPayload);
    final List<ApplyState> follower1ApplyState = expectMatching(follower1CollectorActor, ApplyState.class, 2);
    verifyApplyState(follower1ApplyState.get(0), null, null, currentTerm, 0, largePayload);
    verifyApplyState(follower1ApplyState.get(1), null, null, currentTerm, 1, smallPayload);
    final List<ApplyState> follower2ApplyState = expectMatching(follower2CollectorActor, ApplyState.class, 2);
    verifyApplyState(follower2ApplyState.get(0), null, null, currentTerm, 0, largePayload);
    verifyApplyState(follower2ApplyState.get(1), null, null, currentTerm, 1, smallPayload);
    testLog.info("ReplicationWithSlicedPayloadIntegrationTest ending");
}

Example 13 with MockPayload

use of org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockPayload in project controller by opendaylight.

the class PreLeaderScenarioTest method testUnComittedEntryOnLeaderChange.

@Test
public void testUnComittedEntryOnLeaderChange() throws Exception {
    testLog.info("testUnComittedEntryOnLeaderChange starting");
    createRaftActors();
    // Drop AppendEntriesReply to the leader so it doesn't commit the payload entry.
    leaderActor.underlyingActor().startDropMessages(AppendEntriesReply.class);
    follower2Actor.underlyingActor().startDropMessages(AppendEntries.class);
    // Send a payload and verify AppendEntries is received in follower1.
    MockPayload payload0 = sendPayloadData(leaderActor, "zero");
    AppendEntries appendEntries = expectFirstMatching(follower1CollectorActor, AppendEntries.class);
    assertEquals("AppendEntries - # entries", 1, appendEntries.getEntries().size());
    verifyReplicatedLogEntry(appendEntries.getEntries().get(0), currentTerm, 0, payload0);
    // Kill the leader actor.
    killActor(leaderActor);
    // At this point, the payload entry is in follower1's log but is uncommitted. follower2 has not
    // received the payload entry yet.
    assertEquals("Follower 1 journal log size", 1, follower1Context.getReplicatedLog().size());
    assertEquals("Follower 1 journal last index", 0, follower1Context.getReplicatedLog().lastIndex());
    assertEquals("Follower 1 commit index", -1, follower1Context.getCommitIndex());
    assertEquals("Follower 1 last applied index", -1, follower1Context.getLastApplied());
    assertEquals("Follower 2 journal log size", 0, follower2Context.getReplicatedLog().size());
    follower2Actor.underlyingActor().stopDropMessages(AppendEntries.class);
    clearMessages(follower1NotifierActor);
    // Force follower1 to start an election. It should win since it's journal is more up-to-date than
    // follower2's journal.
    follower1Actor.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
    // Verify the expected raft state changes. It should go to PreLeader since it has an uncommitted entry.
    List<RoleChanged> roleChange = expectMatching(follower1NotifierActor, RoleChanged.class, 3);
    assertEquals("Role change 1", RaftState.Candidate.name(), roleChange.get(0).getNewRole());
    assertEquals("Role change 2", RaftState.PreLeader.name(), roleChange.get(1).getNewRole());
    assertEquals("Role change 3", RaftState.Leader.name(), roleChange.get(2).getNewRole());
    final long previousTerm = currentTerm;
    currentTerm = follower1Context.getTermInformation().getCurrentTerm();
    // Since it went to Leader, it should've appended and successfully replicated a NoopPaylod with the
    // new term to follower2 and committed both entries, including the first payload from the previous term.
    assertEquals("Follower 1 journal log size", 2, follower1Context.getReplicatedLog().size());
    assertEquals("Follower 1 journal last index", 1, follower1Context.getReplicatedLog().lastIndex());
    assertEquals("Follower 1 commit index", 1, follower1Context.getCommitIndex());
    verifyReplicatedLogEntry(follower1Context.getReplicatedLog().get(0), previousTerm, 0, payload0);
    verifyReplicatedLogEntry(follower1Context.getReplicatedLog().get(1), currentTerm, 1, NoopPayload.INSTANCE);
    // Both entries should be applied to the state.
    expectMatching(follower1CollectorActor, ApplyState.class, 2);
    expectMatching(follower2CollectorActor, ApplyState.class, 2);
    assertEquals("Follower 1 last applied index", 1, follower1Context.getLastApplied());
    // Verify follower2's journal matches follower1's.
    assertEquals("Follower 2 journal log size", 2, follower2Context.getReplicatedLog().size());
    assertEquals("Follower 2 journal last index", 1, follower2Context.getReplicatedLog().lastIndex());
    assertEquals("Follower 2 commit index", 1, follower2Context.getCommitIndex());
    assertEquals("Follower 2 last applied index", 1, follower2Context.getLastApplied());
    verifyReplicatedLogEntry(follower2Context.getReplicatedLog().get(0), previousTerm, 0, payload0);
    verifyReplicatedLogEntry(follower2Context.getReplicatedLog().get(1), currentTerm, 1, NoopPayload.INSTANCE);
    // Reinstate follower1.
    killActor(follower1Actor);
    follower1Actor = newTestRaftActor(follower1Id, TestRaftActor.newBuilder().peerAddresses(ImmutableMap.of(leaderId, testActorPath(leaderId), follower2Id, testActorPath(follower2Id))).config(followerConfigParams));
    follower1Actor.underlyingActor().waitForRecoveryComplete();
    follower1Context = follower1Actor.underlyingActor().getRaftActorContext();
    // Verify follower1's journal was persisted and recovered correctly.
    assertEquals("Follower 1 journal log size", 2, follower1Context.getReplicatedLog().size());
    assertEquals("Follower 1 journal last index", 1, follower1Context.getReplicatedLog().lastIndex());
    assertEquals("Follower 1 commit index", 1, follower1Context.getCommitIndex());
    assertEquals("Follower 1 last applied index", 1, follower1Context.getLastApplied());
    verifyReplicatedLogEntry(follower1Context.getReplicatedLog().get(0), previousTerm, 0, payload0);
    verifyReplicatedLogEntry(follower1Context.getReplicatedLog().get(1), currentTerm, 1, NoopPayload.INSTANCE);
    testLog.info("testUnComittedEntryOnLeaderChange ending");
}

Example 14 with MockPayload

use of org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockPayload in project controller by opendaylight.

the class RaftActorRecoverySupportTest method testDataRecoveredWithPersistenceDisabled.

@Test
public void testDataRecoveredWithPersistenceDisabled() {
    doNothing().when(mockCohort).applyRecoverySnapshot(anyObject());
    doReturn(false).when(mockPersistence).isRecoveryApplicable();
    doReturn(10L).when(mockPersistentProvider).getLastSequenceNumber();
    Snapshot snapshot = Snapshot.create(new MockSnapshotState(Arrays.asList(new MockPayload("1"))), Collections.<ReplicatedLogEntry>emptyList(), 3, 1, 3, 1, -1, null, null);
    SnapshotOffer snapshotOffer = new SnapshotOffer(new SnapshotMetadata("test", 6, 12345), snapshot);
    sendMessageToSupport(snapshotOffer);
    sendMessageToSupport(new UpdateElectionTerm(5, "member2"));
    sendMessageToSupport(new SimpleReplicatedLogEntry(4, 1, new MockRaftActorContext.MockPayload("4")));
    sendMessageToSupport(new SimpleReplicatedLogEntry(5, 1, new MockRaftActorContext.MockPayload("5")));
    sendMessageToSupport(new ApplyJournalEntries(4));
    sendMessageToSupport(new DeleteEntries(5));
    assertEquals("Journal log size", 0, context.getReplicatedLog().size());
    assertEquals("Last index", -1, context.getReplicatedLog().lastIndex());
    assertEquals("Last applied", -1, context.getLastApplied());
    assertEquals("Commit index", -1, context.getCommitIndex());
    assertEquals("Snapshot term", -1, context.getReplicatedLog().getSnapshotTerm());
    assertEquals("Snapshot index", -1, context.getReplicatedLog().getSnapshotIndex());
    assertEquals("Current term", 5, context.getTermInformation().getCurrentTerm());
    assertEquals("Voted For", "member2", context.getTermInformation().getVotedFor());
    sendMessageToSupport(RecoveryCompleted.getInstance(), true);
    verify(mockCohort, never()).applyRecoverySnapshot(anyObject());
    verify(mockCohort, never()).getRestoreFromSnapshot();
    verifyNoMoreInteractions(mockCohort);
    verify(mockPersistentProvider).deleteMessages(10L);
}

Example 15 with MockPayload

use of org.opendaylight.controller.cluster.raft.MockRaftActorContext.MockPayload in project controller by opendaylight.

the class RaftActorRecoverySupportTest method testOnSnapshotOfferWithServerConfiguration.

@Test
public void testOnSnapshotOfferWithServerConfiguration() {
    long electionTerm = 2;
    String electionVotedFor = "member-2";
    ServerConfigurationPayload serverPayload = new ServerConfigurationPayload(Arrays.asList(new ServerInfo(localId, true), new ServerInfo("follower1", true), new ServerInfo("follower2", true)));
    MockSnapshotState snapshotState = new MockSnapshotState(Arrays.asList(new MockPayload("1")));
    Snapshot snapshot = Snapshot.create(snapshotState, Collections.<ReplicatedLogEntry>emptyList(), -1, -1, -1, -1, electionTerm, electionVotedFor, serverPayload);
    SnapshotMetadata metadata = new SnapshotMetadata("test", 6, 12345);
    SnapshotOffer snapshotOffer = new SnapshotOffer(metadata, snapshot);
    sendMessageToSupport(snapshotOffer);
    assertEquals("Journal log size", 0, context.getReplicatedLog().size());
    assertEquals("Election term", electionTerm, context.getTermInformation().getCurrentTerm());
    assertEquals("Election votedFor", electionVotedFor, context.getTermInformation().getVotedFor());
    assertTrue("Dynamic server configuration", context.isDynamicServerConfigurationInUse());
    assertEquals("Peer List", Sets.newHashSet("follower1", "follower2"), Sets.newHashSet(context.getPeerIds()));
}