Examples with ApplyJournalEntries org.opendaylight.controller.cluster.raft.persisted.ApplyJournalEntries used on opensource projects

Example 16 with ApplyJournalEntries

use of org.opendaylight.controller.cluster.raft.persisted.ApplyJournalEntries in project controller by opendaylight.

the class ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest method verifyReplicationsAndSnapshotWithNoLaggingAfterInstallSnapshot.

/**
 * Do another round of payloads and snapshot to verify replicatedToAllIndex gets back on track and
 * snapshots works as expected after doing a follower snapshot. In this step we don't lag a follower.
 */
private long verifyReplicationsAndSnapshotWithNoLaggingAfterInstallSnapshot() throws Exception {
    testLog.info("verifyReplicationsAndSnapshotWithNoLaggingAfterInstallSnapshot starting: replicatedToAllIndex: {}", leader.getReplicatedToAllIndex());
    // Send another payload - a snapshot should occur.
    MockPayload payload4 = sendPayloadData(leaderActor, "four");
    // Wait for the snapshot to complete.
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, SaveSnapshotSuccess.class);
    ApplyState applyState = MessageCollectorActor.expectFirstMatching(leaderCollectorActor, ApplyState.class);
    verifyApplyState(applyState, leaderCollectorActor, payload4.toString(), currentTerm, 4, payload4);
    // Verify the leader's last persisted snapshot (previous ones may not be purged yet).
    List<Snapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(leaderId, Snapshot.class);
    Snapshot persistedSnapshot = persistedSnapshots.get(persistedSnapshots.size() - 1);
    // The last (fourth) payload may or may not have been applied when the snapshot is captured depending on the
    // timing when the async persistence completes.
    List<ReplicatedLogEntry> unAppliedEntry = persistedSnapshot.getUnAppliedEntries();
    long leadersSnapshotIndex;
    if (unAppliedEntry.isEmpty()) {
        leadersSnapshotIndex = 4;
        expSnapshotState.add(payload4);
        verifySnapshot("Persisted", persistedSnapshot, currentTerm, 4, currentTerm, 4);
    } else {
        leadersSnapshotIndex = 3;
        verifySnapshot("Persisted", persistedSnapshot, currentTerm, 3, currentTerm, 4);
        assertEquals("Persisted Snapshot getUnAppliedEntries size", 1, unAppliedEntry.size());
        verifyReplicatedLogEntry(unAppliedEntry.get(0), currentTerm, 4, payload4);
        expSnapshotState.add(payload4);
    }
    // Send a couple more payloads.
    MockPayload payload5 = sendPayloadData(leaderActor, "five");
    MockPayload payload6 = sendPayloadData(leaderActor, "six");
    // Verify the leader applies the 2 log entries.
    List<ApplyState> applyStates = MessageCollectorActor.expectMatching(leaderCollectorActor, ApplyState.class, 3);
    verifyApplyState(applyStates.get(1), leaderCollectorActor, payload5.toString(), currentTerm, 5, payload5);
    verifyApplyState(applyStates.get(2), leaderCollectorActor, payload6.toString(), currentTerm, 6, payload6);
    // Verify the leader applies a log entry for at least the last entry index.
    verifyApplyJournalEntries(leaderCollectorActor, 6);
    // Ensure there's at least 1 more heartbeat to trim the log.
    MessageCollectorActor.clearMessages(leaderCollectorActor);
    MessageCollectorActor.expectFirstMatching(leaderCollectorActor, AppendEntriesReply.class);
    // Verify the leader's final state.
    verifyLeadersTrimmedLog(6);
    InMemoryJournal.dumpJournal(leaderId);
    // Verify the leaders's persisted journal log - it should only contain the last 2 ReplicatedLogEntries
    // added after the snapshot as the persisted journal should've been purged to the snapshot
    // sequence number.
    verifyPersistedJournal(leaderId, Arrays.asList(new SimpleReplicatedLogEntry(5, currentTerm, payload5), new SimpleReplicatedLogEntry(6, currentTerm, payload6)));
    // Verify the leaders's persisted journal contains an ApplyJournalEntries for at least the last entry index.
    List<ApplyJournalEntries> persistedApplyJournalEntries = InMemoryJournal.get(leaderId, ApplyJournalEntries.class);
    boolean found = false;
    for (ApplyJournalEntries entry : persistedApplyJournalEntries) {
        if (entry.getToIndex() == 6) {
            found = true;
            break;
        }
    }
    Assert.assertTrue(String.format("ApplyJournalEntries with index %d not found in leader's persisted journal", 6), found);
    // Verify follower 1 applies the 3 log entries.
    applyStates = MessageCollectorActor.expectMatching(follower1CollectorActor, ApplyState.class, 3);
    verifyApplyState(applyStates.get(0), null, null, currentTerm, 4, payload4);
    verifyApplyState(applyStates.get(1), null, null, currentTerm, 5, payload5);
    verifyApplyState(applyStates.get(2), null, null, currentTerm, 6, payload6);
    // Verify follower 1's log state.
    verifyFollowersTrimmedLog(1, follower1Actor, 6);
    // Verify follower 2 applies the 3 log entries.
    applyStates = MessageCollectorActor.expectMatching(follower2CollectorActor, ApplyState.class, 3);
    verifyApplyState(applyStates.get(0), null, null, currentTerm, 4, payload4);
    verifyApplyState(applyStates.get(1), null, null, currentTerm, 5, payload5);
    verifyApplyState(applyStates.get(2), null, null, currentTerm, 6, payload6);
    // Verify follower 2's log state.
    verifyFollowersTrimmedLog(2, follower2Actor, 6);
    expSnapshotState.add(payload5);
    expSnapshotState.add(payload6);
    testLog.info("verifyReplicationsAndSnapshotWithNoLaggingAfterInstallSnapshot ending");
    return leadersSnapshotIndex;
}

Example 17 with ApplyJournalEntries

use of org.opendaylight.controller.cluster.raft.persisted.ApplyJournalEntries in project controller by opendaylight.

the class AbstractRaftActorBehavior method applyLogToStateMachine.

/**
 * Applies the log entries up to the specified index that is known to be committed to the state machine.
 *
 * @param index the log index
 */
protected void applyLogToStateMachine(final long index) {
    // Now maybe we apply to the state machine
    for (long i = context.getLastApplied() + 1; i < index + 1; i++) {
        ReplicatedLogEntry replicatedLogEntry = context.getReplicatedLog().get(i);
        if (replicatedLogEntry != null) {
            // Send a local message to the local RaftActor (it's derived class to be
            // specific to apply the log to it's index)
            final ApplyState applyState;
            final ClientRequestTracker tracker = removeClientRequestTracker(i);
            if (tracker != null) {
                applyState = new ApplyState(tracker.getClientActor(), tracker.getIdentifier(), replicatedLogEntry);
            } else {
                applyState = new ApplyState(null, null, replicatedLogEntry);
            }
            log.debug("{}: Setting last applied to {}", logName(), i);
            context.setLastApplied(i);
            context.getApplyStateConsumer().accept(applyState);
        } else {
            // if one index is not present in the log, no point in looping
            // around as the rest wont be present either
            log.warn("{}: Missing index {} from log. Cannot apply state. Ignoring {} to {}", logName(), i, i, index);
            break;
        }
    }
    // send a message to persist a ApplyLogEntries marker message into akka's persistent journal
    // will be used during recovery
    // in case if the above code throws an error and this message is not sent, it would be fine
    // as the  append entries received later would initiate add this message to the journal
    actor().tell(new ApplyJournalEntries(context.getLastApplied()), actor());
}

Example 18 with ApplyJournalEntries

use of org.opendaylight.controller.cluster.raft.persisted.ApplyJournalEntries in project controller by opendaylight.

the class RaftActor method handleCommand.

/**
 * Handles a message.
 *
 * @deprecated This method is not final for testing purposes. DO NOT OVERRIDE IT, override
 * {@link #handleNonRaftCommand(Object)} instead.
 */
@Deprecated
@Override
protected // FIXME: make this method final once our unit tests do not need to override it
void handleCommand(final Object message) {
    if (serverConfigurationSupport.handleMessage(message, getSender())) {
        return;
    }
    if (snapshotSupport.handleSnapshotMessage(message, getSender())) {
        return;
    }
    if (message instanceof ApplyState) {
        ApplyState applyState = (ApplyState) message;
        if (!hasFollowers()) {
            // for single node, the capture should happen after the apply state
            // as we delete messages from the persistent journal which have made it to the snapshot
            // capturing the snapshot before applying makes the persistent journal and snapshot out of sync
            // and recovery shows data missing
            context.getReplicatedLog().captureSnapshotIfReady(applyState.getReplicatedLogEntry());
            context.getSnapshotManager().trimLog(context.getLastApplied());
        }
        possiblyHandleBehaviorMessage(message);
    } else if (message instanceof ApplyJournalEntries) {
        ApplyJournalEntries applyEntries = (ApplyJournalEntries) message;
        LOG.debug("{}: Persisting ApplyJournalEntries with index={}", persistenceId(), applyEntries.getToIndex());
        persistence().persistAsync(applyEntries, NoopProcedure.instance());
    } else if (message instanceof FindLeader) {
        getSender().tell(new FindLeaderReply(getLeaderAddress()), getSelf());
    } else if (message instanceof GetOnDemandRaftState) {
        onGetOnDemandRaftStats();
    } else if (message instanceof InitiateCaptureSnapshot) {
        captureSnapshot();
    } else if (message instanceof SwitchBehavior) {
        switchBehavior((SwitchBehavior) message);
    } else if (message instanceof LeaderTransitioning) {
        onLeaderTransitioning((LeaderTransitioning) message);
    } else if (message instanceof Shutdown) {
        onShutDown();
    } else if (message instanceof Runnable) {
        ((Runnable) message).run();
    } else if (message instanceof NoopPayload) {
        persistData(null, null, (NoopPayload) message, false);
    } else if (message instanceof RequestLeadership) {
        onRequestLeadership((RequestLeadership) message);
    } else if (!possiblyHandleBehaviorMessage(message)) {
        handleNonRaftCommand(message);
    }
}

Example 19 with ApplyJournalEntries

use of org.opendaylight.controller.cluster.raft.persisted.ApplyJournalEntries in project controller by opendaylight.

the class DistributedDataStoreRemotingIntegrationTest method testWriteTransactionWithSingleShard.

@Test
public void testWriteTransactionWithSingleShard() throws Exception {
    final String testName = "testWriteTransactionWithSingleShard";
    initDatastoresWithCars(testName);
    final String followerCarShardName = "member-2-shard-cars-" + testName;
    DOMStoreWriteTransaction writeTx = followerDistributedDataStore.newWriteOnlyTransaction();
    assertNotNull("newWriteOnlyTransaction returned null", writeTx);
    writeTx.write(CarsModel.BASE_PATH, CarsModel.emptyContainer());
    writeTx.write(CarsModel.CAR_LIST_PATH, CarsModel.newCarMapNode());
    final MapEntryNode car1 = CarsModel.newCarEntry("optima", BigInteger.valueOf(20000));
    final YangInstanceIdentifier car1Path = CarsModel.newCarPath("optima");
    writeTx.merge(car1Path, car1);
    final MapEntryNode car2 = CarsModel.newCarEntry("sportage", BigInteger.valueOf(25000));
    final YangInstanceIdentifier car2Path = CarsModel.newCarPath("sportage");
    writeTx.merge(car2Path, car2);
    followerTestKit.doCommit(writeTx.ready());
    verifyCars(followerDistributedDataStore.newReadOnlyTransaction(), car1, car2);
    verifyCars(leaderDistributedDataStore.newReadOnlyTransaction(), car1, car2);
    // Test delete
    writeTx = followerDistributedDataStore.newWriteOnlyTransaction();
    writeTx.delete(car1Path);
    followerTestKit.doCommit(writeTx.ready());
    verifyExists(followerDistributedDataStore.newReadOnlyTransaction(), car2Path);
    verifyCars(followerDistributedDataStore.newReadOnlyTransaction(), car2);
    verifyCars(leaderDistributedDataStore.newReadOnlyTransaction(), car2);
    // Re-instate the follower member 2 as a single-node to verify replication and recovery.
    // The following is a bit tricky. Before we reinstate the follower we need to ensure it has persisted and
    // applied and all the log entries from the leader. Since we've verified the car data above we know that
    // all the transactions have been applied on the leader so we first read and capture its lastAppliedIndex.
    final AtomicLong leaderLastAppliedIndex = new AtomicLong();
    IntegrationTestKit.verifyShardState(leaderDistributedDataStore, CARS[0], state -> leaderLastAppliedIndex.set(state.getLastApplied()));
    // Now we need to make sure the follower has persisted the leader's lastAppliedIndex via ApplyJournalEntries.
    // However we don't know exactly how many ApplyJournalEntries messages there will be as it can differ between
    // the tell-based and ask-based front-ends. For ask-based there will be exactly 2 ApplyJournalEntries but
    // tell-based persists additional payloads which could be replicated and applied in a batch resulting in
    // either 2 or 3 ApplyJournalEntries. To handle this we read the follower's persisted ApplyJournalEntries
    // until we find the one that encompasses the leader's lastAppliedIndex.
    Stopwatch sw = Stopwatch.createStarted();
    boolean done = false;
    while (!done) {
        final List<ApplyJournalEntries> entries = InMemoryJournal.get(followerCarShardName, ApplyJournalEntries.class);
        for (ApplyJournalEntries aje : entries) {
            if (aje.getToIndex() >= leaderLastAppliedIndex.get()) {
                done = true;
                break;
            }
        }
        assertTrue("Follower did not persist ApplyJournalEntries containing leader's lastAppliedIndex " + leaderLastAppliedIndex + ". Entries persisted: " + entries, sw.elapsed(TimeUnit.SECONDS) <= 5);
        Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
    }
    TestKit.shutdownActorSystem(leaderSystem, Boolean.TRUE);
    TestKit.shutdownActorSystem(followerSystem, Boolean.TRUE);
    final ActorSystem newSystem = newActorSystem("reinstated-member2", "Member2");
    try (AbstractDataStore member2Datastore = new IntegrationTestKit(newSystem, leaderDatastoreContextBuilder, commitTimeout).setupAbstractDataStore(testParameter, testName, "module-shards-member2", true, CARS)) {
        verifyCars(member2Datastore.newReadOnlyTransaction(), car2);
    }
}