Examples with LogEntryProto org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto used on opensource projects

Example 1 with LogEntryProto

use of org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class StateMachineUpdater method run.

@Override
public void run() {
    final RaftStorage storage = server.getState().getStorage();
    while (isRunning()) {
        try {
            synchronized (this) {
                // Thus initially lastAppliedIndex can be greater than lastCommitted.
                while (lastAppliedIndex >= raftLog.getLastCommittedIndex()) {
                    wait();
                }
            }
            final long committedIndex = raftLog.getLastCommittedIndex();
            Preconditions.assertTrue(lastAppliedIndex < committedIndex);
            if (state == State.RELOAD) {
                Preconditions.assertTrue(stateMachine.getLifeCycleState() == LifeCycle.State.PAUSED);
                stateMachine.reinitialize(server.getId(), properties, storage);
                SnapshotInfo snapshot = stateMachine.getLatestSnapshot();
                Preconditions.assertTrue(snapshot != null && snapshot.getIndex() > lastAppliedIndex, "Snapshot: %s, lastAppliedIndex: %s", snapshot, lastAppliedIndex);
                lastAppliedIndex = snapshot.getIndex();
                lastSnapshotIndex = snapshot.getIndex();
                state = State.RUNNING;
            }
            final MemoizedSupplier<List<CompletableFuture<Message>>> futures = MemoizedSupplier.valueOf(() -> new ArrayList<>());
            while (lastAppliedIndex < committedIndex) {
                final long nextIndex = lastAppliedIndex + 1;
                final LogEntryProto next = raftLog.get(nextIndex);
                if (next != null) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("{}: applying nextIndex={}, nextLog={}", this, nextIndex, ServerProtoUtils.toString(next));
                    }
                    final CompletableFuture<Message> f = server.applyLogToStateMachine(next);
                    if (f != null) {
                        futures.get().add(f);
                    }
                    lastAppliedIndex = nextIndex;
                } else {
                    LOG.debug("{}: logEntry {} is null. There may be snapshot to load. state:{}", this, nextIndex, state);
                    break;
                }
            }
            // check if need to trigger a snapshot
            if (shouldTakeSnapshot(lastAppliedIndex)) {
                if (futures.isInitialized()) {
                    JavaUtils.allOf(futures.get()).get();
                }
                stateMachine.takeSnapshot();
                // TODO purge logs, including log cache. but should keep log for leader's RPCSenders
                lastSnapshotIndex = lastAppliedIndex;
            }
        } catch (InterruptedException e) {
            if (!isRunning()) {
                LOG.info("{}: the StateMachineUpdater is interrupted and will exit.", this);
            } else {
                final String s = this + ": the StateMachineUpdater is wrongly interrupted";
                ExitUtils.terminate(1, s, e, LOG);
            }
        } catch (Throwable t) {
            final String s = this + ": the StateMachineUpdater hits Throwable";
            ExitUtils.terminate(2, s, t, LOG);
        }
    }
}

Example 2 with LogEntryProto

use of org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class LogInputStream method scanEditLog.

/**
 * Find the last valid entry index in the stream.
 * If there are invalid or corrupt entries in the middle of the stream,
 * scanEditLog will skip over them.
 *
 * This reads through the stream but does not close it.
 *
 * @param maxIndexToScan Maximum entry index to try to scan. The scan returns
 *                       after reading this or a higher index. The file
 *                       portion beyond this index is potentially being
 *                       updated.
 */
static LogValidation scanEditLog(LogInputStream in, long maxIndexToScan) {
    long lastPos = 0;
    long end = INVALID_LOG_INDEX;
    long numValid = 0;
    boolean hitError = false;
    while (end < maxIndexToScan) {
        long index;
        lastPos = in.getPosition();
        try {
            if (hitError) {
                LogEntryProto entry = in.nextEntry();
                index = entry != null ? entry.getIndex() : INVALID_LOG_INDEX;
                LOG.warn("After resync, position is " + in.getPosition());
            } else {
                index = in.scanNextEntry();
            }
            if (index == INVALID_LOG_INDEX) {
                break;
            } else {
                hitError = false;
            }
        } catch (Throwable t) {
            LOG.warn("Caught exception after scanning through {} ops from {}" + " while determining its valid length. Position was " + lastPos, numValid, in, t);
            hitError = true;
            continue;
        }
        if (end == INVALID_LOG_INDEX || index > end) {
            end = index;
        }
        numValid++;
    }
    return new LogValidation(lastPos, end, false);
}

Example 3 with LogEntryProto

use of org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class LogInputStream method nextEntry.

public LogEntryProto nextEntry() throws IOException {
    LogEntryProto entry = null;
    switch(state) {
        case UNINIT:
            try {
                init();
            } catch (Throwable e) {
                LOG.error("caught exception initializing " + this, e);
                throw IOUtils.asIOException(e);
            }
            Preconditions.assertTrue(state != State.UNINIT);
            return nextEntry();
        case OPEN:
            entry = reader.readEntry();
            if (entry != null) {
                long index = entry.getIndex();
                if (!isOpen() && index >= endIndex) {
                    /**
                     * The end index may be derived from the segment recovery
                     * process. It is possible that we still have some uncleaned garbage
                     * in the end. We should skip them.
                     */
                    long skipAmt = logFile.length() - reader.getPos();
                    if (skipAmt > 0) {
                        LOG.debug("skipping {} bytes at the end of log '{}': reached" + " entry {} out of {}", skipAmt, getName(), index, endIndex);
                        reader.skipFully(skipAmt);
                    }
                }
            }
            break;
        case CLOSED:
            // return null
            break;
    }
    return entry;
}

Example 4 with LogEntryProto

use of org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class LogSegment method readSegmentFile.

private static void readSegmentFile(File file, long start, long end, boolean isOpen, Consumer<LogEntryProto> entryConsumer) throws IOException {
    try (LogInputStream in = new LogInputStream(file, start, end, isOpen)) {
        LogEntryProto next;
        LogEntryProto prev = null;
        while ((next = in.nextEntry()) != null) {
            if (prev != null) {
                Preconditions.assertTrue(next.getIndex() == prev.getIndex() + 1, "gap between entry %s and entry %s", prev, next);
            }
            if (entryConsumer != null) {
                entryConsumer.accept(next);
            }
            prev = next;
        }
    }
}

Example 5 with LogEntryProto

use of org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto in project incubator-ratis by apache.

the class RaftBasicTests method testOldLeaderNotCommit.

@Test
public void testOldLeaderNotCommit() throws Exception {
    LOG.info("Running testOldLeaderNotCommit");
    final MiniRaftCluster cluster = getCluster();
    final RaftPeerId leaderId = waitForLeader(cluster).getId();
    List<RaftServerImpl> followers = cluster.getFollowers();
    final RaftServerImpl followerToCommit = followers.get(0);
    for (int i = 1; i < NUM_SERVERS - 1; i++) {
        RaftServerImpl follower = followers.get(i);
        cluster.killServer(follower.getId());
    }
    SimpleMessage[] messages = SimpleMessage.create(1);
    sendMessageInNewThread(cluster, messages);
    Thread.sleep(cluster.getMaxTimeout() + 100);
    logEntriesContains(followerToCommit.getState().getLog(), messages);
    cluster.killServer(leaderId);
    cluster.killServer(followerToCommit.getId());
    for (int i = 1; i < NUM_SERVERS - 1; i++) {
        RaftServerImpl follower = followers.get(i);
        cluster.restartServer(follower.getId(), false);
    }
    waitForLeader(cluster);
    Thread.sleep(cluster.getMaxTimeout() + 100);
    final Predicate<LogEntryProto> predicate = l -> l.getTerm() != 1;
    cluster.getServerAliveStream().map(s -> s.getState().getLog()).forEach(log -> RaftTestUtil.checkLogEntries(log, messages, predicate));
}