Examples with RaftPeerId org.apache.ratis.protocol.RaftPeerId used on opensource projects

Example 11 with RaftPeerId

use of org.apache.ratis.protocol.RaftPeerId in project incubator-ratis by apache.

the class RaftSnapshotBaseTest method testRestartPeer.

/**
 * Keep generating writing traffic and make sure snapshots are taken.
 * We then restart the whole raft peer and check if it can correctly load
 * snapshots + raft log.
 */
@Test
public void testRestartPeer() throws Exception {
    RaftTestUtil.waitForLeader(cluster);
    final RaftPeerId leaderId = cluster.getLeader().getId();
    int i = 0;
    try (final RaftClient client = cluster.createClient(leaderId)) {
        for (; i < SNAPSHOT_TRIGGER_THRESHOLD * 2 - 1; i++) {
            RaftClientReply reply = client.send(new SimpleMessage("m" + i));
            Assert.assertTrue(reply.isSuccess());
        }
    }
    // wait for the snapshot to be done
    final File snapshotFile = getSnapshotFile(cluster, i);
    int retries = 0;
    do {
        Thread.sleep(1000);
    } while (!snapshotFile.exists() && retries++ < 10);
    Assert.assertTrue(snapshotFile + " does not exist", snapshotFile.exists());
    // restart the peer and check if it can correctly load snapshot
    cluster.restart(false);
    try {
        // 200 messages + two leader elections --> last committed = 201
        assertLeaderContent(cluster);
    } finally {
        cluster.shutdown();
    }
}

Example 12 with RaftPeerId

use of org.apache.ratis.protocol.RaftPeerId in project alluxio by Alluxio.

the class RaftJournalSystem method catchUp.

/**
 * Attempts to catch up. If the master loses leadership during this method, it will return early.
 *
 * The caller is responsible for detecting and responding to leadership changes.
 */
private void catchUp(JournalStateMachine stateMachine, RaftJournalAppender client) throws TimeoutException, InterruptedException {
    long startTime = System.currentTimeMillis();
    long waitBeforeRetry = ServerConfiguration.global().getMs(PropertyKey.MASTER_EMBEDDED_JOURNAL_CATCHUP_RETRY_WAIT);
    // Wait for any outstanding snapshot to complete.
    CommonUtils.waitFor("snapshotting to finish", () -> !stateMachine.isSnapshotting(), WaitForOptions.defaults().setTimeoutMs(10 * Constants.MINUTE_MS));
    OptionalLong endCommitIndex = OptionalLong.empty();
    try {
        // affects the completion time estimate in the logs.
        synchronized (this) {
            // synchronized to appease findbugs; shouldn't make any difference
            RaftPeerId serverId = mServer.getId();
            Optional<RaftProtos.CommitInfoProto> commitInfo = getGroupInfo().getCommitInfos().stream().filter(commit -> serverId.equals(RaftPeerId.valueOf(commit.getServer().getId()))).findFirst();
            if (commitInfo.isPresent()) {
                endCommitIndex = OptionalLong.of(commitInfo.get().getCommitIndex());
            } else {
                throw new IOException("Commit info was not present. Couldn't find the current server's " + "latest commit");
            }
        }
    } catch (IOException e) {
        LogUtils.warnWithException(LOG, "Failed to get raft log information before replay." + " Replay statistics will not be available", e);
    }
    RaftJournalProgressLogger progressLogger = new RaftJournalProgressLogger(mStateMachine, endCommitIndex);
    // leader before trying again.
    while (true) {
        if (mPrimarySelector.getState() != PrimarySelector.State.PRIMARY) {
            return;
        }
        long lastAppliedSN = stateMachine.getLastAppliedSequenceNumber();
        long gainPrimacySN = ThreadLocalRandom.current().nextLong(Long.MIN_VALUE, 0);
        LOG.info("Performing catchup. Last applied SN: {}. Catchup ID: {}", lastAppliedSN, gainPrimacySN);
        Exception ex;
        try {
            CompletableFuture<RaftClientReply> future = client.sendAsync(toRaftMessage(JournalEntry.newBuilder().setSequenceNumber(gainPrimacySN).build()), TimeDuration.valueOf(5, TimeUnit.SECONDS));
            RaftClientReply reply = future.get(5, TimeUnit.SECONDS);
            ex = reply.getException();
        } catch (TimeoutException | ExecutionException | IOException e) {
            ex = e;
        }
        if (ex != null) {
            // LeaderNotReadyException typically indicates Ratis is still replaying the journal.
            if (ex instanceof LeaderNotReadyException) {
                progressLogger.logProgress();
            } else {
                LOG.info("Exception submitting term start entry: {}", ex.toString());
            }
            // avoid excessive retries when server is not ready
            Thread.sleep(waitBeforeRetry);
            continue;
        }
        try {
            CommonUtils.waitFor("term start entry " + gainPrimacySN + " to be applied to state machine", () -> stateMachine.getLastPrimaryStartSequenceNumber() == gainPrimacySN, WaitForOptions.defaults().setInterval(Constants.SECOND_MS).setTimeoutMs(5 * Constants.SECOND_MS));
        } catch (TimeoutException e) {
            LOG.info(e.toString());
            continue;
        }
        // are not leader.
        try {
            CommonUtils.waitFor("check primacySN " + gainPrimacySN + " and lastAppliedSN " + lastAppliedSN + " to be applied to leader", () -> stateMachine.getLastAppliedSequenceNumber() == lastAppliedSN && stateMachine.getLastPrimaryStartSequenceNumber() == gainPrimacySN, WaitForOptions.defaults().setInterval(Constants.SECOND_MS).setTimeoutMs((int) mConf.getMaxElectionTimeoutMs()));
        } catch (TimeoutException e) {
            // Restart the catchup process.
            continue;
        }
        LOG.info("Caught up in {}ms. Last sequence number from previous term: {}.", System.currentTimeMillis() - startTime, stateMachine.getLastAppliedSequenceNumber());
        return;
    }
}

Example 13 with RaftPeerId

use of org.apache.ratis.protocol.RaftPeerId in project alluxio by Alluxio.

the class RaftJournalSystem method removeQuorumServer.

/**
 * Removes from RAFT quorum, a server with given address.
 * For server to be removed, it should be in unavailable state in quorum.
 *
 * @param serverNetAddress address of the server to remove from the quorum
 * @throws IOException
 */
public synchronized void removeQuorumServer(NetAddress serverNetAddress) throws IOException {
    InetSocketAddress serverAddress = InetSocketAddress.createUnresolved(serverNetAddress.getHost(), serverNetAddress.getRpcPort());
    RaftPeerId peerId = RaftJournalUtils.getPeerId(serverAddress);
    try (RaftClient client = createClient()) {
        Collection<RaftPeer> peers = mServer.getGroups().iterator().next().getPeers();
        RaftClientReply reply = client.admin().setConfiguration(peers.stream().filter(peer -> !peer.getId().equals(peerId)).collect(Collectors.toList()));
        if (reply.getException() != null) {
            throw reply.getException();
        }
    }
}

Example 14 with RaftPeerId

use of org.apache.ratis.protocol.RaftPeerId in project alluxio by Alluxio.

the class RaftJournalSystem method transferLeadership.

/**
 * Transfers the leadership of the quorum to another server.
 *
 * @param newLeaderNetAddress the address of the server
 * @return the guid of transfer leader command
 */
public synchronized String transferLeadership(NetAddress newLeaderNetAddress) {
    final boolean allowed = mTransferLeaderAllowed.getAndSet(false);
    String transferId = UUID.randomUUID().toString();
    if (!allowed) {
        String msg = "transfer is not allowed at the moment because the master is " + (mRaftJournalWriter == null ? "still gaining primacy" : "already transferring the ") + "leadership";
        mErrorMessages.put(transferId, TransferLeaderMessage.newBuilder().setMsg(msg).build());
        return transferId;
    }
    try {
        InetSocketAddress serverAddress = InetSocketAddress.createUnresolved(newLeaderNetAddress.getHost(), newLeaderNetAddress.getRpcPort());
        List<RaftPeer> oldPeers = new ArrayList<>(mRaftGroup.getPeers());
        // The NetUtil function is used by Ratis to convert InetSocketAddress to string
        String strAddr = NetUtils.address2String(serverAddress);
        // if you cannot find the address in the quorum, throw exception.
        if (oldPeers.stream().map(RaftPeer::getAddress).noneMatch(addr -> addr.equals(strAddr))) {
            throw new IOException(String.format("<%s> is not part of the quorum <%s>.", strAddr, oldPeers.stream().map(RaftPeer::getAddress).collect(Collectors.toList())));
        }
        if (strAddr.equals(mRaftGroup.getPeer(mPeerId).getAddress())) {
            throw new IOException(String.format("%s is already the leader", strAddr));
        }
        RaftPeerId newLeaderPeerId = RaftJournalUtils.getPeerId(serverAddress);
        /* update priorities to enable transfer */
        List<RaftPeer> peersWithNewPriorities = new ArrayList<>();
        for (RaftPeer peer : oldPeers) {
            peersWithNewPriorities.add(RaftPeer.newBuilder(peer).setPriority(peer.getId().equals(newLeaderPeerId) ? 2 : 1).build());
        }
        try (RaftClient client = createClient()) {
            String stringPeers = "[" + peersWithNewPriorities.stream().map(RaftPeer::toString).collect(Collectors.joining(", ")) + "]";
            LOG.info("Applying new peer state before transferring leadership: {}", stringPeers);
            RaftClientReply reply = client.admin().setConfiguration(peersWithNewPriorities);
            processReply(reply, "failed to set master priorities before initiating election");
            /* transfer leadership */
            LOG.info("Transferring leadership to master with address <{}> and with RaftPeerId <{}>", serverAddress, newLeaderPeerId);
            // fire and forget: need to immediately return as the master will shut down its RPC servers
            // once the TransferLeadershipRequest is initiated.
            final int SLEEP_TIME_MS = 3_000;
            final int TRANSFER_LEADER_WAIT_MS = 30_000;
            new Thread(() -> {
                try {
                    Thread.sleep(SLEEP_TIME_MS);
                    RaftClientReply reply1 = client.admin().transferLeadership(newLeaderPeerId, TRANSFER_LEADER_WAIT_MS);
                    processReply(reply1, "election failed");
                } catch (Throwable t) {
                    LOG.error("caught an error when executing transfer: {}", t.getMessage());
                    // we only allow transfers again if the transfer is unsuccessful: a success means it
                    // will soon lose primacy
                    mTransferLeaderAllowed.set(true);
                    mErrorMessages.put(transferId, TransferLeaderMessage.newBuilder().setMsg(t.getMessage()).build());
                /* checking the transfer happens in {@link QuorumElectCommand} */
                }
            }).start();
            LOG.info("Transferring leadership initiated");
        }
    } catch (Throwable t) {
        mTransferLeaderAllowed.set(true);
        LOG.warn(t.getMessage());
        mErrorMessages.put(transferId, TransferLeaderMessage.newBuilder().setMsg(t.getMessage()).build());
    }
    return transferId;
}

Example 15 with RaftPeerId

use of org.apache.ratis.protocol.RaftPeerId in project alluxio by Alluxio.

the class SnapshotReplicationManager method requestInfo.

private void requestInfo() {
    Preconditions.checkState(mDownloadState.get() == DownloadState.REQUEST_INFO);
    try {
        SingleFileSnapshotInfo latestSnapshot = mStorage.getLatestSnapshot();
        SnapshotMetadata snapshotMetadata = latestSnapshot == null ? null : SnapshotMetadata.newBuilder().setSnapshotTerm(latestSnapshot.getTerm()).setSnapshotIndex(latestSnapshot.getIndex()).build();
        // build SnapshotInfoRequests
        Map<RaftPeerId, CompletableFuture<RaftClientReply>> jobs = mJournalSystem.getQuorumServerInfoList().stream().filter(server -> server.getServerState() == QuorumServerState.AVAILABLE).map(server -> RaftJournalUtils.getPeerId(server.getServerAddress().getHost(), server.getServerAddress().getRpcPort())).filter(peerId -> !peerId.equals(mJournalSystem.getLocalPeerId())).collect(Collectors.toMap(Function.identity(), peerId -> mJournalSystem.sendMessageAsync(peerId, toMessage(JournalQueryRequest.newBuilder().setSnapshotInfoRequest(GetSnapshotInfoRequest.getDefaultInstance()).build()))));
        // query all secondary masters for information about their latest snapshot
        for (Map.Entry<RaftPeerId, CompletableFuture<RaftClientReply>> job : jobs.entrySet()) {
            RaftPeerId peerId = job.getKey();
            try {
                RaftClientReply reply = job.getValue().get();
                if (reply.getException() != null) {
                    throw reply.getException();
                }
                JournalQueryResponse response = JournalQueryResponse.parseFrom(reply.getMessage().getContent().asReadOnlyByteBuffer());
                if (!response.hasSnapshotInfoResponse()) {
                    throw new IOException("Invalid response for GetSnapshotInfoRequest " + response);
                }
                LOG.debug("Received snapshot info from follower {} - {}", peerId, response);
                SnapshotMetadata latest = response.getSnapshotInfoResponse().getLatest();
                if (snapshotMetadata == null || (latest.getSnapshotTerm() >= snapshotMetadata.getSnapshotTerm()) && latest.getSnapshotIndex() > snapshotMetadata.getSnapshotIndex()) {
                    mSnapshotCandidates.add(new Pair<>(latest, peerId));
                }
            } catch (Exception e) {
                LOG.warn("Error while requesting snapshot info from {}: {}", peerId, e.toString());
            }
        }
    } catch (Exception e) {
        LogUtils.warnWithException(LOG, "Failed to request snapshot info from followers", e);
    }
}