Examples with RaftClient org.apache.ratis.client.RaftClient used on opensource projects

Example 21 with RaftClient

use of org.apache.ratis.client.RaftClient in project alluxio by Alluxio.

the class RaftJournalSystem method transferLeadership.

/**
 * Transfers the leadership of the quorum to another server.
 *
 * @param newLeaderNetAddress the address of the server
 * @return the guid of transfer leader command
 */
public synchronized String transferLeadership(NetAddress newLeaderNetAddress) {
    final boolean allowed = mTransferLeaderAllowed.getAndSet(false);
    String transferId = UUID.randomUUID().toString();
    if (!allowed) {
        String msg = "transfer is not allowed at the moment because the master is " + (mRaftJournalWriter == null ? "still gaining primacy" : "already transferring the ") + "leadership";
        mErrorMessages.put(transferId, TransferLeaderMessage.newBuilder().setMsg(msg).build());
        return transferId;
    }
    try {
        InetSocketAddress serverAddress = InetSocketAddress.createUnresolved(newLeaderNetAddress.getHost(), newLeaderNetAddress.getRpcPort());
        List<RaftPeer> oldPeers = new ArrayList<>(mRaftGroup.getPeers());
        // The NetUtil function is used by Ratis to convert InetSocketAddress to string
        String strAddr = NetUtils.address2String(serverAddress);
        // if you cannot find the address in the quorum, throw exception.
        if (oldPeers.stream().map(RaftPeer::getAddress).noneMatch(addr -> addr.equals(strAddr))) {
            throw new IOException(String.format("<%s> is not part of the quorum <%s>.", strAddr, oldPeers.stream().map(RaftPeer::getAddress).collect(Collectors.toList())));
        }
        if (strAddr.equals(mRaftGroup.getPeer(mPeerId).getAddress())) {
            throw new IOException(String.format("%s is already the leader", strAddr));
        }
        RaftPeerId newLeaderPeerId = RaftJournalUtils.getPeerId(serverAddress);
        /* update priorities to enable transfer */
        List<RaftPeer> peersWithNewPriorities = new ArrayList<>();
        for (RaftPeer peer : oldPeers) {
            peersWithNewPriorities.add(RaftPeer.newBuilder(peer).setPriority(peer.getId().equals(newLeaderPeerId) ? 2 : 1).build());
        }
        try (RaftClient client = createClient()) {
            String stringPeers = "[" + peersWithNewPriorities.stream().map(RaftPeer::toString).collect(Collectors.joining(", ")) + "]";
            LOG.info("Applying new peer state before transferring leadership: {}", stringPeers);
            RaftClientReply reply = client.admin().setConfiguration(peersWithNewPriorities);
            processReply(reply, "failed to set master priorities before initiating election");
            /* transfer leadership */
            LOG.info("Transferring leadership to master with address <{}> and with RaftPeerId <{}>", serverAddress, newLeaderPeerId);
            // fire and forget: need to immediately return as the master will shut down its RPC servers
            // once the TransferLeadershipRequest is initiated.
            final int SLEEP_TIME_MS = 3_000;
            final int TRANSFER_LEADER_WAIT_MS = 30_000;
            new Thread(() -> {
                try {
                    Thread.sleep(SLEEP_TIME_MS);
                    RaftClientReply reply1 = client.admin().transferLeadership(newLeaderPeerId, TRANSFER_LEADER_WAIT_MS);
                    processReply(reply1, "election failed");
                } catch (Throwable t) {
                    LOG.error("caught an error when executing transfer: {}", t.getMessage());
                    // we only allow transfers again if the transfer is unsuccessful: a success means it
                    // will soon lose primacy
                    mTransferLeaderAllowed.set(true);
                    mErrorMessages.put(transferId, TransferLeaderMessage.newBuilder().setMsg(t.getMessage()).build());
                /* checking the transfer happens in {@link QuorumElectCommand} */
                }
            }).start();
            LOG.info("Transferring leadership initiated");
        }
    } catch (Throwable t) {
        mTransferLeaderAllowed.set(true);
        LOG.warn(t.getMessage());
        mErrorMessages.put(transferId, TransferLeaderMessage.newBuilder().setMsg(t.getMessage()).build());
    }
    return transferId;
}

Example 22 with RaftClient

use of org.apache.ratis.client.RaftClient in project incubator-ratis by apache.

the class TestRaftLogMetrics method runTestRaftLogMetrics.

static void runTestRaftLogMetrics(MiniRaftCluster cluster) throws Exception {
    int numMsg = 2;
    final RaftTestUtil.SimpleMessage[] messages = RaftTestUtil.SimpleMessage.create(numMsg);
    try (final RaftClient client = cluster.createClient()) {
        for (RaftTestUtil.SimpleMessage message : messages) {
            client.io().send(message);
        }
    }
    // For leader, flush must happen before client can get replies.
    assertFlushCount(cluster.getLeader());
    assertRaftLogWritePathMetrics(cluster.getLeader());
    // For followers, flush can be lagged behind.  Attempt multiple times.
    for (RaftServer.Division f : cluster.getFollowers()) {
        JavaUtils.attempt(() -> assertFlushCount(f), 10, HUNDRED_MILLIS, f.getId() + "-assertFlushCount", null);
        // We have already waited enough for follower metrics to populate.
        assertRaftLogWritePathMetrics(f);
    }
    // Wait for commits to happen on leader
    JavaUtils.attempt(() -> assertCommitCount(cluster.getLeader(), numMsg), 10, HUNDRED_MILLIS, cluster.getLeader().getId() + "-assertCommitCount", null);
}

Example 23 with RaftClient

use of org.apache.ratis.client.RaftClient in project incubator-ratis by apache.

the class ServerRestartTests method runTestRestartCommitIndex.

void runTestRestartCommitIndex(MiniRaftCluster cluster) throws Exception {
    final SimpleMessage[] messages = SimpleMessage.create(100);
    final List<CompletableFuture<Void>> futures = new ArrayList<>(messages.length);
    for (int i = 0; i < messages.length; i++) {
        final CompletableFuture<Void> f = new CompletableFuture<>();
        futures.add(f);
        final SimpleMessage m = messages[i];
        new Thread(() -> {
            try (final RaftClient client = cluster.createClient()) {
                Assert.assertTrue(client.io().send(m).isSuccess());
            } catch (IOException e) {
                throw new IllegalStateException("Failed to send " + m, e);
            }
            f.complete(null);
        }).start();
    }
    JavaUtils.allOf(futures).get();
    final List<RaftPeerId> ids = new ArrayList<>();
    final RaftServer.Division leader = cluster.getLeader();
    final RaftLog leaderLog = leader.getRaftLog();
    final RaftPeerId leaderId = leader.getId();
    ids.add(leaderId);
    RaftTestUtil.getStateMachineLogEntries(leaderLog);
    // check that the last metadata entry is written to the log
    JavaUtils.attempt(() -> assertLastLogEntry(leader), 20, HUNDRED_MILLIS, "leader last metadata entry", LOG);
    final long lastIndex = leaderLog.getLastEntryTermIndex().getIndex();
    LOG.info("{}: leader lastIndex={}", leaderId, lastIndex);
    final LogEntryProto lastEntry = leaderLog.get(lastIndex);
    LOG.info("{}: leader lastEntry entry[{}] = {}", leaderId, lastIndex, LogProtoUtils.toLogEntryString(lastEntry));
    final long loggedCommitIndex = lastEntry.getMetadataEntry().getCommitIndex();
    final LogEntryProto lastCommittedEntry = leaderLog.get(loggedCommitIndex);
    LOG.info("{}: leader lastCommittedEntry = entry[{}] = {}", leaderId, loggedCommitIndex, LogProtoUtils.toLogEntryString(lastCommittedEntry));
    final SimpleStateMachine4Testing leaderStateMachine = SimpleStateMachine4Testing.get(leader);
    final TermIndex lastAppliedTermIndex = leaderStateMachine.getLastAppliedTermIndex();
    LOG.info("{}: leader lastAppliedTermIndex = {}", leaderId, lastAppliedTermIndex);
    // check follower logs
    for (RaftServer.Division s : cluster.iterateDivisions()) {
        if (!s.getId().equals(leaderId)) {
            ids.add(s.getId());
            JavaUtils.attempt(() -> RaftTestUtil.assertSameLog(leaderLog, s.getRaftLog()), 10, HUNDRED_MILLIS, "assertRaftLog-" + s.getId(), LOG);
        }
    }
    // take snapshot and truncate last (metadata) entry
    leaderStateMachine.takeSnapshot();
    leaderLog.truncate(lastIndex);
    // kill all servers
    ids.forEach(cluster::killServer);
    // Restart and kill servers one by one so that they won't talk to each other.
    for (RaftPeerId id : ids) {
        cluster.restartServer(id, false);
        final RaftServer.Division server = cluster.getDivision(id);
        final RaftLog raftLog = server.getRaftLog();
        JavaUtils.attemptRepeatedly(() -> {
            Assert.assertTrue(raftLog.getLastCommittedIndex() >= loggedCommitIndex);
            return null;
        }, 10, HUNDRED_MILLIS, id + "(commitIndex >= loggedCommitIndex)", LOG);
        JavaUtils.attemptRepeatedly(() -> {
            Assert.assertTrue(server.getInfo().getLastAppliedIndex() >= loggedCommitIndex);
            return null;
        }, 10, HUNDRED_MILLIS, id + "(lastAppliedIndex >= loggedCommitIndex)", LOG);
        LOG.info("{}: commitIndex={}, lastAppliedIndex={}", id, raftLog.getLastCommittedIndex(), server.getInfo().getLastAppliedIndex());
        cluster.killServer(id);
    }
}

Example 24 with RaftClient

use of org.apache.ratis.client.RaftClient in project incubator-ratis by apache.

the class ServerRestartTests method runTestRestartWithCorruptedLogEntry.

private void runTestRestartWithCorruptedLogEntry(CLUSTER cluster) throws Exception {
    // this is the only server
    final RaftServer.Division leader = RaftTestUtil.waitForLeader(cluster);
    final RaftPeerId id = leader.getId();
    // send a few messages
    final SimpleMessage[] messages = SimpleMessage.create(10);
    final SimpleMessage lastMessage = messages[messages.length - 1];
    try (final RaftClient client = cluster.createClient()) {
        for (SimpleMessage m : messages) {
            Assert.assertTrue(client.io().send(m).isSuccess());
        }
        // assert that the last message exists
        Assert.assertTrue(client.io().sendReadOnly(lastMessage).isSuccess());
    }
    final RaftLog log = leader.getRaftLog();
    final long size = TestSegmentedRaftLog.getOpenSegmentSize(log);
    leader.getRaftServer().close();
    // corrupt the log
    final File openLogFile = JavaUtils.attemptRepeatedly(() -> getOpenLogFile(leader), 10, HUNDRED_MILLIS, id + "-getOpenLogFile", LOG);
    try (final RandomAccessFile raf = new RandomAccessFile(openLogFile, "rw")) {
        final long mid = size / 2;
        raf.seek(mid);
        for (long i = mid; i < size; i++) {
            raf.write(0);
        }
    }
    // after the log is corrupted and the server is restarted, the last entry should no longer exist.
    cluster.restartServer(id, false);
    testFailureCase("last-entry-not-found", () -> {
        try (final RaftClient client = cluster.createClient()) {
            client.io().sendReadOnly(lastMessage);
        }
    }, StateMachineException.class, IndexOutOfBoundsException.class);
}

Example 25 with RaftClient

use of org.apache.ratis.client.RaftClient in project incubator-ratis by apache.

the class TestRaftServerWithGrpc method runTestLeaderRestart.

void runTestLeaderRestart(MiniRaftClusterWithGrpc cluster) throws Exception {
    final RaftServer.Division leader = RaftTestUtil.waitForLeader(cluster);
    try (final RaftClient client = cluster.createClient()) {
        // send a request to make sure leader is ready
        final CompletableFuture<RaftClientReply> f = client.async().send(new SimpleMessage("testing"));
        Assert.assertTrue(f.get().isSuccess());
    }
    try (final RaftClient client = cluster.createClient()) {
        final RaftClientRpc rpc = client.getClientRpc();
        final AtomicLong seqNum = new AtomicLong();
        {
            // send a request using rpc directly
            final RaftClientRequest request = newRaftClientRequest(client, leader.getId(), seqNum.incrementAndGet());
            final CompletableFuture<RaftClientReply> f = rpc.sendRequestAsync(request);
            Assert.assertTrue(f.get().isSuccess());
        }
        // send another request which will be blocked
        final SimpleStateMachine4Testing stateMachine = SimpleStateMachine4Testing.get(leader);
        stateMachine.blockStartTransaction();
        final RaftClientRequest requestBlocked = newRaftClientRequest(client, leader.getId(), seqNum.incrementAndGet());
        final CompletableFuture<RaftClientReply> futureBlocked = rpc.sendRequestAsync(requestBlocked);
        // change leader
        RaftTestUtil.changeLeader(cluster, leader.getId());
        Assert.assertNotEquals(RaftPeerRole.LEADER, leader.getInfo().getCurrentRole());
        // the blocked request should fail
        testFailureCase("request should fail", futureBlocked::get, ExecutionException.class, AlreadyClosedException.class);
        stateMachine.unblockStartTransaction();
        // send one more request which should timeout.
        final RaftClientRequest requestTimeout = newRaftClientRequest(client, leader.getId(), seqNum.incrementAndGet());
        rpc.handleException(leader.getId(), new Exception(), true);
        final CompletableFuture<RaftClientReply> f = rpc.sendRequestAsync(requestTimeout);
        testFailureCase("request should timeout", f::get, ExecutionException.class, TimeoutIOException.class);
    }
}