Examples with PeerChanges org.apache.ratis.MiniRaftCluster.PeerChanges used on opensource projects

Example 1 with PeerChanges

use of org.apache.ratis.MiniRaftCluster.PeerChanges in project incubator-ratis by apache.

the class RetryCacheTests method testRetryOnNewLeader.

/**
 * Test retry while the leader changes to another peer
 */
@Test
public void testRetryOnNewLeader() throws Exception {
    final MiniRaftCluster cluster = getCluster();
    RaftTestUtil.waitForLeader(cluster);
    final RaftPeerId leaderId = cluster.getLeaderAndSendFirstMessage().getId();
    final RaftClient client = cluster.createClient(leaderId);
    RaftClientRpc rpc = client.getClientRpc();
    final long callId = 999;
    final long seqNum = 111;
    RaftClientRequest r = cluster.newRaftClientRequest(client.getId(), leaderId, callId, seqNum, new SimpleMessage("message"));
    RaftClientReply reply = rpc.sendRequest(r);
    Assert.assertEquals(callId, reply.getCallId());
    Assert.assertTrue(reply.isSuccess());
    long oldLastApplied = cluster.getLeader().getState().getLastAppliedIndex();
    // trigger the reconfiguration, make sure the original leader is kicked out
    PeerChanges change = cluster.addNewPeers(2, true);
    RaftPeer[] allPeers = cluster.removePeers(2, true, asList(change.newPeers)).allPeersInNewConf;
    // trigger setConfiguration
    cluster.setConfiguration(allPeers);
    RaftTestUtil.waitForLeader(cluster);
    final RaftPeerId newLeaderId = cluster.getLeader().getId();
    Assert.assertNotEquals(leaderId, newLeaderId);
    // same clientId and callId in the request
    r = cluster.newRaftClientRequest(client.getId(), newLeaderId, callId, seqNum, new SimpleMessage("message"));
    for (int i = 0; i < 10; i++) {
        try {
            reply = rpc.sendRequest(r);
            LOG.info("successfully sent out the retry request_" + i);
            Assert.assertEquals(client.getId(), reply.getClientId());
            Assert.assertEquals(callId, reply.getCallId());
            Assert.assertTrue(reply.isSuccess());
        } catch (Exception e) {
            LOG.info("hit exception while retrying the same request: " + e);
        }
        Thread.sleep(100);
    }
    // check the new leader and make sure the retry did not get committed
    Assert.assertEquals(oldLastApplied + 3, cluster.getLeader().getState().getLastAppliedIndex());
    client.close();
}

Example 2 with PeerChanges

use of org.apache.ratis.MiniRaftCluster.PeerChanges in project incubator-ratis by apache.

the class RaftReconfigurationBaseTest method testRevertConfigurationChange.

/**
 * Test a scenario where the follower truncates its log entries which causes
 * configuration change.
 */
@Test
public void testRevertConfigurationChange() throws Exception {
    LOG.info("Start testRevertConfigurationChange");
    RaftLog log2 = null;
    final MiniRaftCluster cluster = getCluster(5);
    try {
        cluster.start();
        RaftTestUtil.waitForLeader(cluster);
        final RaftServerImpl leader = cluster.getLeader();
        final RaftPeerId leaderId = leader.getId();
        final RaftLog log = leader.getState().getLog();
        log2 = log;
        Thread.sleep(1000);
        // we block the incoming msg for the leader and block its requests to
        // followers, so that we force the leader change and the old leader will
        // not know
        LOG.info("start blocking the leader");
        BlockRequestHandlingInjection.getInstance().blockReplier(leaderId.toString());
        cluster.setBlockRequestsFrom(leaderId.toString(), true);
        PeerChanges change = cluster.removePeers(1, false, new ArrayList<>());
        AtomicBoolean gotNotLeader = new AtomicBoolean(false);
        final Thread clientThread = new Thread(() -> {
            try (final RaftClient client = cluster.createClient(leaderId)) {
                LOG.info("client starts to change conf");
                final RaftClientRpc sender = client.getClientRpc();
                RaftClientReply reply = sender.sendRequest(cluster.newSetConfigurationRequest(client.getId(), leaderId, change.allPeersInNewConf));
                if (reply.getNotLeaderException() != null) {
                    gotNotLeader.set(true);
                }
            } catch (IOException e) {
                LOG.warn("Got unexpected exception when client1 changes conf", e);
            }
        });
        clientThread.start();
        // find CONFIGURATIONENTRY, there may be NOOP before and after it.
        final long confIndex = JavaUtils.attempt(() -> {
            final long last = log.getLastEntryTermIndex().getIndex();
            for (long i = 1; i <= last; i++) {
                if (log.get(i).getLogEntryBodyCase() == CONFIGURATIONENTRY) {
                    return i;
                }
            }
            throw new Exception("CONFIGURATIONENTRY not found: last=" + last);
        }, 10, 500, "confIndex", LOG);
        // wait till the old leader persist the new conf
        JavaUtils.attempt(() -> log.getLatestFlushedIndex() >= confIndex, 10, 500L, "FLUSH", LOG);
        final long committed = log.getLastCommittedIndex();
        Assert.assertTrue(committed < confIndex);
        // unblock the old leader
        BlockRequestHandlingInjection.getInstance().unblockReplier(leaderId.toString());
        cluster.setBlockRequestsFrom(leaderId.toString(), false);
        // the client should get NotLeaderException
        clientThread.join(5000);
        Assert.assertTrue(gotNotLeader.get());
        // the old leader should have truncated the setConf from the log
        JavaUtils.attempt(() -> log.getLastCommittedIndex() >= confIndex, 10, 500L, "COMMIT", LOG);
        Assert.assertEquals(NOOP, log.get(confIndex).getLogEntryBodyCase());
        log2 = null;
    } finally {
        RaftStorageTestUtils.printLog(log2, s -> LOG.info(s));
        cluster.shutdown();
    }
}

Example 3 with PeerChanges

use of org.apache.ratis.MiniRaftCluster.PeerChanges in project incubator-ratis by apache.

the class RaftReconfigurationBaseTest method testBootstrapReconf.

@Test
public void testBootstrapReconf() throws Exception {
    LOG.info("Start testBootstrapReconf");
    // originally 3 peers
    final MiniRaftCluster cluster = getCluster(3);
    cluster.start();
    try {
        RaftTestUtil.waitForLeader(cluster);
        final RaftPeerId leaderId = cluster.getLeader().getId();
        final RaftClient client = cluster.createClient(leaderId);
        // submit some msgs before reconf
        for (int i = 0; i < getStagingGap() * 2; i++) {
            RaftClientReply reply = client.send(new SimpleMessage("m" + i));
            Assert.assertTrue(reply.isSuccess());
        }
        PeerChanges c1 = cluster.addNewPeers(2, true);
        LOG.info("Start changing the configuration: {}", asList(c1.allPeersInNewConf));
        final AtomicReference<Boolean> success = new AtomicReference<>();
        Thread clientThread = new Thread(() -> {
            try {
                RaftClientReply reply = client.setConfiguration(c1.allPeersInNewConf);
                success.set(reply.isSuccess());
                client.close();
            } catch (IOException ioe) {
                LOG.error("FAILED", ioe);
            }
        });
        clientThread.start();
        Thread.sleep(5000);
        LOG.info(cluster.printServers());
        assertSuccess(success);
        final RaftLog leaderLog = cluster.getLeader().getState().getLog();
        for (RaftPeer newPeer : c1.newPeers) {
            Assert.assertArrayEquals(leaderLog.getEntries(0, Long.MAX_VALUE), cluster.getServer(newPeer.getId()).getImpl().getState().getLog().getEntries(0, Long.MAX_VALUE));
        }
    } finally {
        cluster.shutdown();
    }
}

Example 4 with PeerChanges

use of org.apache.ratis.MiniRaftCluster.PeerChanges in project incubator-ratis by apache.

the class RaftReconfigurationBaseTest method testAddRemovePeers.

private void testAddRemovePeers(boolean leaderStepdown) throws Exception {
    MiniRaftCluster cluster = getCluster(5);
    cluster.start();
    try {
        RaftTestUtil.waitForLeader(cluster);
        PeerChanges change = cluster.addNewPeers(2, true);
        RaftPeer[] allPeers = cluster.removePeers(2, leaderStepdown, asList(change.newPeers)).allPeersInNewConf;
        // trigger setConfiguration
        cluster.setConfiguration(allPeers);
        // wait for the new configuration to take effect
        waitAndCheckNewConf(cluster, allPeers, 2, null);
    } finally {
        cluster.shutdown();
    }
}

Example 5 with PeerChanges

use of org.apache.ratis.MiniRaftCluster.PeerChanges in project incubator-ratis by apache.

the class RaftReconfigurationBaseTest method testReconfTwice.

@Test(timeout = 30000)
public void testReconfTwice() throws Exception {
    LOG.info("Start testReconfTwice");
    final MiniRaftCluster cluster = getCluster(3);
    cluster.start();
    try {
        RaftTestUtil.waitForLeader(cluster);
        final RaftPeerId leaderId = cluster.getLeader().getId();
        final RaftClient client = cluster.createClient(leaderId);
        // submit some msgs before reconf
        for (int i = 0; i < getStagingGap() * 2; i++) {
            RaftClientReply reply = client.send(new SimpleMessage("m" + i));
            Assert.assertTrue(reply.isSuccess());
        }
        final AtomicBoolean reconf1 = new AtomicBoolean(false);
        final AtomicBoolean reconf2 = new AtomicBoolean(false);
        final AtomicReference<RaftPeer[]> finalPeers = new AtomicReference<>(null);
        final AtomicReference<RaftPeer[]> deadPeers = new AtomicReference<>(null);
        CountDownLatch latch = new CountDownLatch(1);
        Thread clientThread = new Thread(() -> {
            try {
                PeerChanges c1 = cluster.addNewPeers(2, true);
                LOG.info("Start changing the configuration: {}", asList(c1.allPeersInNewConf));
                RaftClientReply reply = client.setConfiguration(c1.allPeersInNewConf);
                reconf1.set(reply.isSuccess());
                PeerChanges c2 = cluster.removePeers(2, true, asList(c1.newPeers));
                finalPeers.set(c2.allPeersInNewConf);
                deadPeers.set(c2.removedPeers);
                LOG.info("Start changing the configuration again: {}", asList(c2.allPeersInNewConf));
                reply = client.setConfiguration(c2.allPeersInNewConf);
                reconf2.set(reply.isSuccess());
                latch.countDown();
                client.close();
            } catch (IOException ignored) {
            }
        });
        clientThread.start();
        latch.await();
        Assert.assertTrue(reconf1.get());
        Assert.assertTrue(reconf2.get());
        waitAndCheckNewConf(cluster, finalPeers.get(), 2, null);
        // check configuration manager's internal state
        // each reconf will generate two configurations: (old, new) and (new)
        cluster.getServerAliveStream().forEach(server -> {
            ConfigurationManager confManager = (ConfigurationManager) Whitebox.getInternalState(server.getState(), "configurationManager");
            // each reconf will generate two configurations: (old, new) and (new)
            Assert.assertEquals(5, confManager.numOfConf());
        });
    } finally {
        cluster.shutdown();
    }
}