Search in sources :

Example 1 with ExponentialBackoffTimeoutStrategy

use of org.apache.ignite.raft.jraft.util.ExponentialBackoffTimeoutStrategy in project ignite-3 by apache.

the class ItNodeTest method testElectionTimeoutAutoAdjustWhenBlockedMessages.

private void testElectionTimeoutAutoAdjustWhenBlockedMessages(BiPredicate<Object, String> blockingPredicate) throws Exception {
    List<PeerId> peers = TestUtils.generatePeers(4);
    int maxElectionRoundsWithoutAdjusting = 3;
    cluster = new TestCluster("unittest", dataPath, peers, new LinkedHashSet<>(), ELECTION_TIMEOUT_MILLIS, opts -> opts.setElectionTimeoutStrategy(new ExponentialBackoffTimeoutStrategy(11_000, maxElectionRoundsWithoutAdjusting)), testInfo);
    for (PeerId peer : peers) {
        assertTrue(cluster.start(peer.getEndpoint()));
    }
    cluster.waitLeader();
    Node leader = cluster.getLeader();
    int initElectionTimeout = leader.getOptions().getElectionTimeoutMs();
    LOG.warn("Current leader {}, electTimeout={}", leader.getNodeId().getPeerId(), leader.getOptions().getElectionTimeoutMs());
    List<Node> followers = cluster.getFollowers();
    for (Node follower : followers) {
        NodeImpl follower0 = (NodeImpl) follower;
        assertEquals(initElectionTimeout, follower0.getOptions().getElectionTimeoutMs());
    }
    blockMessagesOnFollowers(followers, blockingPredicate);
    LOG.warn("Stop leader {}, curTerm={}", leader.getNodeId().getPeerId(), ((NodeImpl) leader).getCurrentTerm());
    assertTrue(cluster.stop(leader.getNodeId().getPeerId().getEndpoint()));
    assertNull(cluster.getLeader());
    assertTrue(waitForCondition(() -> followers.stream().allMatch(f -> f.getOptions().getElectionTimeoutMs() > initElectionTimeout), (long) maxElectionRoundsWithoutAdjusting * // need to multiply to 2 because stepDown happens after voteTimer timeout
    (initElectionTimeout + followers.get(0).getOptions().getRaftOptions().getMaxElectionDelayMs()) * 2));
    stopBlockingMessagesOnFollowers(followers);
    // elect new leader
    cluster.waitLeader();
    leader = cluster.getLeader();
    LOG.info("Elected new leader is {}, curTerm={}", leader.getLeaderId(), ((NodeImpl) leader).getCurrentTerm());
    assertTrue(waitForCondition(() -> followers.stream().allMatch(f -> f.getOptions().getElectionTimeoutMs() == initElectionTimeout), 3_000));
}
Also used : LinkedHashSet(java.util.LinkedHashSet) SynchronizedClosure(org.apache.ignite.raft.jraft.closure.SynchronizedClosure) BeforeEach(org.junit.jupiter.api.BeforeEach) Arrays(java.util.Arrays) ExponentialBackoffTimeoutStrategy(org.apache.ignite.raft.jraft.util.ExponentialBackoffTimeoutStrategy) TaskClosure(org.apache.ignite.raft.jraft.closure.TaskClosure) SnapshotThrottle(org.apache.ignite.raft.jraft.storage.SnapshotThrottle) Disabled(org.junit.jupiter.api.Disabled) BooleanSupplier(java.util.function.BooleanSupplier) AfterAll(org.junit.jupiter.api.AfterAll) Future(java.util.concurrent.Future) ExtendWith(org.junit.jupiter.api.extension.ExtendWith) BeforeAll(org.junit.jupiter.api.BeforeAll) Path(java.nio.file.Path) Collections.synchronizedList(java.util.Collections.synchronizedList) Set(java.util.Set) Assertions.assertNotSame(org.junit.jupiter.api.Assertions.assertNotSame) ELECTION_TIMEOUT_MILLIS(org.apache.ignite.raft.jraft.core.TestCluster.ELECTION_TIMEOUT_MILLIS) TestUtils(org.apache.ignite.raft.jraft.test.TestUtils) ExecutorServiceHelper(org.apache.ignite.raft.jraft.util.ExecutorServiceHelper) Executors(java.util.concurrent.Executors) CountDownLatch(java.util.concurrent.CountDownLatch) Stream(java.util.stream.Stream) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Endpoint(org.apache.ignite.raft.jraft.util.Endpoint) RaftError(org.apache.ignite.raft.jraft.error.RaftError) RpcServer(org.apache.ignite.raft.jraft.rpc.RpcServer) Assertions.fail(org.junit.jupiter.api.Assertions.fail) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) StateMachine(org.apache.ignite.raft.jraft.StateMachine) RaftException(org.apache.ignite.raft.jraft.error.RaftException) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) ArrayList(java.util.ArrayList) Task(org.apache.ignite.raft.jraft.entity.Task) IgniteRpcServer(org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer) TestScaleCubeClusterServiceFactory(org.apache.ignite.network.scalecube.TestScaleCubeClusterServiceFactory) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) LinkedHashSet(java.util.LinkedHashSet) NodeManager(org.apache.ignite.raft.jraft.NodeManager) LogIndexOutOfBoundsException(org.apache.ignite.raft.jraft.error.LogIndexOutOfBoundsException) RaftGroupService(org.apache.ignite.raft.jraft.RaftGroupService) LogNotFoundException(org.apache.ignite.raft.jraft.error.LogNotFoundException) DefaultRaftClientService(org.apache.ignite.raft.jraft.rpc.impl.core.DefaultRaftClientService) File(java.io.File) Assertions.assertSame(org.junit.jupiter.api.Assertions.assertSame) WorkDirectory(org.apache.ignite.internal.testframework.WorkDirectory) NetworkAddress(org.apache.ignite.network.NetworkAddress) Assertions.assertArrayEquals(org.junit.jupiter.api.Assertions.assertArrayEquals) AfterEach(org.junit.jupiter.api.AfterEach) ConsoleReporter(com.codahale.metrics.ConsoleReporter) RaftOptions(org.apache.ignite.raft.jraft.option.RaftOptions) EnumOutter(org.apache.ignite.raft.jraft.entity.EnumOutter) Assertions.assertNotEquals(org.junit.jupiter.api.Assertions.assertNotEquals) TestUtils.sender(org.apache.ignite.raft.jraft.test.TestUtils.sender) IgniteLogger(org.apache.ignite.lang.IgniteLogger) ByteBuffer(java.nio.ByteBuffer) ReadOnlyOption(org.apache.ignite.raft.jraft.option.ReadOnlyOption) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) NodeFinder(org.apache.ignite.network.NodeFinder) Node(org.apache.ignite.raft.jraft.Node) JoinableClosure(org.apache.ignite.raft.jraft.closure.JoinableClosure) TestIgniteRpcServer(org.apache.ignite.raft.jraft.rpc.TestIgniteRpcServer) SnapshotReader(org.apache.ignite.raft.jraft.storage.snapshot.SnapshotReader) Status(org.apache.ignite.raft.jraft.Status) BootstrapOptions(org.apache.ignite.raft.jraft.option.BootstrapOptions) TestInfo(org.junit.jupiter.api.TestInfo) Utils(org.apache.ignite.raft.jraft.util.Utils) Test(org.junit.jupiter.api.Test) List(java.util.List) StaticNodeFinder(org.apache.ignite.network.StaticNodeFinder) RpcClientEx(org.apache.ignite.raft.jraft.rpc.RpcClientEx) Configuration(org.apache.ignite.raft.jraft.conf.Configuration) ReadIndexClosure(org.apache.ignite.raft.jraft.closure.ReadIndexClosure) Bits(org.apache.ignite.raft.jraft.util.Bits) ClusterServiceTestUtils(org.apache.ignite.utils.ClusterServiceTestUtils) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) FixedThreadsExecutorGroup(org.apache.ignite.raft.jraft.util.concurrent.FixedThreadsExecutorGroup) AtomicReference(java.util.concurrent.atomic.AtomicReference) NodeOptions(org.apache.ignite.raft.jraft.option.NodeOptions) HashSet(java.util.HashSet) BiPredicate(java.util.function.BiPredicate) Iterator(org.apache.ignite.raft.jraft.Iterator) ExecutorService(java.util.concurrent.ExecutorService) JRaftUtils(org.apache.ignite.raft.jraft.JRaftUtils) UTF_8(java.nio.charset.StandardCharsets.UTF_8) RpcRequests(org.apache.ignite.raft.jraft.rpc.RpcRequests) IgniteRpcClient(org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcClient) TimeUnit(java.util.concurrent.TimeUnit) PeerId(org.apache.ignite.raft.jraft.entity.PeerId) Collectors.toList(java.util.stream.Collectors.toList) ThroughputSnapshotThrottle(org.apache.ignite.raft.jraft.storage.snapshot.ThroughputSnapshotThrottle) ClusterService(org.apache.ignite.network.ClusterService) WorkDirectoryExtension(org.apache.ignite.internal.testframework.WorkDirectoryExtension) UserLog(org.apache.ignite.raft.jraft.entity.UserLog) Collections(java.util.Collections) Node(org.apache.ignite.raft.jraft.Node) Endpoint(org.apache.ignite.raft.jraft.util.Endpoint) PeerId(org.apache.ignite.raft.jraft.entity.PeerId) ExponentialBackoffTimeoutStrategy(org.apache.ignite.raft.jraft.util.ExponentialBackoffTimeoutStrategy)

Example 2 with ExponentialBackoffTimeoutStrategy

use of org.apache.ignite.raft.jraft.util.ExponentialBackoffTimeoutStrategy in project ignite-3 by apache.

the class TestCluster method start.

public boolean start(Endpoint listenAddr, boolean emptyPeers, int snapshotIntervalSecs, boolean enableMetrics, SnapshotThrottle snapshotThrottle, RaftOptions raftOptions, int priority) throws IOException {
    this.lock.lock();
    try {
        if (this.serverMap.get(listenAddr) != null) {
            return true;
        }
        // Start node in non shared pools mode. Pools will be managed by node itself.
        NodeOptions nodeOptions = new NodeOptions();
        nodeOptions.setServerName(listenAddr.toString());
        nodeOptions.setElectionTimeoutMs(this.electionTimeoutMs);
        nodeOptions.setEnableMetrics(enableMetrics);
        nodeOptions.setSnapshotThrottle(snapshotThrottle);
        nodeOptions.setSnapshotIntervalSecs(snapshotIntervalSecs);
        nodeOptions.setServiceFactory(this.raftServiceFactory);
        if (raftOptions != null) {
            nodeOptions.setRaftOptions(raftOptions);
        }
        String serverDataPath = this.dataPath + File.separator + listenAddr.toString().replace(':', '_');
        new File(serverDataPath).mkdirs();
        nodeOptions.setLogUri(serverDataPath + File.separator + "logs");
        nodeOptions.setRaftMetaUri(serverDataPath + File.separator + "meta");
        nodeOptions.setSnapshotUri(serverDataPath + File.separator + "snapshot");
        nodeOptions.setElectionPriority(priority);
        // Align rpc options with election timeout.
        nodeOptions.setRpcConnectTimeoutMs(this.electionTimeoutMs / 3);
        nodeOptions.setRpcDefaultTimeout(this.electionTimeoutMs / 2);
        // Reduce default threads count per test node.
        nodeOptions.setRaftRpcThreadPoolSize(Utils.cpus());
        nodeOptions.setTimerPoolSize(Utils.cpus() * 2);
        nodeOptions.setRpcProcessorThreadPoolSize(Utils.cpus() * 3);
        nodeOptions.setElectionTimeoutStrategy(new ExponentialBackoffTimeoutStrategy());
        MockStateMachine fsm = new MockStateMachine(listenAddr);
        nodeOptions.setFsm(fsm);
        if (!emptyPeers)
            nodeOptions.setInitialConf(new Configuration(this.peers, this.learners));
        List<NetworkAddress> addressList = (emptyPeers ? Stream.<PeerId>empty() : peers.stream()).map(PeerId::getEndpoint).map(JRaftUtils::addressFromEndpoint).collect(toList());
        NodeManager nodeManager = new NodeManager();
        ClusterService clusterService = ClusterServiceTestUtils.clusterService(testInfo, listenAddr.getPort(), new StaticNodeFinder(addressList), new TestScaleCubeClusterServiceFactory());
        var rpcClient = new IgniteRpcClient(clusterService);
        nodeOptions.setRpcClient(rpcClient);
        ExecutorService requestExecutor = JRaftUtils.createRequestExecutor(nodeOptions);
        var rpcServer = new TestIgniteRpcServer(clusterService, nodeManager, nodeOptions, requestExecutor);
        clusterService.start();
        if (optsClo != null)
            optsClo.accept(nodeOptions);
        RaftGroupService server = new RaftGroupService(this.name, new PeerId(listenAddr, 0, priority), nodeOptions, rpcServer, nodeManager) {

            @Override
            public synchronized void shutdown() {
                // This stop order is consistent with JRaftServerImpl
                rpcServer.shutdown();
                ExecutorServiceHelper.shutdownAndAwaitTermination(requestExecutor);
                super.shutdown();
                // Network service must be stopped after a node because raft initiates timeoutnowrequest on stop for faster
                // leader election.
                clusterService.stop();
            }
        };
        this.serverMap.put(listenAddr, server);
        Node node = server.start();
        this.fsms.put(new PeerId(listenAddr, 0), fsm);
        this.nodes.add((NodeImpl) node);
        return true;
    } finally {
        this.lock.unlock();
    }
}
Also used : TestIgniteRpcServer(org.apache.ignite.raft.jraft.rpc.TestIgniteRpcServer) Configuration(org.apache.ignite.raft.jraft.conf.Configuration) StaticNodeFinder(org.apache.ignite.network.StaticNodeFinder) IgniteRpcClient(org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcClient) RaftGroupService(org.apache.ignite.raft.jraft.RaftGroupService) Node(org.apache.ignite.raft.jraft.Node) NodeOptions(org.apache.ignite.raft.jraft.option.NodeOptions) NodeManager(org.apache.ignite.raft.jraft.NodeManager) ClusterService(org.apache.ignite.network.ClusterService) TestScaleCubeClusterServiceFactory(org.apache.ignite.network.scalecube.TestScaleCubeClusterServiceFactory) NetworkAddress(org.apache.ignite.network.NetworkAddress) ExecutorService(java.util.concurrent.ExecutorService) File(java.io.File) ExponentialBackoffTimeoutStrategy(org.apache.ignite.raft.jraft.util.ExponentialBackoffTimeoutStrategy) PeerId(org.apache.ignite.raft.jraft.entity.PeerId)

Aggregations

File (java.io.File)2 ExecutorService (java.util.concurrent.ExecutorService)2 ClusterService (org.apache.ignite.network.ClusterService)2 NetworkAddress (org.apache.ignite.network.NetworkAddress)2 StaticNodeFinder (org.apache.ignite.network.StaticNodeFinder)2 TestScaleCubeClusterServiceFactory (org.apache.ignite.network.scalecube.TestScaleCubeClusterServiceFactory)2 Node (org.apache.ignite.raft.jraft.Node)2 NodeManager (org.apache.ignite.raft.jraft.NodeManager)2 RaftGroupService (org.apache.ignite.raft.jraft.RaftGroupService)2 Configuration (org.apache.ignite.raft.jraft.conf.Configuration)2 PeerId (org.apache.ignite.raft.jraft.entity.PeerId)2 NodeOptions (org.apache.ignite.raft.jraft.option.NodeOptions)2 TestIgniteRpcServer (org.apache.ignite.raft.jraft.rpc.TestIgniteRpcServer)2 IgniteRpcClient (org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcClient)2 ExponentialBackoffTimeoutStrategy (org.apache.ignite.raft.jraft.util.ExponentialBackoffTimeoutStrategy)2 ConsoleReporter (com.codahale.metrics.ConsoleReporter)1 ByteBuffer (java.nio.ByteBuffer)1 UTF_8 (java.nio.charset.StandardCharsets.UTF_8)1 Path (java.nio.file.Path)1 ArrayList (java.util.ArrayList)1