Examples with Leader org.apache.zookeeper.server.quorum.Leader used on opensource projects

Example 1 with Leader

use of org.apache.zookeeper.server.quorum.Leader in project zookeeper by apache.

the class MonitorCommand method commandRun.

@Override
public void commandRun() {
    if (!isZKServerRunning()) {
        pw.println(ZK_NOT_SERVING);
        return;
    }
    ZKDatabase zkdb = zkServer.getZKDatabase();
    ServerStats stats = zkServer.serverStats();
    print("version", Version.getFullVersion());
    print("avg_latency", stats.getAvgLatency());
    print("max_latency", stats.getMaxLatency());
    print("min_latency", stats.getMinLatency());
    print("packets_received", stats.getPacketsReceived());
    print("packets_sent", stats.getPacketsSent());
    print("num_alive_connections", stats.getNumAliveClientConnections());
    print("outstanding_requests", stats.getOutstandingRequests());
    print("server_state", stats.getServerState());
    print("znode_count", zkdb.getNodeCount());
    print("watch_count", zkdb.getDataTree().getWatchCount());
    print("ephemerals_count", zkdb.getDataTree().getEphemeralsCount());
    print("approximate_data_size", zkdb.getDataTree().approximateDataSize());
    OSMXBean osMbean = new OSMXBean();
    if (osMbean != null && osMbean.getUnix() == true) {
        print("open_file_descriptor_count", osMbean.getOpenFileDescriptorCount());
        print("max_file_descriptor_count", osMbean.getMaxFileDescriptorCount());
    }
    if (stats.getServerState().equals("leader")) {
        Leader leader = ((LeaderZooKeeperServer) zkServer).getLeader();
        print("followers", leader.getLearners().size());
        print("synced_followers", leader.getForwardingFollowers().size());
        print("pending_syncs", leader.getNumPendingSyncs());
        print("last_proposal_size", leader.getProposalStats().getLastProposalSize());
        print("max_proposal_size", leader.getProposalStats().getMaxProposalSize());
        print("min_proposal_size", leader.getProposalStats().getMinProposalSize());
    }
}

Example 2 with Leader

use of org.apache.zookeeper.server.quorum.Leader in project zookeeper by apache.

the class QuorumTest method testLeaderShutdown.

@Test
public void testLeaderShutdown() throws IOException, InterruptedException, KeeperException {
    ZooKeeper zk = new DisconnectableZooKeeper(qb.hostPort, ClientBase.CONNECTION_TIMEOUT, DummyWatcher.INSTANCE);
    zk.create("/blah", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    zk.create("/blah/blah", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    Leader leader = qb.s1.leader;
    if (leader == null) {
        leader = qb.s2.leader;
    }
    if (leader == null) {
        leader = qb.s3.leader;
    }
    if (leader == null) {
        leader = qb.s4.leader;
    }
    if (leader == null) {
        leader = qb.s5.leader;
    }
    assertNotNull(leader);
    for (int i = 0; i < 5000; i++) {
        zk.setData("/blah/blah", new byte[0], -1, (rc, path, ctx, stat) -> {
            counter++;
            if (rc != 0) {
                errors++;
            }
        }, null);
    }
    for (LearnerHandler f : leader.getForwardingFollowers()) {
        f.getSocket().shutdownInput();
    }
    for (int i = 0; i < 5000; i++) {
        zk.setData("/blah/blah", new byte[0], -1, (rc, path, ctx, stat) -> {
            counter++;
            if (rc != 0) {
                errors++;
            }
        }, null);
    }
    // check if all the followers are alive
    assertTrue(qb.s1.isAlive());
    assertTrue(qb.s2.isAlive());
    assertTrue(qb.s3.isAlive());
    assertTrue(qb.s4.isAlive());
    assertTrue(qb.s5.isAlive());
    zk.close();
}

Example 3 with Leader

use of org.apache.zookeeper.server.quorum.Leader in project zookeeper by apache.

the class FollowerResyncConcurrencyTest method testResyncByDiffAfterFollowerCrashes.

/**
 * This test:
 * Starts up 3 ZKs. The non-leader ZKs are writing to cluster
 * Shut down one of the non-leader ZKs.
 * Restart after sessions have expired but less than 500 txns have taken place (get a diff)
 * Shut down immediately after restarting, start running separate thread with other transactions
 * Restart to a diff while transactions are running in leader
 *
 * Before fixes for ZOOKEEPER-962, restarting off of diff could get an inconsistent view of data missing transactions that
 * completed during diff syncing. Follower would also be considered "restarted" before all forwarded transactions
 * were completely processed, so restarting would cause a snap file with a too-high zxid to be written, and transactions
 * would be missed
 *
 * This test should pretty reliably catch the failure of restarting the server before all diff messages have been processed,
 * however, due to the transient nature of the system it may not catch failures due to concurrent processing of transactions
 * during the leader's diff forwarding.
 *
 * @throws IOException
 * @throws InterruptedException
 * @throws KeeperException
 * @throws Throwable
 */
@Test
public void testResyncByDiffAfterFollowerCrashes() throws IOException, InterruptedException, KeeperException, Throwable {
    final Semaphore sem = new Semaphore(0);
    QuorumUtil qu = new QuorumUtil(1);
    qu.startAll();
    CountdownWatcher watcher1 = new CountdownWatcher();
    CountdownWatcher watcher2 = new CountdownWatcher();
    CountdownWatcher watcher3 = new CountdownWatcher();
    int index = 1;
    while (qu.getPeer(index).peer.leader == null) {
        index++;
    }
    Leader leader = qu.getPeer(index).peer.leader;
    assertNotNull(leader);
    /* Reusing the index variable to select a follower to connect to */
    index = (index == 1) ? 2 : 1;
    LOG.info("Connecting to follower: {}", index);
    final ZooKeeper zk1 = createClient(qu.getPeer(index).peer.getClientPort(), watcher1);
    LOG.info("zk1 has session id 0x{}", Long.toHexString(zk1.getSessionId()));
    final ZooKeeper zk2 = createClient(qu.getPeer(index).peer.getClientPort(), watcher2);
    LOG.info("zk2 has session id 0x{}", Long.toHexString(zk2.getSessionId()));
    final ZooKeeper zk3 = createClient(qu.getPeer(3).peer.getClientPort(), watcher3);
    LOG.info("zk3 has session id 0x{}", Long.toHexString(zk3.getSessionId()));
    zk1.create("/first", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    zk2.create("/mybar", null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
    final AtomicBoolean runNow = new AtomicBoolean(false);
    Thread mytestfooThread = new Thread(new Runnable() {

        @Override
        public void run() {
            int inSyncCounter = 0;
            while (inSyncCounter < 400) {
                if (runNow.get()) {
                    zk3.create("/mytestfoo", null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL, (rc, path, ctx, name) -> {
                        pending.decrementAndGet();
                        counter.incrementAndGet();
                        if (rc != 0) {
                            errors.incrementAndGet();
                        }
                        if (counter.get() > 7300) {
                            sem.release();
                        }
                    }, null);
                    pending.incrementAndGet();
                    try {
                        Thread.sleep(10);
                    } catch (Exception e) {
                    }
                    inSyncCounter++;
                } else {
                    Thread.yield();
                }
            }
        }
    });
    mytestfooThread.start();
    for (int i = 0; i < 5000; i++) {
        zk2.create("/mybar", null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL, (rc, path, ctx, name) -> {
            pending.decrementAndGet();
            counter.incrementAndGet();
            if (rc != 0) {
                errors.incrementAndGet();
            }
            if (counter.get() > 7300) {
                sem.release();
            }
        }, null);
        pending.incrementAndGet();
        if (i == 1000) {
            qu.shutdown(index);
            Thread.sleep(1100);
            LOG.info("Shutting down s1");
        }
        if (i == 1100 || i == 1150 || i == 1200) {
            Thread.sleep(1000);
        }
        if (i == 1200) {
            qu.startThenShutdown(index);
            runNow.set(true);
            qu.restart(index);
            LOG.info("Setting up server: {}", index);
        }
        if (i >= 1000 && i % 2 == 0) {
            zk3.create("/newbaz", null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL, (rc, path, ctx, name) -> {
                pending.decrementAndGet();
                counter.incrementAndGet();
                if (rc != 0) {
                    errors.incrementAndGet();
                }
                if (counter.get() > 7300) {
                    sem.release();
                }
            }, null);
            pending.incrementAndGet();
        }
        if (i == 1050 || i == 1100 || i == 1150) {
            Thread.sleep(1000);
        }
    }
    // Wait until all updates return
    if (!sem.tryAcquire(ClientBase.CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS)) {
        LOG.warn("Did not aquire semaphore fast enough");
    }
    mytestfooThread.join(ClientBase.CONNECTION_TIMEOUT);
    if (mytestfooThread.isAlive()) {
        LOG.error("mytestfooThread is still alive");
    }
    assertTrue(waitForPendingRequests(60));
    assertTrue(waitForSync(qu, index, 10));
    // Verify that server is following and has the same epoch as the leader
    verifyState(qu, index, leader);
    zk1.close();
    zk2.close();
    zk3.close();
    qu.shutdownAll();
}

Example 4 with Leader

use of org.apache.zookeeper.server.quorum.Leader in project zookeeper by apache.

the class QuorumTest method testNoLogBeforeLeaderEstablishment.

/**
     * Tests if closeSession can be logged before a leader gets established, which
     * could lead to a locked-out follower (see ZOOKEEPER-790). 
     * 
     * The test works as follows. It has a client connecting to a follower f and
     * sending batches of 1,000 updates. The goal is that f has a zxid higher than
     * all other servers in the initial leader election. This way we can crash and
     * recover the follower so that the follower believes it is the leader once it
     * recovers (LE optimization: once a server receives a message from all other 
     * servers, it picks a leader.
     * 
     * It also makes the session timeout very short so that we force the false 
     * leader to close the session and write it to the log in the buggy code (before 
     * ZOOKEEPER-790). Once f drops leadership and finds the current leader, its epoch
     * is higher, and it rejects the leader. Now, if we prevent the leader from closing
     * the session by only starting up (see Leader.lead()) once it obtains a quorum of 
     * supporters, then f will find the current leader and support it because it won't
     * have a highe epoch.
     * 
     */
@Test
public void testNoLogBeforeLeaderEstablishment() throws Exception {
    final Semaphore sem = new Semaphore(0);
    qu = new QuorumUtil(2, 10);
    qu.startQuorum();
    int index = 1;
    while (qu.getPeer(index).peer.leader == null) index++;
    Leader leader = qu.getPeer(index).peer.leader;
    Assert.assertNotNull(leader);
    /*
         * Reusing the index variable to select a follower to connect to
         */
    index = (index == 1) ? 2 : 1;
    ZooKeeper zk = new DisconnectableZooKeeper("127.0.0.1:" + qu.getPeer(index).peer.getClientPort(), ClientBase.CONNECTION_TIMEOUT, new Watcher() {

        public void process(WatchedEvent event) {
        }
    });
    zk.create("/blah", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    for (int i = 0; i < 50000; i++) {
        zk.setData("/blah", new byte[0], -1, new AsyncCallback.StatCallback() {

            public void processResult(int rc, String path, Object ctx, Stat stat) {
                counter++;
                if (rc != 0) {
                    errors++;
                }
                if (counter == 20000) {
                    sem.release();
                }
            }
        }, null);
        if (i == 5000) {
            qu.shutdown(index);
            LOG.info("Shutting down s1");
        }
        if (i == 12000) {
            qu.start(index);
            LOG.info("Setting up server: " + index);
        }
        if ((i % 1000) == 0) {
            Thread.sleep(500);
        }
    }
    // Wait until all updates return
    sem.tryAcquire(15, TimeUnit.SECONDS);
    // Verify that server is following and has the same epoch as the leader
    Assert.assertTrue("Not following", qu.getPeer(index).peer.follower != null);
    long epochF = (qu.getPeer(index).peer.getActiveServer().getZxid() >> 32L);
    long epochL = (leader.getEpoch() >> 32L);
    Assert.assertTrue("Zxid: " + qu.getPeer(index).peer.getActiveServer().getZxid() + "Current epoch: " + epochF, epochF == epochL);
    zk.close();
}

Example 5 with Leader

use of org.apache.zookeeper.server.quorum.Leader in project zookeeper by apache.

the class QuorumOracleMajTest method dropConnectionTest.

private void dropConnectionTest(QuorumPeer s, int leader) {
    Leader.Proposal p = new Leader.Proposal();
    p.addQuorumVerifier(s.getQuorumVerifier());
    ArrayList<LearnerHandler> fake = new ArrayList<>();
    LearnerHandler f = null;
    fake.add(f);
    s.getQuorumVerifier().updateNeedOracle(fake);
    // still have valid followers, the oracle should not take place
    assertEquals(false, s.getQuorumVerifier().getNeedOracle());
    fake.remove(0);
    s.getQuorumVerifier().updateNeedOracle(fake);
    // lose all of followers, the oracle should take place
    assertEquals(true, s.getQuorumVerifier().getNeedOracle());
    // when leader is 1, we expect false.
    // when leader is 2, we expect true.
    assertEquals(leader != 1, p.hasAllQuorums());
}