Examples with ZooKeeper com.linkedin.d2.discovery.stores.zk.ZooKeeper used on opensource projects

Example 16 with ZooKeeper

use of com.linkedin.d2.discovery.stores.zk.ZooKeeper in project rest.li by linkedin.

the class TestD2ZKQuorumFailover method testD2WithQuorumLeaderPeerDown.

public void testD2WithQuorumLeaderPeerDown() throws Exception {
    setup();
    assertQuorumProcessAllRequests(D2_CONFIG_DATA);
    // Shutdown zk _quorum leader peer
    ZKPeer peer = _quorum.getQuorumLeader();
    if (peer != null) {
        int peerId = peer.getId();
        peer.shutdownPeer();
        // Verify requests are processed while leader goes down
        long start = System.currentTimeMillis();
        while (_quorum.getQuorumLeader() != null && _quorum.getQuorumLeader().equals(peer) && System.currentTimeMillis() < (start + TIMEOUT)) {
            assertQuorumProcessAllRequests(D2_CONFIG_DATA);
        }
        start = System.currentTimeMillis();
        while (_quorum.getQuorumLeader() == null && System.currentTimeMillis() < (start + TIMEOUT)) {
            // wait while zookeeper is establishing a new leader
            assertQuorumProcessAllRequests(D2_CONFIG_DATA);
        }
        if (_quorum.getQuorumLeader() == null) {
            _quorum.restartAll();
        } else {
            assertTrue(!_quorum.getQuorumLeader().equals(peer), "No new _quorum leader was established. New leader id=" + _quorum.getQuorumLeader().getId());
            // Restart leader peer
            _quorum.restart(peerId);
        }
        start = System.currentTimeMillis();
        peer = _quorum.get(peerId);
        // Verify requests are processed while peer is restarting
        while (System.currentTimeMillis() < (start + TIMEOUT) && (peer.getZKServer() == null || !_quorum.areAllPeersUp() || !peer.getZKServer().isRunning())) {
            assertQuorumProcessAllRequests(D2_CONFIG_DATA);
        }
        // After restart
        assertQuorumProcessAllRequests(D2_CONFIG_DATA);
    } else {
        fail("Quorum is unable to identify zk leader. No tests were executed.");
    }
}

Example 17 with ZooKeeper

use of com.linkedin.d2.discovery.stores.zk.ZooKeeper in project rest.li by linkedin.

the class EmailClientExample method main.

public static void main(String[] args) throws Exception {
    //get client configuration
    JSONObject json = parseConfig();
    String zkConnectString = (String) json.get("zkConnectString");
    Long zkSessionTimeout = (Long) json.get("zkSessionTimeout");
    String zkBasePath = (String) json.get("zkBasePath");
    Long zkStartupTimeout = (Long) json.get("zkStartupTimeout");
    Long zkLoadBalancerNotificationTimeout = (Long) json.get("zkLoadBalancerNotificationTimeout");
    String zkFlagFile = (String) json.get("zkFlagFile");
    String fsBasePath = (String) json.get("fsBasePath");
    final Map<String, Long> trafficProportion = (Map<String, Long>) json.get("trafficProportion");
    final Long clientShutdownTimeout = (Long) json.get("clientShutdownTimeout");
    final Long clientStartTimeout = (Long) json.get("clientStartTimeout");
    System.out.println("Finished parsing client config");
    //create d2 client
    final D2Client d2Client = new D2ClientBuilder().setZkHosts(zkConnectString).setZkSessionTimeout(zkSessionTimeout, TimeUnit.MILLISECONDS).setZkStartupTimeout(zkStartupTimeout, TimeUnit.MILLISECONDS).setLbWaitTimeout(zkLoadBalancerNotificationTimeout, TimeUnit.MILLISECONDS).setFlagFile(zkFlagFile).setBasePath(zkBasePath).setFsBasePath(fsBasePath).build();
    System.out.println("Finished creating d2 client, starting d2 client...");
    ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
    final CountDownLatch latch = new CountDownLatch(1);
    //start d2 client by connecting to zookeeper
    startClient(d2Client, executorService, clientStartTimeout, new Callback<None>() {

        @Override
        public void onError(Throwable e) {
            System.exit(1);
        }

        @Override
        public void onSuccess(None result) {
            latch.countDown();
        }
    });
    latch.await();
    System.out.println("D2 client is sending traffic to both " + "partition 0 and partition1.");
    System.out.println("Note that traffic for server1 will be 22x more than server2");
    ScheduledFuture task = executorService.scheduleAtFixedRate(new Runnable() {

        @Override
        public void run() {
            try {
                sendTraffic(trafficProportion, d2Client);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }, 0, 1000, TimeUnit.MILLISECONDS);
    System.out.println("Press enter to shutdown");
    System.out.println("===========================================================\n\n");
    System.in.read();
    task.cancel(false);
    System.out.println("Shutting down...");
    shutdown(d2Client, executorService, clientShutdownTimeout);
}

Example 18 with ZooKeeper

use of com.linkedin.d2.discovery.stores.zk.ZooKeeper in project rest.li by linkedin.

the class ProfileClientExample method main.

public static void main(String[] args) throws Exception {
    //get client configuration
    JSONObject json = parseConfig();
    String zkConnectString = (String) json.get("zkConnectString");
    Long zkSessionTimeout = (Long) json.get("zkSessionTimeout");
    String zkBasePath = (String) json.get("zkBasePath");
    Long zkStartupTimeout = (Long) json.get("zkStartupTimeout");
    Long zkLoadBalancerNotificationTimeout = (Long) json.get("zkLoadBalancerNotificationTimeout");
    String zkFlagFile = (String) json.get("zkFlagFile");
    String fsBasePath = (String) json.get("fsBasePath");
    final Map<String, Map<String, Long>> trafficProportion = (Map<String, Map<String, Long>>) json.get("trafficProportion");
    final Long clientShutdownTimeout = (Long) json.get("clientShutdownTimeout");
    final Long clientStartTimeout = (Long) json.get("clientStartTimeout");
    System.out.println("Finished parsing client config");
    //create d2 client
    final D2Client d2Client = new D2ClientBuilder().setZkHosts(zkConnectString).setZkSessionTimeout(zkSessionTimeout, TimeUnit.MILLISECONDS).setZkStartupTimeout(zkStartupTimeout, TimeUnit.MILLISECONDS).setLbWaitTimeout(zkLoadBalancerNotificationTimeout, TimeUnit.MILLISECONDS).setFlagFile(zkFlagFile).setBasePath(zkBasePath).setFsBasePath(fsBasePath).build();
    System.out.println("Finished creating d2 client, starting d2 client...");
    ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
    final CountDownLatch latch = new CountDownLatch(1);
    //start d2 client by connecting to zookeeper
    startClient(d2Client, executorService, clientStartTimeout, new Callback<None>() {

        @Override
        public void onError(Throwable e) {
            System.exit(1);
        }

        @Override
        public void onSuccess(None result) {
            latch.countDown();
        }
    });
    latch.await();
    System.out.println("D2 client is sending traffic.");
    System.out.println("Note that traffic for 'member' will go mostly to ProfileService 1,3,5 servers.");
    System.out.println("Because we make ProfileService 2,4,6 servers respond slowly \n");
    ScheduledFuture task = executorService.scheduleAtFixedRate(new Runnable() {

        @Override
        public void run() {
            try {
                sendTraffic(trafficProportion, d2Client, null);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }, 0, 1000, TimeUnit.MILLISECONDS);
    System.out.println("Press enter to restore the health of all the servers\n\n\n");
    System.out.println("After this line you will see d2 client will start logging warning" + " message because server 2,4,6's latencies are higher than" + "the threshold (high water mark).");
    System.out.println("===========================================================");
    System.in.read();
    task.cancel(false);
    task = executorService.scheduleAtFixedRate(new Runnable() {

        @Override
        public void run() {
            try {
                sendTraffic(trafficProportion, d2Client, 0l);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }, 0, 1000, TimeUnit.MILLISECONDS);
    System.out.println("=========================================================\n\n\n");
    System.out.println("Now all servers are healthy. Traffic for 'member' " + "will be balanced between profile service 1,2,3,4,5,6.");
    System.out.println("Press enter to shut down\n\n");
    System.in.read();
    task.cancel(false);
    System.out.println("Shutting down...");
    shutdown(d2Client, executorService, clientShutdownTimeout);
}

Example 19 with ZooKeeper

use of com.linkedin.d2.discovery.stores.zk.ZooKeeper in project rest.li by linkedin.

the class CacheClientExample method main.

public static void main(String[] args) throws Exception {
    //get client configuration
    JSONObject json = parseConfig();
    String zkConnectString = (String) json.get("zkConnectString");
    Long zkSessionTimeout = (Long) json.get("zkSessionTimeout");
    String zkBasePath = (String) json.get("zkBasePath");
    Long zkStartupTimeout = (Long) json.get("zkStartupTimeout");
    Long zkLoadBalancerNotificationTimeout = (Long) json.get("zkLoadBalancerNotificationTimeout");
    String zkFlagFile = (String) json.get("zkFlagFile");
    String fsBasePath = (String) json.get("fsBasePath");
    final Map<String, Long> trafficProportion = (Map<String, Long>) json.get("trafficProportion");
    final List<String> keys = (List<String>) json.get("keys");
    final Long clientShutdownTimeout = (Long) json.get("clientShutdownTimeout");
    final Long clientStartTimeout = (Long) json.get("clientStartTimeout");
    System.out.println("Finished parsing client config");
    //create d2 client
    final D2Client d2Client = new D2ClientBuilder().setZkHosts(zkConnectString).setZkSessionTimeout(zkSessionTimeout, TimeUnit.MILLISECONDS).setZkStartupTimeout(zkStartupTimeout, TimeUnit.MILLISECONDS).setLbWaitTimeout(zkLoadBalancerNotificationTimeout, TimeUnit.MILLISECONDS).setFlagFile(zkFlagFile).setBasePath(zkBasePath).setFsBasePath(fsBasePath).build();
    System.out.println("Finished creating d2 client, starting d2 client...");
    ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
    final CountDownLatch latch = new CountDownLatch(1);
    //start d2 client by connecting to zookeeper
    startClient(d2Client, executorService, clientStartTimeout, new Callback<None>() {

        @Override
        public void onError(Throwable e) {
            System.exit(1);
        }

        @Override
        public void onSuccess(None result) {
            latch.countDown();
        }
    });
    latch.await();
    System.out.println("D2 client is sending traffic ");
    ScheduledFuture task = executorService.scheduleAtFixedRate(new Runnable() {

        @Override
        public void run() {
            try {
                sendTraffic(trafficProportion, d2Client, keys);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }, 0, 1000, TimeUnit.MILLISECONDS);
    System.out.println("Press enter to shutdown");
    System.out.println("===========================================================\n\n");
    System.in.read();
    task.cancel(false);
    System.out.println("Shutting down...");
    shutdown(d2Client, executorService, clientShutdownTimeout);
}

Example 20 with ZooKeeper

use of com.linkedin.d2.discovery.stores.zk.ZooKeeper in project rest.li by linkedin.

the class ExampleD2Server method main.

public static void main(String[] args) throws IOException, ParseException {
    //get server configuration
    JSONObject json = parseConfig();
    System.out.println("Finished parsing the server config");
    List<Map<String, Object>> d2ServersConfigs = (List<Map<String, Object>>) json.get("d2Servers");
    List<Map<String, Object>> echoServerConfigs = (List<Map<String, Object>>) json.get("echoServers");
    Long timeout = (Long) json.get("announcerStartTimeout");
    Long shutdownTimeout = (Long) json.get("announcerShutdownTimeout");
    //create and start echo servers
    List<EchoServer> echoServers = createAndStartEchoServers(echoServerConfigs);
    System.out.println("Finished creating echo servers");
    //create d2 announcers (for announcing the existence of these servers to the world)
    ZooKeeperAnnouncer[] zkAnnouncers = createZkAnnouncers(d2ServersConfigs);
    //manager will keep track of the lifecycle of d2 announcers i.e. start publishing
    //once connected to zookeeper, reconnect if zookeeper is disconnected, etc
    ZooKeeperConnectionManager manager = createZkManager(json, zkAnnouncers);
    System.out.println("Starting zookeeper announcers");
    ExecutorService executorService = Executors.newSingleThreadExecutor();
    startAnnouncers(manager, executorService, timeout);
    //waiting for user to turn off this process
    System.in.read();
    System.out.println("shutting down...");
    shutdown(echoServers, manager, executorService, shutdownTimeout);
}