Examples with LastSeenZKStore com.linkedin.d2.discovery.stores.zk.LastSeenZKStore used on opensource projects

Example 1 with LastSeenZKStore

use of com.linkedin.d2.discovery.stores.zk.LastSeenZKStore in project rest.li by linkedin.

the class LastSeenBalancerWithFacilitiesFactory method create.

@Override
public LoadBalancerWithFacilities create(D2ClientConfig config) {
    LOG.info("Creating D2 LoadBalancer based on LastSeenLoadBalancerWithFacilities");
    D2ClientJmxManager d2ClientJmxManager = new D2ClientJmxManager(config.d2JmxManagerPrefix, config.jmxManager);
    // init connection
    ZKConnectionBuilder zkConnectionBuilder = new ZKConnectionBuilder(config.zkHosts);
    zkConnectionBuilder.setShutdownAsynchronously(config.shutdownAsynchronously).setIsSymlinkAware(config.isSymlinkAware).setTimeout((int) config.zkSessionTimeoutInMs);
    ZKPersistentConnection zkPersistentConnection;
    if (config.zkConnectionToUseForLB != null) {
        LOG.info("LastSeenLoadBalancer using shared connection to zookeeper");
        zkPersistentConnection = config.zkConnectionToUseForLB;
    } else {
        LOG.info("LastSeenLoadBalancer using its own connection to zookeeper");
        zkPersistentConnection = new ZKPersistentConnection(zkConnectionBuilder);
    }
    // init all the stores
    LastSeenZKStore<ClusterProperties> lsClusterStore = getClusterPropertiesLastSeenZKStore(config, zkPersistentConnection, d2ClientJmxManager, config._executorService, config.zookeeperReadWindowMs);
    PropertyEventBus<ClusterProperties> clusterBus = new PropertyEventBusImpl<>(config._executorService);
    clusterBus.setPublisher(lsClusterStore);
    LastSeenZKStore<ServiceProperties> lsServiceStore = getServicePropertiesLastSeenZKStore(config, zkPersistentConnection, d2ClientJmxManager, config._executorService, config.zookeeperReadWindowMs);
    PropertyEventBus<ServiceProperties> serviceBus = new PropertyEventBusImpl<>(config._executorService);
    serviceBus.setPublisher(lsServiceStore);
    LastSeenZKStore<UriProperties> lsUrisStore = getUriPropertiesLastSeenZKStore(config, zkPersistentConnection, d2ClientJmxManager, config._executorService, config.zookeeperReadWindowMs);
    PropertyEventBus<UriProperties> uriBus = new PropertyEventBusImpl<>(config._executorService);
    uriBus.setPublisher(lsUrisStore);
    // create the simple load balancer
    SimpleLoadBalancerState state = new SimpleLoadBalancerState(config._executorService, uriBus, clusterBus, serviceBus, config.clientFactories, config.loadBalancerStrategyFactories, config.sslContext, config.sslParameters, config.isSSLEnabled, config.partitionAccessorRegistry, config.sslSessionValidatorFactory, config.deterministicSubsettingMetadataProvider);
    d2ClientJmxManager.setSimpleLoadBalancerState(state);
    SimpleLoadBalancer simpleLoadBalancer = new SimpleLoadBalancer(state, config.lbWaitTimeout, config.lbWaitUnit, config._executorService);
    d2ClientJmxManager.setSimpleLoadBalancer(simpleLoadBalancer);
    // add facilities
    LastSeenLoadBalancerWithFacilities lastSeenLoadBalancer = new LastSeenLoadBalancerWithFacilities(simpleLoadBalancer, config.basePath, config.d2ServicePath, zkPersistentConnection, lsClusterStore, lsServiceStore, lsUrisStore);
    LoadBalancerWithFacilities balancer = lastSeenLoadBalancer;
    if (config.warmUp) {
        balancer = new WarmUpLoadBalancer(balancer, lastSeenLoadBalancer, config.startUpExecutorService, config.fsBasePath, config.d2ServicePath, config.downstreamServicesFetcher, config.warmUpTimeoutSeconds, config.warmUpConcurrentRequests);
    }
    return balancer;
}

Example 2 with LastSeenZKStore

use of com.linkedin.d2.discovery.stores.zk.LastSeenZKStore in project rest.li by linkedin.

the class LastSeenZKStoreTest method testLastSeenLifeCycle.

/**
 * The test aims at
 * 1) write data in the FS store
 * 2) Shutdown the ZKServer and check if a new LastSeenZKStore will read data from disk
 * 3) Restart ZKServer and see if this LastSeenZKStore which could never access to disk will retrieve latest
 *    information from there
 */
@Test(timeOut = 10000, enabled = false)
public void testLastSeenLifeCycle() throws InterruptedException, ExecutionException, TimeoutException, IOException, PropertyStoreException {
    createZKServer();
    // Fill the store with data
    File dataPath = ZKTestUtil.createTempDir("randomFileDataPath");
    LastSeenZKStore<String> store = ZkStoreTestOnlyUtil.getLastSeenZKStore(dataPath.getPath(), PORT);
    ZooKeeperEphemeralStore<String> storeWriter = ZkStoreTestOnlyUtil.getZooKeeperEphemeralStore(PORT);
    storeWriter.put(TEST_ZK_PROP_NAME, "randomData");
    PropertyEventBusImpl<String> propertyEventBus = new PropertyEventBusImpl<>(Executors.newSingleThreadExecutor());
    propertyEventBus.setPublisher(store);
    CountDownLatch initializedLatch = new CountDownLatch(1);
    propertyEventBus.register(Collections.singleton(TEST_ZK_PROP_NAME), new LatchSubscriber(initializedLatch, null));
    initializedLatch.await(5, TimeUnit.SECONDS);
    if (initializedLatch.getCount() != 0) {
        fail("Initialized not received");
    }
    // stopping ZK without removing data. This make ZK unreachable
    _zkServer.shutdown(false);
    // create new last seen, without ZK Connection, and see if it fetches from the server
    store = ZkStoreTestOnlyUtil.getLastSeenZKStore(dataPath.getPath(), PORT);
    propertyEventBus = new PropertyEventBusImpl<>(Executors.newSingleThreadExecutor());
    propertyEventBus.setPublisher(store);
    CountDownLatch initializedLatch2 = new CountDownLatch(1);
    CountDownLatch addLatch2 = new CountDownLatch(1);
    propertyEventBus.register(Collections.singleton(TEST_ZK_PROP_NAME), new LatchSubscriber(initializedLatch2, addLatch2));
    initializedLatch2.await(5, TimeUnit.SECONDS);
    if (initializedLatch2.getCount() != 0) {
        fail("Initialized not received");
    }
    if (addLatch2.getCount() != 1) {
        fail("The add latch should have not been invoked yet");
    }
    // restart ZK and see if it reads the most updated value, the most updated value in this case is identical
    _zkServer.restart();
    addLatch2.await(50, TimeUnit.SECONDS);
    if (addLatch2.getCount() != 0) {
        fail("When ZK restarted we didn't read the most updated value from ZK");
    }
    // shutting everything down
    final FutureCallback<None> shutdownCallback = new FutureCallback<>();
    store.shutdown(shutdownCallback);
    shutdownCallback.get(5, TimeUnit.SECONDS);
    final FutureCallback<None> shutdownCallback2 = new FutureCallback<>();
    storeWriter.shutdown(shutdownCallback2);
    shutdownCallback2.get(5, TimeUnit.SECONDS);
    _zkServer.shutdown();
}

Example 3 with LastSeenZKStore

use of com.linkedin.d2.discovery.stores.zk.LastSeenZKStore in project rest.li by linkedin.

the class LastSeenBalancerWithFacilitiesFactory method getClusterPropertiesLastSeenZKStore.

private LastSeenZKStore<ClusterProperties> getClusterPropertiesLastSeenZKStore(D2ClientConfig config, ZKPersistentConnection zkPersistentConnection, D2ClientJmxManager d2ClientJmxManager, ScheduledExecutorService executorService, int zookeeperReadWindowMs) {
    ZooKeeperPermanentStoreBuilder<ClusterProperties> zkClusterStoreBuilder = new ZooKeeperPermanentStoreBuilder<ClusterProperties>().setSerializer(new ClusterPropertiesJsonSerializer()).setPath(ZKFSUtil.clusterPath(config.basePath)).setExecutorService(executorService).setZookeeperReadWindowMs(zookeeperReadWindowMs).addOnBuildListener(d2ClientJmxManager::setZkClusterRegistry);
    FileStore<ClusterProperties> fileStore = new FileStore<>(FileSystemDirectory.getClusterDirectory(config.fsBasePath), new ClusterPropertiesJsonSerializer());
    d2ClientJmxManager.setFsClusterStore(fileStore);
    return new LastSeenZKStore<>(fileStore, zkClusterStoreBuilder, zkPersistentConnection, config._executorService, config.warmUpTimeoutSeconds, config.warmUpConcurrentRequests);
}

Example 4 with LastSeenZKStore

use of com.linkedin.d2.discovery.stores.zk.LastSeenZKStore in project rest.li by linkedin.

the class LastSeenBalancerWithFacilitiesFactory method getUriPropertiesLastSeenZKStore.

private LastSeenZKStore<UriProperties> getUriPropertiesLastSeenZKStore(D2ClientConfig config, ZKPersistentConnection zkPersistentConnection, D2ClientJmxManager d2ClientJmxManager, ScheduledExecutorService executorService, int zookeeperReadWindowMs) {
    ZooKeeperEphemeralStoreBuilder<UriProperties> zkUrisStoreBuilder = new ZooKeeperEphemeralStoreBuilder<UriProperties>().setSerializer(new UriPropertiesJsonSerializer()).setPath(ZKFSUtil.uriPath(config.basePath)).setMerger(new UriPropertiesMerger()).setUseNewWatcher(config.useNewEphemeralStoreWatcher).setExecutorService(executorService).setZookeeperReadWindowMs(zookeeperReadWindowMs).addOnBuildListener(d2ClientJmxManager::setZkUriRegistry);
    FileStore<UriProperties> fileStore = new FileStore<>(config.fsBasePath + File.separator + ZKFSUtil.URI_PATH, new UriPropertiesJsonSerializer());
    d2ClientJmxManager.setFsUriStore(fileStore);
    if (config.enableSaveUriDataOnDisk) {
        zkUrisStoreBuilder.setBackupStoreFilePath(config.fsBasePath);
    }
    return new LastSeenZKStore<>(fileStore, zkUrisStoreBuilder, zkPersistentConnection, config._executorService, config.warmUpTimeoutSeconds, config.warmUpConcurrentRequests);
}

Example 5 with LastSeenZKStore

use of com.linkedin.d2.discovery.stores.zk.LastSeenZKStore in project rest.li by linkedin.

the class LastSeenBalancerWithFacilitiesFactory method getServicePropertiesLastSeenZKStore.

private LastSeenZKStore<ServiceProperties> getServicePropertiesLastSeenZKStore(D2ClientConfig config, ZKPersistentConnection zkPersistentConnection, D2ClientJmxManager d2ClientJmxManager, ScheduledExecutorService executorService, int zookeeperReadWindowMs) {
    ZooKeeperPermanentStoreBuilder<ServiceProperties> zkServiceStoreBuilder = new ZooKeeperPermanentStoreBuilder<ServiceProperties>().setSerializer(new ServicePropertiesJsonSerializer(config.clientServicesConfig)).setPath(ZKFSUtil.servicePath(config.basePath, config.d2ServicePath)).setExecutorService(executorService).setZookeeperReadWindowMs(zookeeperReadWindowMs).addOnBuildListener(d2ClientJmxManager::setZkServiceRegistry);
    FileStore<ServiceProperties> fileStore = new FileStore<>(FileSystemDirectory.getServiceDirectory(config.fsBasePath, config.d2ServicePath), new ServicePropertiesJsonSerializer());
    d2ClientJmxManager.setFsServiceStore(fileStore);
    return new LastSeenZKStore<>(fileStore, zkServiceStoreBuilder, zkPersistentConnection, config._executorService, config.warmUpTimeoutSeconds, config.warmUpConcurrentRequests);
}