Examples with ClusterProperties com.linkedin.d2.balancer.properties.ClusterProperties used on opensource projects

Example 6 with ClusterProperties

use of com.linkedin.d2.balancer.properties.ClusterProperties in project rest.li by linkedin.

the class LoadBalancerClientCli method printStores.

public static String printStores(ZKConnection zkclient, String zkserver, String d2path) throws IOException, IllegalStateException, URISyntaxException, PropertyStoreException, Exception {
    int serviceCount = 0;
    String zkstr = "\nZKServer:" + zkserver;
    StringBuilder sb = new StringBuilder();
    Set<String> currentservices = new HashSet<String>();
    Map<String, ZooKeeperPermanentStore<ServiceProperties>> zkServiceRegistryMap = new HashMap<String, ZooKeeperPermanentStore<ServiceProperties>>();
    Map<String, List<String>> servicesGroupMap = new HashMap<String, List<String>>();
    // zk stores
    String clstoreString = zkserver + ZKFSUtil.clusterPath(d2path);
    String uristoreString = zkserver + ZKFSUtil.uriPath(d2path);
    ZooKeeperPermanentStore<ClusterProperties> zkClusterRegistry = (ZooKeeperPermanentStore<ClusterProperties>) getStore(zkclient, clstoreString, new ClusterPropertiesJsonSerializer());
    ZooKeeperEphemeralStore<UriProperties> zkUriRegistry = (ZooKeeperEphemeralStore<UriProperties>) getEphemeralStore(zkclient, uristoreString, new UriPropertiesJsonSerializer(), new UriPropertiesMerger());
    List<String> currentclusters = zkClusterRegistry.ls();
    List<String> currenturis = zkUriRegistry.ls();
    List<String> servicesGroups = getServicesGroups(zkclient, d2path);
    for (String serviceGroup : servicesGroups) {
        String scstoreString = zkserver + ZKFSUtil.servicePath(d2path, serviceGroup);
        ZooKeeperPermanentStore<ServiceProperties> zkServiceRegistry = (ZooKeeperPermanentStore<ServiceProperties>) getStore(zkclient, scstoreString, new ServicePropertiesJsonSerializer());
        zkServiceRegistryMap.put(serviceGroup, zkServiceRegistry);
        List<String> services = zkServiceRegistry.ls();
        currentservices.addAll(services);
        servicesGroupMap.put(serviceGroup, services);
        serviceCount += services.size();
    }
    sb.append(zkstr);
    sb.append(" Total Clusters:");
    sb.append(currentclusters.size());
    sb.append(zkstr);
    sb.append(" Total Services:");
    sb.append(serviceCount);
    sb.append(zkstr);
    sb.append(" Total URIs:");
    sb.append(currenturis.size());
    sb.append("\n============================================================");
    sb.append("\nSERVICE GROUPS");
    for (String serviceGroup : servicesGroupMap.keySet()) {
        sb.append("\nGROUP:" + serviceGroup + "           Services:" + servicesGroupMap.get(serviceGroup));
    }
    for (String cluster : currentclusters) {
        int count = 0;
        sb.append("\n============================================================");
        sb.append("\nCLUSTER '");
        sb.append(cluster);
        sb.append("':");
        for (String service : currentservices) {
            for (String serviceGroup : servicesGroupMap.keySet()) {
                ZooKeeperPermanentStore<ServiceProperties> zkStorePropsForSerivceGroup = zkServiceRegistryMap.get(serviceGroup);
                if (zkStorePropsForSerivceGroup != null) {
                    ServiceProperties serviceProps = zkStorePropsForSerivceGroup.get(service);
                    if (serviceProps != null) {
                        if (cluster.equals(serviceProps.getClusterName())) {
                            sb.append("\n-------------------");
                            sb.append("\nSERVICE '" + service + "':");
                            sb.append(printStore(zkClusterRegistry, zkUriRegistry, zkServiceRegistryMap.get(serviceGroup), cluster, service));
                            count++;
                            break;
                        }
                    }
                }
            }
        }
        if (count == 0) {
            sb.append(printStore(zkClusterRegistry, zkUriRegistry, cluster));
            sb.append("\nNo services were found in this cluster.");
        }
    }
    return sb.toString();
}

Example 7 with ClusterProperties

use of com.linkedin.d2.balancer.properties.ClusterProperties in project rest.li by linkedin.

the class ZKFSTogglingLoadBalancerFactoryImpl method createLoadBalancer.

@Override
public TogglingLoadBalancer createLoadBalancer(ZKConnection zkConnection, ScheduledExecutorService executorService) {
    _log.info("Using d2ServicePath: " + _d2ServicePath);
    ZooKeeperPermanentStore<ClusterProperties> zkClusterRegistry = createPermanentStore(zkConnection, ZKFSUtil.clusterPath(_baseZKPath), new ClusterPropertiesJsonSerializer());
    ZooKeeperPermanentStore<ServiceProperties> zkServiceRegistry = createPermanentStore(zkConnection, ZKFSUtil.servicePath(_baseZKPath, _d2ServicePath), new ServicePropertiesJsonSerializer());
    ZooKeeperEphemeralStore<UriProperties> zkUriRegistry = createEphemeralStore(zkConnection, ZKFSUtil.uriPath(_baseZKPath), new UriPropertiesJsonSerializer(), new UriPropertiesMerger(), _useNewEphemeralStoreWatcher);
    FileStore<ClusterProperties> fsClusterStore = createFileStore("clusters", new ClusterPropertiesJsonSerializer());
    FileStore<ServiceProperties> fsServiceStore = createFileStore(_d2ServicePath, new ServicePropertiesJsonSerializer());
    FileStore<UriProperties> fsUriStore = createFileStore("uris", new UriPropertiesJsonSerializer());
    PropertyEventBus<ClusterProperties> clusterBus = new PropertyEventBusImpl<ClusterProperties>(executorService);
    PropertyEventBus<ServiceProperties> serviceBus = new PropertyEventBusImpl<ServiceProperties>(executorService);
    PropertyEventBus<UriProperties> uriBus = new PropertyEventBusImpl<UriProperties>(executorService);
    // This ensures the filesystem store receives the events from the event bus so that
    // it can keep a local backup.
    clusterBus.register(fsClusterStore);
    serviceBus.register(fsServiceStore);
    uriBus.register(fsUriStore);
    TogglingPublisher<ClusterProperties> clusterToggle = _factory.createClusterToggle(zkClusterRegistry, fsClusterStore, clusterBus);
    TogglingPublisher<ServiceProperties> serviceToggle = _factory.createServiceToggle(zkServiceRegistry, fsServiceStore, serviceBus);
    TogglingPublisher<UriProperties> uriToggle = _factory.createUriToggle(zkUriRegistry, fsUriStore, uriBus);
    SimpleLoadBalancerState state = new SimpleLoadBalancerState(executorService, uriBus, clusterBus, serviceBus, _clientFactories, _loadBalancerStrategyFactories, _sslContext, _sslParameters, _isSSLEnabled, _clientServicesConfig);
    SimpleLoadBalancer balancer = new SimpleLoadBalancer(state, _lbTimeout, _lbTimeoutUnit);
    TogglingLoadBalancer togLB = _factory.createBalancer(balancer, state, clusterToggle, serviceToggle, uriToggle);
    togLB.start(new Callback<None>() {

        @Override
        public void onError(Throwable e) {
            _log.warn("Failed to run start on the TogglingLoadBalancer, may not have registered " + "SimpleLoadBalancer and State with JMX.");
        }

        @Override
        public void onSuccess(None result) {
            _log.info("Registered SimpleLoadBalancer and State with JMX.");
        }
    });
    return togLB;
}

Example 8 with ClusterProperties

use of com.linkedin.d2.balancer.properties.ClusterProperties in project rest.li by linkedin.

the class D2Config method configure.

public int configure() throws Exception {
    // original map derived from properties file
    Map<String, Object> clusterServiceConfiguration = merge(_clusterServiceConfigurations);
    // map of clusterName -> cluster configuration
    Map<String, Map<String, Object>> clusters = new HashMap<String, Map<String, Object>>();
    // map of serviceName -> service configuration
    Map<String, Map<String, Object>> services = new HashMap<String, Map<String, Object>>();
    // Ugly. But this is a map of service groups, so it needs to reflect multiple services maps.
    Map<String, Map<String, Map<String, Object>>> serviceVariants = new HashMap<String, Map<String, Map<String, Object>>>();
    // temporary mapping from cluster name to services map, to aid in create cluster variants and
    // service groups.
    Map<String, Map<String, Map<String, Object>>> clusterToServiceMapping = new HashMap<String, Map<String, Map<String, Object>>>();
    int status;
    // temporary mapping from cluster name to the list of colo variants it has.
    Map<String, List<String>> variantToVariantsMapping = new HashMap<String, List<String>>();
    // temporary mapping from cluster name to coloVariant ClusterNames list.
    Map<String, List<String>> clusterToColoClustersMapping = new HashMap<String, List<String>>();
    // mapping from regular cluster name to the list of containing services
    // which will be added as children of the regular cluster znode.
    Map<String, List<String>> regularClusterToServicesMapping = new HashMap<>();
    _log.info("basePath: " + _basePath);
    _log.info("clusterDefaults: " + _clusterDefaults);
    _log.info("serviceDefaults: " + _serviceDefaults);
    final String defaultColo = (String) _clusterDefaults.remove(PropertyKeys.DEFAULT_COLO);
    // Solution 2 is the approach taken below.
    for (String clusterName : clusterServiceConfiguration.keySet()) {
        @SuppressWarnings("unchecked") Map<String, Object> clusterConfig = (Map<String, Object>) clusterServiceConfiguration.get(clusterName);
        clusterConfig.put(PropertyKeys.CLUSTER_NAME, clusterName);
        final Object servicesProperty = clusterConfig.remove(PropertyKeys.SERVICES);
        @SuppressWarnings("unchecked") Map<String, Map<String, Object>> servicesConfigs = (Map<String, Map<String, Object>>) servicesProperty;
        final Object clusterVariantProperty = clusterConfig.remove(PropertyKeys.CLUSTER_VARIANTS);
        @SuppressWarnings("unchecked") Map<String, Map<String, Object>> clusterVariantConfig = (Map<String, Map<String, Object>>) clusterVariantProperty;
        final Object coloVariantsProperty = clusterConfig.remove(PropertyKeys.COLO_VARIANTS);
        @SuppressWarnings("unchecked") List<String> coloVariants = (List<String>) coloVariantsProperty;
        final String masterColo = (String) clusterConfig.remove(PropertyKeys.MASTER_COLO);
        final String enableSymlinkString = (String) clusterConfig.remove(PropertyKeys.ENABLE_SYMLINK);
        final boolean enableSymlink;
        regularClusterToServicesMapping.put(clusterName, servicesConfigs.keySet().stream().collect(Collectors.toList()));
        if (enableSymlinkString != null && "true".equalsIgnoreCase(enableSymlinkString)) {
            enableSymlink = true;
        } else {
            enableSymlink = false;
        }
        // do some sanity check for partitions if any
        // Moving handling of partitionProperties before any coloVariant manipulations
        final Object partitionPropertiesProperty = clusterConfig.get(PropertyKeys.PARTITION_PROPERTIES);
        @SuppressWarnings("unchecked") Map<String, Object> partitionProperties = (Map<String, Object>) partitionPropertiesProperty;
        if (partitionProperties != null) {
            status = handlePartitionProperties(partitionProperties, clusterConfig, clusterName);
            if (status != 0) {
                return status;
            }
        }
        Map<String, String> clusterProperties = new HashMap<>();
        if (coloVariants != null && coloVariants.size() > 0 && !(coloVariants.size() == 1 && coloVariants.contains(""))) {
            clusterProperties.put(PropertyKeys.COLO_VARIANTS, String.join(LIST_SEPARATOR, coloVariants));
        }
        if (masterColo != null && !masterColo.equals("")) {
            clusterProperties.put(PropertyKeys.MASTER_COLO, masterColo);
        }
        if (clusterVariantConfig != null && clusterVariantConfig.size() > 0) {
            clusterProperties.put(PropertyKeys.CLUSTER_VARIANTS, String.join(LIST_SEPARATOR, clusterVariantConfig.keySet()));
        }
        clusterConfig.put(PropertyKeys.CLUSTER_PROPERTIES, clusterProperties);
        // lots of if/else.
        if (coloVariants == null || (coloVariants.size() == 1 && coloVariants.contains(""))) {
            coloVariants = Collections.singletonList("");
        } else {
            // one of the peer colos, if applicable.
            if (!coloVariants.contains(defaultColo)) {
                throw new IllegalStateException("The default colo: " + defaultColo + " is not one of the peer colos = " + coloVariants);
            }
            if (masterColo != null && !coloVariants.contains(masterColo) && !enableSymlink) {
                throw new IllegalStateException("The master colo: " + masterColo + " is not one of the peer colos = " + coloVariants);
            }
        }
        boolean defaultServicesCreated = false;
        for (String colo : coloVariants) {
            // the coloClusterName will be equal to the original cluster name if colo is the empty string
            String coloClusterName = D2Utils.addSuffixToBaseName(clusterName, colo);
            // coloServicesConfigs are the set of d2 services in this cluster in this colo
            // for the regular cluster case I could avoid creation of a new HashMap for both coloServicesConfig
            // and coloServiceConfig, as an optimization at the expense of simplicity.
            Map<String, Map<String, Object>> coloServicesConfigs = new HashMap<String, Map<String, Object>>();
            // Only create the default services once, and only when we have an empty colo string or the
            // colo matches the default colo.
            boolean createDefaultServices = (defaultServicesCreated == false) ? shouldCreateDefaultServices(colo, defaultColo) : false;
            for (String serviceName : servicesConfigs.keySet()) {
                // "resource" level config
                Map<String, Object> serviceConfig = servicesConfigs.get(serviceName);
                // There are some cases where we may not want to create colo variants of a particular service
                // We can't remove properties from the serviceConfig here because we might need to loop
                // over it multiple times.
                String createColoVariants = (String) serviceConfig.get(PropertyKeys.HAS_COLO_VARIANTS);
                boolean createColoVariantsForService = shouldCreateColoVariantsForService(colo, createColoVariants);
                String coloServiceName = serviceName;
                final boolean defaultRoutingToMasterColo = serviceConfig.containsKey(PropertyKeys.DEFAULT_ROUTING) && PropertyKeys.MASTER_SUFFIX.equals(serviceConfig.get(PropertyKeys.DEFAULT_ROUTING));
                // any colo variants of that serviceName.
                if (createColoVariantsForService) {
                    coloServiceName = D2Utils.addSuffixToBaseName(serviceName, colo);
                }
                final Object transportClientProperty = serviceConfig.get(PropertyKeys.TRANSPORT_CLIENT_PROPERTIES);
                @SuppressWarnings("unchecked") Map<String, Object> transportClientConfig = (Map<String, Object>) transportClientProperty;
                serviceConfig.put(PropertyKeys.TRANSPORT_CLIENT_PROPERTIES, transportClientConfig);
                Map<String, Object> coloServiceConfig = new HashMap<String, Object>(serviceConfig);
                // so it does not have to know about what are the default services.
                if (createDefaultServices && !defaultServicesCreated) {
                    // create the Master version of this service.
                    if (masterColo != null && createColoVariantsForService) {
                        // we need to create a "Master" version of this service to point to the current Master
                        // Cluster. Why not just use the original service name? We will point the original
                        // service name at the local cluster, as well as to make it explicit that requests
                        // sent to this service might cross colos, if the master is located in another colo.
                        Map<String, Object> masterServiceConfig = new HashMap<String, Object>(serviceConfig);
                        String masterServiceName = serviceName + PropertyKeys.MASTER_SUFFIX;
                        String masterClusterName;
                        if (enableSymlink) {
                            masterClusterName = D2Utils.getSymlinkNameForMaster(clusterName);
                        } else {
                            masterClusterName = D2Utils.addSuffixToBaseName(clusterName, masterColo);
                        }
                        masterServiceConfig.put(PropertyKeys.CLUSTER_NAME, masterClusterName);
                        masterServiceConfig.put(PropertyKeys.SERVICE_NAME, masterServiceName);
                        masterServiceConfig.put(PropertyKeys.IS_MASTER_SERVICE, "true");
                        coloServicesConfigs.put(masterServiceName, masterServiceConfig);
                    }
                    // this block will handle:
                    // the colo-agnostic service -> colo-specific default cluster mapping (fooService -> FooCluster-WestCoast)
                    // the colo-agnostic service -> colo-agnostic cluster mapping (fooService -> FooCluster)
                    // the latter only being done for regular clusters, the former only being done for clusters
                    // that have coloVariants specified.
                    Map<String, Object> regularServiceConfig = new HashMap<String, Object>(serviceConfig);
                    if (createColoVariantsForService) {
                        // we set isDefaultService flag only if it is a multi-colo aware service.
                        regularServiceConfig.put(PropertyKeys.IS_DEFAULT_SERVICE, "true");
                        if (defaultRoutingToMasterColo) {
                            regularServiceConfig.put(PropertyKeys.DEFAULT_ROUTING_TO_MASTER, "true");
                        }
                    }
                    final String defaultColoClusterName = clusterNameWithRouting(clusterName, colo, defaultColo, masterColo, defaultRoutingToMasterColo, enableSymlink);
                    regularServiceConfig.put(PropertyKeys.CLUSTER_NAME, defaultColoClusterName);
                    regularServiceConfig.put(PropertyKeys.SERVICE_NAME, serviceName);
                    coloServicesConfigs.put(serviceName, regularServiceConfig);
                }
                if (!serviceName.equals(coloServiceName)) {
                    // this block will handle:
                    // the colo-specific service-> colo-specific cluster mapping (fooService-WestCoast -> FooCluster-WestCoast,
                    // fooService-EastCoast -> FooCluster-EastCoast)
                    coloServiceConfig.put(PropertyKeys.CLUSTER_NAME, coloClusterName);
                    coloServiceConfig.put(PropertyKeys.SERVICE_NAME, coloServiceName);
                    coloServicesConfigs.put(coloServiceName, coloServiceConfig);
                }
            }
            // end for each service
            status = addServicesToServicesMap(coloServicesConfigs, services, coloClusterName);
            if (status != NO_ERROR_EXIT_CODE) {
                return status;
            }
            // Now that we've created colo-specific service to colo-specific cluster mappings, we now need
            // to actually create those colo-specific clusters.
            Map<String, Object> coloClusterConfig = clusterConfig;
            if (!clusterName.equals(coloClusterName)) {
                coloClusterConfig = new HashMap<String, Object>(clusterConfig);
                coloClusterConfig.put(PropertyKeys.CLUSTER_NAME, coloClusterName);
                if (createDefaultServices) {
                    clusters.put(clusterName, clusterConfig);
                }
            }
            clusters.put(coloClusterName, coloClusterConfig);
            // list before the cluster variants.
            if (clusterVariantConfig != null) {
                Map<String, Map<String, Object>> coloClusterVariantConfig = new HashMap<String, Map<String, Object>>(clusterVariantConfig);
                status = handleClusterVariants(coloClusterVariantConfig, clusterConfig, clusters, coloServicesConfigs, clusterToServiceMapping, colo, variantToVariantsMapping, masterColo, enableSymlink);
                if (status != 0) {
                    return status;
                }
            } else {
                // even if clusterVariant is not defined, it is still needed to save the coloServicesConfigs
                // in case the serviceGroup directly refers the cluster name
                clusterToServiceMapping.put(coloClusterName, coloServicesConfigs);
                // also save the coloClusterName
                addNewVariantToVariantsList(clusterToColoClustersMapping, clusterName, coloClusterName);
            }
            // the set the flag marking the default services for this cluster as created.
            if (!defaultServicesCreated && createDefaultServices == true) {
                defaultServicesCreated = true;
            }
        }
    // end for each colo variant
    }
    // there are service variants
    if (_serviceVariants != null) {
        for (String serviceGroup : _serviceVariants.keySet()) {
            // each service group contains a list of cluster names and a type field that
            // describes how to treat the list. We group together the services described by these
            // listed clusters, and prep that for writing to a different znode than the default service
            // znode directory. Note that we had already pointed those services to the appropriate cluster
            // variant earlier.
            Map<String, Map<String, Object>> servicesGroupConfig = new HashMap<String, Map<String, Object>>();
            @SuppressWarnings("unchecked") Map<String, Object> configGroupMap = (Map<String, Object>) _serviceVariants.get(serviceGroup);
            String type = (String) configGroupMap.get(PropertyKeys.TYPE);
            final Object clusterListProperty = configGroupMap.get(PropertyKeys.CLUSTER_LIST);
            @SuppressWarnings("unchecked") List<String> clusterList = (List<String>) clusterListProperty;
            // create an alternate service table for the services specified by these cluster variants
            for (Iterator<String> iter = clusterList.listIterator(); iter.hasNext(); ) {
                String clusterItem = iter.next();
                List<String> coloClusterVariantList = variantToVariantsMapping.get(clusterItem);
                if (coloClusterVariantList == null && PropertyKeys.FULL_CLUSTER_LIST.equals(type)) {
                    // For full_cluster_list type, it is allowed to specify real cluster name, not
                    // necessarily always clusterVariant. Check the clusterToColoClustersMappings.
                    coloClusterVariantList = clusterToColoClustersMapping.get(clusterItem);
                }
                if (coloClusterVariantList == null) {
                    // the service group had an unknown cluster!
                    _log.error("Unknown cluster specified: " + clusterItem);
                    return EXCEPTION_EXIT_CODE;
                }
                // in those coloVariants to this service group's list of services.
                for (String coloClusterVariant : coloClusterVariantList) {
                    Map<String, Map<String, Object>> candidateServices = clusterToServiceMapping.get(coloClusterVariant);
                    if (candidateServices == null) {
                        // the service group had an unknown cluster!
                        _log.error("Unknown cluster specified: " + coloClusterVariant);
                        return EXCEPTION_EXIT_CODE;
                    }
                    for (Map.Entry<String, Map<String, Object>> mapEntry : candidateServices.entrySet()) {
                        Object testValue = servicesGroupConfig.put(mapEntry.getKey(), mapEntry.getValue());
                        if (testValue != null) {
                            // We shouldn't have had conflicting services, two variants of the same cluster
                            // were probably specified in the same service group.
                            _log.error("Service group has variants of the same cluster: " + serviceGroup);
                            return EXCEPTION_EXIT_CODE;
                        }
                    }
                }
            }
            if (PropertyKeys.CLUSTER_VARIANTS_LIST.equals(type)) {
                // start from the full list of services, and then overwrite the services specified by the
                // cluster variants.
                Map<String, Map<String, Object>> fullServiceList = new HashMap<String, Map<String, Object>>(services);
                fullServiceList.putAll(servicesGroupConfig);
                serviceVariants.put(serviceGroup, fullServiceList);
            } else if (PropertyKeys.FULL_CLUSTER_LIST.equals(type)) {
                // The use has explicitly indicated that we should put these and only the services that
                // correspond to the named clusters in the serviceGroup.
                serviceVariants.put(serviceGroup, servicesGroupConfig);
            } else {
                _log.error("unknown serviceVariant type: " + type);
                return EXCEPTION_EXIT_CODE;
            }
        }
    }
    _log.debug("serviceVariants: " + serviceVariants);
    _zkConnection.start();
    try {
        _log.info("Cluster configuration:\n" + clusters);
        writeConfig(ZKFSUtil.clusterPath(_basePath), new ClusterPropertiesJsonSerializer(), new ClusterPropertiesJsonSerializer(), clusters, _clusterDefaults);
        _log.info("Wrote cluster configuration");
        _log.info("Service configuration:\n" + services);
        writeConfig(ZKFSUtil.servicePath(_basePath), new ServicePropertiesJsonSerializer(), new ServicePropertiesJsonSerializer(), services, _serviceDefaults);
        _log.info("Wrote service configuration");
        writeChildren(regularClusterToServicesMapping);
        _log.info("Wrote service children nodes under clusters");
        if (!serviceVariants.isEmpty()) {
            for (Map.Entry<String, Map<String, Map<String, Object>>> entry : serviceVariants.entrySet()) {
                if (_log.isDebugEnabled()) {
                    _log.info("serviceVariant: " + entry + "\n");
                } else {
                    _log.info("serviceVariant: " + entry.getKey() + "\n");
                }
                writeConfig(ZKFSUtil.servicePath(_basePath, entry.getKey()), new ServicePropertiesJsonSerializer(), new ServicePropertiesJsonSerializer(), entry.getValue(), _serviceDefaults);
            }
            _log.info("Wrote service variant configurations");
        }
        _log.info("Configuration complete");
        return NO_ERROR_EXIT_CODE;
    } finally {
        try {
            _zkConnection.shutdown();
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            _log.warn("ZooKeeper shutdown interrupted", e);
        }
    }
}

Example 9 with ClusterProperties

use of com.linkedin.d2.balancer.properties.ClusterProperties in project rest.li by linkedin.

the class PartitionedLoadBalancerTestState method getClusterProperties.

@Override
public LoadBalancerStateItem<ClusterProperties> getClusterProperties(String clusterName) {
    List<String> prioritizedSchemes = new ArrayList<String>();
    prioritizedSchemes.add("http");
    ClusterProperties clusterProperties = new ClusterProperties(_cluster, prioritizedSchemes);
    return new LoadBalancerStateItem<ClusterProperties>(clusterProperties, 1, 1);
}

Example 10 with ClusterProperties

use of com.linkedin.d2.balancer.properties.ClusterProperties in project rest.li by linkedin.

the class SimpleLoadBalancerDelayTest method testLoadBalancerWithSlowStartClient.

@Test(groups = { "small", "back-end" }, enabled = false)
public void testLoadBalancerWithSlowStartClient() throws Exception {
    // Generate service, cluster and uri properties for d2
    URI uri1 = URI.create("http://test.qa1.com:1234");
    URI uri2 = URI.create("http://test.qa2.com:2345");
    URI uri3 = URI.create("http://test.qa3.com:6789");
    String clusterName = "cluster-2";
    Map<Integer, PartitionData> partitionData = new HashMap<>(1);
    partitionData.put(DefaultPartitionAccessor.DEFAULT_PARTITION_ID, new PartitionData(1d));
    Map<URI, Map<Integer, PartitionData>> uriData = new HashMap<>(3);
    uriData.put(uri1, partitionData);
    uriData.put(uri2, partitionData);
    uriData.put(uri3, partitionData);
    ClusterProperties clusterProperties = new ClusterProperties(clusterName);
    List<String> prioritizedSchemes = Collections.singletonList("http");
    // enable multi-probe consistent hashing
    Map<String, Object> lbStrategyProperties = Collections.singletonMap(PropertyKeys.HTTP_LB_CONSISTENT_HASH_ALGORITHM, DegraderRingFactory.MULTI_PROBE_CONSISTENT_HASH);
    // set initial drop rate and slow start threshold
    Map<String, String> degraderProperties = new HashMap<>();
    degraderProperties.put(PropertyKeys.DEGRADER_INITIAL_DROP_RATE, "0.99");
    degraderProperties.put(PropertyKeys.DEGRADER_SLOW_START_THRESHOLD, "0.1");
    // constant delay generator
    LoadBalancerSimulator.TimedValueGenerator<String> delayGenerator = (uri, time, unit) -> 100l;
    // constant QPS generator
    LoadBalancerSimulator.QPSGenerator qpsGenerator = () -> 1000;
    Map<String, Object> transportClientProperties = Collections.singletonMap("DelayGenerator", delayGenerator);
    ServiceProperties serviceProperties = new ServiceProperties("foo", clusterName, "/foo", Arrays.asList("degrader"), lbStrategyProperties, transportClientProperties, degraderProperties, prioritizedSchemes, null);
    UriProperties uriProperties = new UriProperties(clusterName, uriData);
    // pass all the info to the simulator
    LoadBalancerSimulator loadBalancerSimulator = new LoadBalancerSimulator(serviceProperties, clusterProperties, uriProperties, delayGenerator, qpsGenerator);
    // Start the simulation, wait for 10 UPDATE_INTERVALS to make sure all uris are fully ramped.
    loadBalancerSimulator.runWait(DegraderLoadBalancerStrategyConfig.DEFAULT_UPDATE_INTERVAL_MS * 20);
    printStates(loadBalancerSimulator);
    URI uri4 = URI.create("http://test.qa4.com:9876");
    uriData.put(uri4, partitionData);
    uriProperties = new UriProperties(clusterName, uriData);
    loadBalancerSimulator.updateUriProperties(uriProperties);
    loadBalancerSimulator.runWait(DegraderLoadBalancerStrategyConfig.DEFAULT_UPDATE_INTERVAL_MS);
    printStates(loadBalancerSimulator);
    // Create the delay generator for the uris
    URI expectedUri4 = URI.create("http://test.qa4.com:9876/foo");
    loadBalancerSimulator.getCountPercent(expectedUri4);
    // the points for uri4 should be 1 and call count percentage is 0.3%.
    double callCountPercent = loadBalancerSimulator.getCountPercent(expectedUri4);
    assertTrue(callCountPercent <= 0.006, "expected percentage is less than 0.006, actual is " + callCountPercent);
    // wait for 2 intervals due to call dropping
    loadBalancerSimulator.runWait(DegraderLoadBalancerStrategyConfig.DEFAULT_UPDATE_INTERVAL_MS * 2);
    printStates(loadBalancerSimulator);
    // the points for uri4 should be 4 and call count percentage is 1.3%
    callCountPercent = loadBalancerSimulator.getCountPercent(expectedUri4);
    assertTrue(callCountPercent <= 0.02, "expected percentage is less than 0.02, actual is " + callCountPercent);
    // wait for 2 intervals due to call dropping
    loadBalancerSimulator.runWait(DegraderLoadBalancerStrategyConfig.DEFAULT_UPDATE_INTERVAL_MS * 2);
    printStates(loadBalancerSimulator);
    // the points for uri4 should be 16 and call count percentage is 5%
    callCountPercent = loadBalancerSimulator.getCountPercent(expectedUri4);
    assertTrue(callCountPercent <= 0.07, "expected percentage is less than 0.07, actual is " + callCountPercent);
    assertTrue(callCountPercent >= 0.03, "expected percentage is larger than 0.03, actual is " + callCountPercent);
    // wait for 2 intervals due to call dropping
    loadBalancerSimulator.runWait(DegraderLoadBalancerStrategyConfig.DEFAULT_UPDATE_INTERVAL_MS * 2);
    printStates(loadBalancerSimulator);
    // the points for uri4 should be 56 and call count percentage is 16%
    callCountPercent = loadBalancerSimulator.getCountPercent(expectedUri4);
    assertTrue(callCountPercent <= 0.18, "expected percentage is less than 0.18, actual is " + callCountPercent);
    assertTrue(callCountPercent >= 0.12, "expected percentage is larger than 0.12, actual is " + callCountPercent);
    // wait for 2 intervals due to call dropping
    loadBalancerSimulator.runWait(DegraderLoadBalancerStrategyConfig.DEFAULT_UPDATE_INTERVAL_MS * 2);
    printStates(loadBalancerSimulator);
    // the points for uri4 should be 96 and call count percentage is 24%
    callCountPercent = loadBalancerSimulator.getCountPercent(expectedUri4);
    assertTrue(callCountPercent <= 0.28, "expected percentage is less than 0.26, actual is " + callCountPercent);
    assertTrue(callCountPercent >= 0.20, "expected percentage is larger than 0.22, actual is " + callCountPercent);
}