Examples with DegraderLoadBalancerStrategyV3 com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyV3 used on opensource projects

Example 1 with DegraderLoadBalancerStrategyV3

use of com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyV3 in project rest.li by linkedin.

the class RetryClientTest method prepareLoadBalancer.

public SimpleLoadBalancer prepareLoadBalancer(List<String> uris) throws URISyntaxException {
    String serviceName = "retryService";
    String clusterName = "cluster";
    String path = "";
    String strategyName = "degrader";
    // setup partition
    Map<URI, Map<Integer, PartitionData>> partitionDescriptions = new HashMap<URI, Map<Integer, PartitionData>>();
    for (String uri : uris) {
        final URI foo = URI.create(uri);
        Map<Integer, PartitionData> foo1Data = new HashMap<Integer, PartitionData>();
        foo1Data.put(0, new PartitionData(1.0));
        partitionDescriptions.put(foo, foo1Data);
    }
    DegraderLoadBalancerStrategyV3 strategy = new DegraderLoadBalancerStrategyV3(new DegraderLoadBalancerStrategyConfig(5000), serviceName, null);
    List<LoadBalancerState.SchemeStrategyPair> orderedStrategies = new ArrayList<LoadBalancerState.SchemeStrategyPair>();
    orderedStrategies.add(new LoadBalancerState.SchemeStrategyPair("http", strategy));
    PartitionAccessor accessor = new TestRetryPartitionAccessor();
    SimpleLoadBalancer balancer = new SimpleLoadBalancer(new PartitionedLoadBalancerTestState(clusterName, serviceName, path, strategyName, partitionDescriptions, orderedStrategies, accessor));
    return balancer;
}

Example 2 with DegraderLoadBalancerStrategyV3

use of com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyV3 in project rest.li by linkedin.

the class LoadBalancerStrategyPropertiesConverter method toConfig.

@SuppressWarnings({ "unchecked" })
public static D2LoadBalancerStrategyProperties toConfig(Map<String, Object> properties) {
    D2LoadBalancerStrategyProperties config = new D2LoadBalancerStrategyProperties();
    if (properties.containsKey(PropertyKeys.HTTP_LB_GLOBAL_STEP_DOWN)) {
        config.setGlobalStepDown(coerce(properties.get(PropertyKeys.HTTP_LB_GLOBAL_STEP_DOWN), Double.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_GLOBAL_STEP_UP)) {
        config.setGlobalStepUp(coerce(properties.get(PropertyKeys.HTTP_LB_GLOBAL_STEP_UP), Double.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_INITIAL_RECOVERY_LEVEL)) {
        config.setInitialRecoveryLevel(coerce(properties.get(PropertyKeys.HTTP_LB_INITIAL_RECOVERY_LEVEL), Double.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_RING_RAMP_FACTOR)) {
        config.setRingRampFactor(coerce(properties.get(PropertyKeys.HTTP_LB_RING_RAMP_FACTOR), Double.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_HIGH_WATER_MARK)) {
        config.setHighWaterMark(coerce(properties.get(PropertyKeys.HTTP_LB_HIGH_WATER_MARK), Double.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_LOW_WATER_MARK)) {
        config.setLowWaterMark(coerce(properties.get(PropertyKeys.HTTP_LB_LOW_WATER_MARK), Double.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_POINTS_PER_WEIGHT)) {
        config.setPointsPerWeight(coerce(properties.get(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_POINTS_PER_WEIGHT), Integer.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_INTERVAL_MS)) {
        config.setUpdateIntervalMs(coerce(properties.get(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_INTERVAL_MS), Long.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_CLUSTER_MIN_CALL_COUNT_HIGH_WATER_MARK)) {
        config.setMinCallCountHighWaterMark(coerce(properties.get(PropertyKeys.HTTP_LB_CLUSTER_MIN_CALL_COUNT_HIGH_WATER_MARK), Long.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_CLUSTER_MIN_CALL_COUNT_LOW_WATER_MARK)) {
        config.setMinCallCountLowWaterMark(coerce(properties.get(PropertyKeys.HTTP_LB_CLUSTER_MIN_CALL_COUNT_LOW_WATER_MARK), Long.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_HASHRING_POINT_CLEANUP_RATE)) {
        config.setHashRingPointCleanupRate(coerce(properties.get(PropertyKeys.HTTP_LB_HASHRING_POINT_CLEANUP_RATE), Double.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_HASH_METHOD)) {
        String hashMethodString = coerce(properties.get(PropertyKeys.HTTP_LB_HASH_METHOD), String.class);
        if (DegraderLoadBalancerStrategyV3.HASH_METHOD_NONE.equalsIgnoreCase(hashMethodString)) {
            config.setHashMethod(hashMethodEnum.RANDOM);
        } else if (DegraderLoadBalancerStrategyV3.HASH_METHOD_URI_REGEX.equalsIgnoreCase(hashMethodString)) {
            config.setHashMethod(hashMethodEnum.URI_REGEX);
        }
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_HASH_CONFIG)) {
        hashConfigType hashConfig = new hashConfigType();
        Map<String, Object> hashConfigProperties = (Map<String, Object>) properties.get(PropertyKeys.HTTP_LB_HASH_CONFIG);
        if (hashConfigProperties.containsKey(URIRegexHash.KEY_REGEXES)) {
            List<String> uriRegexes = (List<String>) hashConfigProperties.get(URIRegexHash.KEY_REGEXES);
            hashConfig.setUriRegexes(new StringArray(uriRegexes));
        }
        if (hashConfigProperties.containsKey(URIRegexHash.KEY_WARN_ON_NO_MATCH)) {
            String warnOnNoMatchString = (String) hashConfigProperties.get(URIRegexHash.KEY_WARN_ON_NO_MATCH);
            hashConfig.setWarnOnNoMatch(Boolean.parseBoolean(warnOnNoMatchString));
        }
        if (hashConfigProperties.containsKey(URIRegexHash.KEY_FAIL_ON_NO_MATCH)) {
            String failOnNoMatchString = (String) hashConfigProperties.get(URIRegexHash.KEY_FAIL_ON_NO_MATCH);
            hashConfig.setFailOnNoMatch(Boolean.parseBoolean(failOnNoMatchString));
        }
        config.setHashConfig(hashConfig);
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_ONLY_AT_INTERVAL)) {
        config.setUpdateOnlyAtInterval(coerce(properties.get(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_ONLY_AT_INTERVAL), Boolean.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_CONSISTENT_HASH_ALGORITHM)) {
        String consistentHashAlgorithm = coerce(properties.get(PropertyKeys.HTTP_LB_CONSISTENT_HASH_ALGORITHM), String.class);
        if (DelegatingRingFactory.POINT_BASED_CONSISTENT_HASH.equalsIgnoreCase(consistentHashAlgorithm)) {
            config.setConsistentHashAlgorithm(ConsistentHashAlgorithmEnum.POINT_BASED);
        } else if (DelegatingRingFactory.MULTI_PROBE_CONSISTENT_HASH.equalsIgnoreCase(consistentHashAlgorithm)) {
            config.setConsistentHashAlgorithm(ConsistentHashAlgorithmEnum.MULTI_PROBE);
        } else if (DelegatingRingFactory.DISTRIBUTION_NON_HASH.equalsIgnoreCase(consistentHashAlgorithm)) {
            config.setConsistentHashAlgorithm(ConsistentHashAlgorithmEnum.DISTRIBUTION_BASED);
        }
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_CONSISTENT_HASH_NUM_PROBES)) {
        config.setNumberOfProbes(coerce(properties.get(PropertyKeys.HTTP_LB_CONSISTENT_HASH_NUM_PROBES), Integer.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_CONSISTENT_HASH_POINTS_PER_HOST)) {
        config.setNumberOfPointsPerHost(coerce(properties.get(PropertyKeys.HTTP_LB_CONSISTENT_HASH_POINTS_PER_HOST), Integer.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_CONSISTENT_HASH_BOUNDED_LOAD_BALANCING_FACTOR)) {
        config.setBoundedLoadBalancingFactor(coerce(properties.get(PropertyKeys.HTTP_LB_CONSISTENT_HASH_BOUNDED_LOAD_BALANCING_FACTOR), Double.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_QUARANTINE_MAX_PERCENT) || properties.containsKey(PropertyKeys.HTTP_LB_QUARANTINE_METHOD)) {
        quarantineInfo quarantineInfo = new quarantineInfo();
        if (properties.containsKey(PropertyKeys.HTTP_LB_QUARANTINE_MAX_PERCENT)) {
            quarantineInfo.setQuarantineMaxPercent(coerce(properties.get(PropertyKeys.HTTP_LB_QUARANTINE_MAX_PERCENT), Double.class));
        }
        if (properties.containsKey(PropertyKeys.HTTP_LB_QUARANTINE_METHOD)) {
            quarantineInfo.setQuarantineMethod(coerce(properties.get(PropertyKeys.HTTP_LB_QUARANTINE_METHOD), String.class));
        }
        config.setQuarantineCfg(quarantineInfo);
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_ERROR_STATUS_REGEX)) {
        config.setErrorStatusRegex(coerce(properties.get(PropertyKeys.HTTP_LB_ERROR_STATUS_REGEX), String.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_LOW_EVENT_EMITTING_INTERVAL)) {
        config.setLowEmittingInterval(coerce(properties.get(PropertyKeys.HTTP_LB_LOW_EVENT_EMITTING_INTERVAL), Integer.class));
    }
    if (properties.containsKey(PropertyKeys.HTTP_LB_HIGH_EVENT_EMITTING_INTERVAL)) {
        config.setHighEmittingInterval(coerce(properties.get(PropertyKeys.HTTP_LB_HIGH_EVENT_EMITTING_INTERVAL), Integer.class));
    }
    return config;
}

Example 3 with DegraderLoadBalancerStrategyV3

use of com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyV3 in project rest.li by linkedin.

the class LoadBalancerStrategyPropertiesConverter method toProperties.

public static Map<String, Object> toProperties(D2LoadBalancerStrategyProperties config) {
    if (config == null) {
        return Collections.emptyMap();
    }
    Map<String, Object> map = new HashMap<>();
    if (config.hasGlobalStepDown()) {
        map.put(PropertyKeys.HTTP_LB_GLOBAL_STEP_DOWN, config.getGlobalStepDown().toString());
    }
    if (config.hasGlobalStepUp()) {
        map.put(PropertyKeys.HTTP_LB_GLOBAL_STEP_UP, config.getGlobalStepUp().toString());
    }
    if (config.hasInitialRecoveryLevel()) {
        map.put(PropertyKeys.HTTP_LB_INITIAL_RECOVERY_LEVEL, config.getInitialRecoveryLevel().toString());
    }
    if (config.hasRingRampFactor()) {
        map.put(PropertyKeys.HTTP_LB_RING_RAMP_FACTOR, config.getRingRampFactor().toString());
    }
    if (config.hasHighWaterMark()) {
        map.put(PropertyKeys.HTTP_LB_HIGH_WATER_MARK, config.getHighWaterMark().toString());
    }
    if (config.hasLowWaterMark()) {
        map.put(PropertyKeys.HTTP_LB_LOW_WATER_MARK, config.getLowWaterMark().toString());
    }
    if (config.hasPointsPerWeight()) {
        map.put(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_POINTS_PER_WEIGHT, config.getPointsPerWeight().toString());
    }
    if (config.hasUpdateIntervalMs()) {
        map.put(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_INTERVAL_MS, config.getUpdateIntervalMs().toString());
    }
    if (config.hasMinCallCountHighWaterMark()) {
        map.put(PropertyKeys.HTTP_LB_CLUSTER_MIN_CALL_COUNT_HIGH_WATER_MARK, config.getMinCallCountHighWaterMark().toString());
    }
    if (config.hasHashRingPointCleanupRate()) {
        map.put(PropertyKeys.HTTP_LB_HASHRING_POINT_CLEANUP_RATE, config.getHashRingPointCleanupRate().toString());
    }
    if (config.hasMinCallCountLowWaterMark()) {
        map.put(PropertyKeys.HTTP_LB_CLUSTER_MIN_CALL_COUNT_LOW_WATER_MARK, config.getMinCallCountLowWaterMark().toString());
    }
    if (config.hasHashMethod()) {
        switch(config.getHashMethod()) {
            case RANDOM:
                map.put(PropertyKeys.HTTP_LB_HASH_METHOD, DegraderLoadBalancerStrategyV3.HASH_METHOD_NONE);
                break;
            case URI_REGEX:
                map.put(PropertyKeys.HTTP_LB_HASH_METHOD, DegraderLoadBalancerStrategyV3.HASH_METHOD_URI_REGEX);
                break;
            default:
                // default to random hash method.
                map.put(PropertyKeys.HTTP_LB_HASH_METHOD, DegraderLoadBalancerStrategyV3.HASH_METHOD_NONE);
        }
    }
    if (config.hasHashConfig()) {
        hashConfigType hashConfig = config.getHashConfig();
        Map<String, Object> hashConfigProperties = new HashMap<>();
        if (hashConfig.hasUriRegexes()) {
            hashConfigProperties.put(URIRegexHash.KEY_REGEXES, hashConfig.getUriRegexes().stream().collect(Collectors.toList()));
        }
        if (hashConfig.hasFailOnNoMatch()) {
            hashConfigProperties.put(URIRegexHash.KEY_FAIL_ON_NO_MATCH, hashConfig.isFailOnNoMatch().toString());
        }
        if (hashConfig.hasWarnOnNoMatch()) {
            hashConfigProperties.put(URIRegexHash.KEY_WARN_ON_NO_MATCH, hashConfig.isWarnOnNoMatch().toString());
        }
        map.put(PropertyKeys.HTTP_LB_HASH_CONFIG, hashConfigProperties);
    }
    if (config.hasUpdateOnlyAtInterval()) {
        map.put(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_ONLY_AT_INTERVAL, config.isUpdateOnlyAtInterval().toString());
    }
    if (config.hasConsistentHashAlgorithm()) {
        switch(config.getConsistentHashAlgorithm()) {
            case MULTI_PROBE:
                map.put(PropertyKeys.HTTP_LB_CONSISTENT_HASH_ALGORITHM, DelegatingRingFactory.MULTI_PROBE_CONSISTENT_HASH);
                break;
            case POINT_BASED:
                map.put(PropertyKeys.HTTP_LB_CONSISTENT_HASH_ALGORITHM, DelegatingRingFactory.POINT_BASED_CONSISTENT_HASH);
                break;
            case DISTRIBUTION_BASED:
                map.put(PropertyKeys.HTTP_LB_CONSISTENT_HASH_ALGORITHM, DelegatingRingFactory.DISTRIBUTION_NON_HASH);
        }
    }
    if (config.hasNumberOfProbes()) {
        map.put(PropertyKeys.HTTP_LB_CONSISTENT_HASH_NUM_PROBES, config.getNumberOfProbes().toString());
    }
    if (config.hasNumberOfPointsPerHost()) {
        map.put(PropertyKeys.HTTP_LB_CONSISTENT_HASH_POINTS_PER_HOST, config.getNumberOfPointsPerHost().toString());
    }
    if (config.hasBoundedLoadBalancingFactor()) {
        map.put(PropertyKeys.HTTP_LB_CONSISTENT_HASH_BOUNDED_LOAD_BALANCING_FACTOR, config.getBoundedLoadBalancingFactor().toString());
    }
    if (config.hasQuarantineCfg()) {
        quarantineInfo quarantineInfo = config.getQuarantineCfg();
        if (quarantineInfo.hasQuarantineMaxPercent()) {
            map.put(PropertyKeys.HTTP_LB_QUARANTINE_MAX_PERCENT, quarantineInfo.getQuarantineMaxPercent().toString());
        }
        if (quarantineInfo.hasQuarantineMethod()) {
            map.put(PropertyKeys.HTTP_LB_QUARANTINE_METHOD, quarantineInfo.getQuarantineMethod().toString());
        }
    }
    if (config.hasErrorStatusRegex()) {
        map.put(PropertyKeys.HTTP_LB_ERROR_STATUS_REGEX, config.getErrorStatusRegex());
    }
    if (config.hasLowEmittingInterval()) {
        map.put(PropertyKeys.HTTP_LB_LOW_EVENT_EMITTING_INTERVAL, config.getLowEmittingInterval().toString());
    }
    if (config.hasHighEmittingInterval()) {
        map.put(PropertyKeys.HTTP_LB_HIGH_EVENT_EMITTING_INTERVAL, config.getHighEmittingInterval().toString());
    }
    return map;
}

Example 4 with DegraderLoadBalancerStrategyV3

use of com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyV3 in project rest.li by linkedin.

the class DegraderLoadBalancerTest method createRaceCondition.

private List<Runnable> createRaceCondition(final URI uri, Clock clock, final DegraderLoadBalancerStrategyV3 strategy, final CountDownLatch joinLatch) {
    final CountDownLatch clientLatch = new CountDownLatch(1);
    DegraderTrackerClient evilClient = new EvilClient(uri, getDefaultPartitionData(1, 2), new DegraderTrackerClientTest.TestClient(), clock, null, clientLatch);
    final List<DegraderTrackerClient> clients = Collections.singletonList(evilClient);
    final Runnable update = new Runnable() {

        @Override
        public void run() {
            // getRing will wait for latch in getPartitionWeight
            strategy.getRing(1, 0, toMap(clients));
            joinLatch.countDown();
        }
    };
    final Runnable resize = new Runnable() {

        @Override
        public void run() {
            // releases latch for partition 0
            clientLatch.countDown();
            // resize
            strategy.getRing(1, 1, toMap(clients));
            joinLatch.countDown();
        }
    };
    List<Runnable> actions = new ArrayList<>();
    actions.add(update);
    actions.add(resize);
    return actions;
}

Example 5 with DegraderLoadBalancerStrategyV3

use of com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyV3 in project rest.li by linkedin.

the class DegraderLoadBalancerTest method testClusterRecovery2TC.

@Test(groups = { "small", "back-end" })
public void testClusterRecovery2TC() {
    final int NUM_CHECKS = 5;
    final Long TIME_INTERVAL = 5000L;
    Map<String, Object> myMap = lbDefaultConfig();
    // 1,2,4,8,16,32,64,100% steps, given a 2x recovery step coefficient
    int localStepsToFullRecovery = 8;
    myMap.put(PropertyKeys.HTTP_LB_INITIAL_RECOVERY_LEVEL, 0.005);
    myMap.put(PropertyKeys.HTTP_LB_RING_RAMP_FACTOR, 2d);
    myMap.put(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_INTERVAL_MS, TIME_INTERVAL);
    TestClock clock = new TestClock();
    myMap.put(PropertyKeys.CLOCK, clock);
    DegraderLoadBalancerStrategyConfig config = DegraderLoadBalancerStrategyConfig.createHttpConfigFromMap(myMap);
    DegraderLoadBalancerStrategyV3 strategy = new DegraderLoadBalancerStrategyV3(config, "DegraderLoadBalancerTest", null, DEGRADER_STATE_LISTENER_FACTORIES);
    List<DegraderTrackerClient> clients = new ArrayList<>();
    URI uri1 = URI.create("http://test.linkedin.com:3242/fdsaf");
    URI uri2 = URI.create("http://test.linkedin.com:3243/fdsaf");
    URIRequest request = new URIRequest(uri1);
    List<CallCompletion> ccList = new ArrayList<>();
    CallCompletion cc;
    DegraderTrackerClient client1 = new DegraderTrackerClientImpl(uri1, getDefaultPartitionData(1d), new TestLoadBalancerClient(uri1), clock, null);
    DegraderTrackerClient client2 = new DegraderTrackerClientImpl(uri2, getDefaultPartitionData(1d), new TestLoadBalancerClient(uri2), clock, null);
    clients.add(client1);
    clients.add(client2);
    // force client1 to be disabled if we encounter errors/high latency
    DegraderControl dcClient1Default = client1.getDegraderControl(DEFAULT_PARTITION_ID);
    dcClient1Default.setMinCallCount(5);
    dcClient1Default.setOverrideMinCallCount(5);
    dcClient1Default.setUpStep(1.0);
    // force client2 to be disabled if we encounter errors/high latency
    DegraderControl dcClient2Default = client2.getDegraderControl(DEFAULT_PARTITION_ID);
    dcClient2Default.setOverrideMinCallCount(5);
    dcClient2Default.setMinCallCount(5);
    dcClient2Default.setUpStep(0.4);
    // Have one cycle of successful calls to verify valid tracker clients returned.
    // try load balancing on this updateState, need to updateState before forcing the strategy.
    TrackerClient resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    strategy.setStrategy(DEFAULT_PARTITION_ID, PartitionDegraderLoadBalancerState.Strategy.LOAD_BALANCE);
    resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    assertNotNull(resultTC, "expected non-null trackerclient");
    for (int j = 0; j < NUM_CHECKS; j++) {
        ccList.add(client1.getCallTracker().startCall());
        ccList.add(client2.getCallTracker().startCall());
    }
    clock.addMs(1);
    for (Iterator<CallCompletion> iter = ccList.listIterator(); iter.hasNext(); ) {
        cc = iter.next();
        cc.endCall();
    }
    // bump to next interval, and get stats.
    clock.addMs(5000);
    // try Load balancing on this updateState
    strategy.setStrategy(DEFAULT_PARTITION_ID, PartitionDegraderLoadBalancerState.Strategy.LOAD_BALANCE);
    resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    assertNotNull(resultTC, "expected non-null trackerclient");
    Assert.assertEquals(dcClient1Default.getCurrentComputedDropRate(), 0.0);
    Assert.assertEquals(dcClient2Default.getCurrentComputedDropRate(), 0.0);
    // now simulate a bad cluster state with high error and high latency
    for (int j = 0; j < NUM_CHECKS; j++) {
        ccList.add(client1.getCallTracker().startCall());
        ccList.add(client2.getCallTracker().startCall());
    }
    clock.addMs(3500);
    for (Iterator<CallCompletion> iter = ccList.listIterator(); iter.hasNext(); ) {
        cc = iter.next();
        cc.endCallWithError();
    }
    // go to next interval
    clock.addMs(5000);
    Assert.assertEquals(dcClient1Default.getCurrentComputedDropRate(), 1.0);
    Assert.assertEquals(dcClient2Default.getCurrentComputedDropRate(), 0.4);
    // trigger a state update, the returned TrackerClient should be client2
    // because client 1 should have gone up to a 1.0 drop rate, and the cluster should
    // be unhealthy
    strategy.setStrategy(DEFAULT_PARTITION_ID, PartitionDegraderLoadBalancerState.Strategy.LOAD_BALANCE);
    resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    assertEquals(resultTC, client2);
    // it's current drop rate.
    do {
        // go to next time interval.
        clock.addMs(TIME_INTERVAL);
        // adjust the hash ring this time.
        strategy.setStrategy(DEFAULT_PARTITION_ID, PartitionDegraderLoadBalancerState.Strategy.LOAD_BALANCE);
        resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
        localStepsToFullRecovery--;
    } while (localStepsToFullRecovery > 0);
    assertNotNull(resultTC, "expected non-null trackerclient");
    assertTrue(strategy.getState().getPartitionState(DEFAULT_PARTITION_ID).getPointsMap().get(client1.getUri()) == client1.getPartitionWeight(DEFAULT_PARTITION_ID) * config.getPointsPerWeight(), "client1 did not recover to full weight in hash map.");
    Assert.assertEquals(dcClient2Default.getCurrentComputedDropRate(), 0.4, "client2 drop rate not as expected");
    cc = client1.getCallTracker().startCall();
    clock.addMs(10);
    cc.endCall();
    clock.addMs(TIME_INTERVAL);
    resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    assertNotNull(resultTC, "expected non-null trackerclient");
}