use of com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyConfig in project rest.li by linkedin.
the class DegraderLoadBalancerTest method clusterTotalRecovery1TC.
/**
* simulates the situation where a cluster latency gets so high that we will reduce the number of
* points in hashring to 0 and then increase the call drop rate to 1.0
* This will causes the cluster to receive no traffic and we want to see if the cluster can recover
* from such situation.
* @param myMap
* @param clock
* @param timeInterval
* @param strategy
*/
public void clusterTotalRecovery1TC(Map<String, Object> myMap, TestClock clock, Long timeInterval, DegraderLoadBalancerStrategyAdapter strategy) {
final int NUM_CHECKS = 5;
final Long TIME_INTERVAL = timeInterval;
DegraderLoadBalancerStrategyConfig config = DegraderLoadBalancerStrategyConfig.createHttpConfigFromMap(myMap);
List<TrackerClient> clients = new ArrayList<TrackerClient>();
URI uri1 = URI.create("http://test.linkedin.com:3242/fdsaf");
URIRequest request = new URIRequest(uri1);
TrackerClient client1 = new TrackerClient(uri1, getDefaultPartitionData(1d), new TestLoadBalancerClient(uri1), clock, null);
clients.add(client1);
// force client1 to be disabled
DegraderControl dcClient1Default = client1.getDegraderControl(DEFAULT_PARTITION_ID);
dcClient1Default.setOverrideMinCallCount(5);
dcClient1Default.setMinCallCount(5);
dcClient1Default.setMaxDropRate(1d);
dcClient1Default.setUpStep(1.0d);
List<CallCompletion> ccList = new ArrayList<CallCompletion>();
CallCompletion cc;
for (int j = 0; j < NUM_CHECKS; j++) {
cc = client1.getCallTracker().startCall();
ccList.add(cc);
}
// add high latency and errors to shut off traffic to this tracker client.
clock.addMs(3500);
for (Iterator<CallCompletion> iter = ccList.listIterator(); iter.hasNext(); ) {
cc = iter.next();
cc.endCallWithError();
iter.remove();
}
// go to next time interval.
clock.addMs(TIME_INTERVAL);
Assert.assertEquals(dcClient1Default.getCurrentComputedDropRate(), 1.0);
// trigger a state update
TrackerClient resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
// now we mimic the high latency and force the state to drop all calls so to make
// the overrideClusterDropRate to 1.0
ccList = new ArrayList<CallCompletion>();
for (int j = 0; j < NUM_CHECKS; j++) {
cc = client1.getCallTracker().startCall();
ccList.add(cc);
}
//make sure that the latency is really high
clock.addMs(3500);
for (Iterator<CallCompletion> iter = ccList.listIterator(); iter.hasNext(); ) {
cc = iter.next();
cc.endCallWithError();
iter.remove();
}
// go to next time interval.
clock.addMs(TIME_INTERVAL);
// trigger a state update
resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
//this time the cluster override drop rate is set to 1.0 so resultTC should be null because we drop the client
assertNull(resultTC);
assertEquals(strategy.getCurrentOverrideDropRate(), config.getGlobalStepUp());
// add another time interval
clock.addMs(TIME_INTERVAL);
// usually we alternate between LoadBalancing and CallDropping strategy but we want to test
// call dropping strategy
strategy.setStrategyToCallDrop();
// we simulate call drop by not calling callCompletion endCall() or endCallWithEror() like we did above
// because override drop rate is set to 1.0 that means all call will be dropped so resultTc should be null
resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
// this time the cluster override drop rate is set to 0.2 because we're recovering
assertEquals(strategy.getCurrentOverrideDropRate(), 1 - config.getGlobalStepDown());
// add another time interval
clock.addMs(TIME_INTERVAL);
// set the strategy to callDropping again
strategy.setStrategyToCallDrop();
// because override drop rate is set to 0.2 and we simulate as if we still don't get any call
// this cycle we will set the override drop rate to 0
resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
assertEquals(strategy.getCurrentOverrideDropRate(), 0.0);
}
use of com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyConfig in project rest.li by linkedin.
the class DegraderLoadBalancerTest method DegraderLoadBalancerQuarantineTest.
/**
* DegraderLoadBalancerQuarantineTest
*/
@Test(groups = { "small", "back-end" })
public void DegraderLoadBalancerQuarantineTest() {
DegraderLoadBalancerStrategyConfig config = new DegraderLoadBalancerStrategyConfig(1000);
Map<Integer, PartitionData> partitionDataMap = new HashMap<Integer, PartitionData>(2);
TestClock clock = new TestClock();
DegraderImpl.Config degraderConfig = DegraderConfigFactory.toDegraderConfig(Collections.emptyMap());
List<TrackerClient> trackerClients = createTrackerClient(3, clock, degraderConfig);
TrackerClientUpdater trackerClientUpdater = new TrackerClientUpdater(trackerClients.get(0), DEFAULT_PARTITION_ID);
DegraderLoadBalancerQuarantine quarantine = new DegraderLoadBalancerQuarantine(trackerClientUpdater, config, "abc0");
TransportHealthCheck healthCheck = (TransportHealthCheck) quarantine.getHealthCheckClient();
RestRequest restRequest = healthCheck.getRestRequest();
Assert.assertTrue(restRequest.getURI().equals(URI.create("http://test.linkedin.com:10010/abc0")));
Assert.assertTrue(restRequest.getMethod().equals("OPTIONS"));
DegraderLoadBalancerStrategyConfig config1 = new DegraderLoadBalancerStrategyConfig(1000, DegraderLoadBalancerStrategyConfig.DEFAULT_UPDATE_ONLY_AT_INTERVAL, 100, null, Collections.<String, Object>emptyMap(), DegraderLoadBalancerStrategyConfig.DEFAULT_CLOCK, DegraderLoadBalancerStrategyConfig.DEFAULT_INITIAL_RECOVERY_LEVEL, DegraderLoadBalancerStrategyConfig.DEFAULT_RAMP_FACTOR, DegraderLoadBalancerStrategyConfig.DEFAULT_HIGH_WATER_MARK, DegraderLoadBalancerStrategyConfig.DEFAULT_LOW_WATER_MARK, DegraderLoadBalancerStrategyConfig.DEFAULT_GLOBAL_STEP_UP, DegraderLoadBalancerStrategyConfig.DEFAULT_GLOBAL_STEP_DOWN, DegraderLoadBalancerStrategyConfig.DEFAULT_CLUSTER_MIN_CALL_COUNT_HIGH_WATER_MARK, DegraderLoadBalancerStrategyConfig.DEFAULT_CLUSTER_MIN_CALL_COUNT_LOW_WATER_MARK, DegraderLoadBalancerStrategyConfig.DEFAULT_HASHRING_POINT_CLEANUP_RATE, null, DegraderLoadBalancerStrategyConfig.DEFAULT_NUM_PROBES, null, DegraderLoadBalancerStrategyConfig.DEFAULT_QUARANTINE_MAXPERCENT, null, null, "GET", "/test/admin", DegraderLoadBalancerStrategyConfig.DEFAULT_QUARANTINE_LATENCY);
TrackerClientUpdater updater1 = new TrackerClientUpdater(trackerClients.get(1), DEFAULT_PARTITION_ID);
quarantine = new DegraderLoadBalancerQuarantine(updater1, config1, "abc0");
healthCheck = (TransportHealthCheck) quarantine.getHealthCheckClient();
restRequest = healthCheck.getRestRequest();
Assert.assertTrue(restRequest.getURI().equals(URI.create("http://test.linkedin.com:10010/test/admin")));
Assert.assertTrue(restRequest.getMethod().equals("GET"));
DegraderLoadBalancerStrategyConfig config2 = new DegraderLoadBalancerStrategyConfig(1000, DegraderLoadBalancerStrategyConfig.DEFAULT_UPDATE_ONLY_AT_INTERVAL, 100, null, Collections.<String, Object>emptyMap(), DegraderLoadBalancerStrategyConfig.DEFAULT_CLOCK, DegraderLoadBalancerStrategyConfig.DEFAULT_INITIAL_RECOVERY_LEVEL, DegraderLoadBalancerStrategyConfig.DEFAULT_RAMP_FACTOR, DegraderLoadBalancerStrategyConfig.DEFAULT_HIGH_WATER_MARK, DegraderLoadBalancerStrategyConfig.DEFAULT_LOW_WATER_MARK, DegraderLoadBalancerStrategyConfig.DEFAULT_GLOBAL_STEP_UP, DegraderLoadBalancerStrategyConfig.DEFAULT_GLOBAL_STEP_DOWN, DegraderLoadBalancerStrategyConfig.DEFAULT_CLUSTER_MIN_CALL_COUNT_HIGH_WATER_MARK, DegraderLoadBalancerStrategyConfig.DEFAULT_CLUSTER_MIN_CALL_COUNT_LOW_WATER_MARK, DegraderLoadBalancerStrategyConfig.DEFAULT_HASHRING_POINT_CLEANUP_RATE, null, DegraderLoadBalancerStrategyConfig.DEFAULT_NUM_PROBES, null, DegraderLoadBalancerStrategyConfig.DEFAULT_QUARANTINE_MAXPERCENT, null, null, "OPTIONS", null, DegraderLoadBalancerStrategyConfig.DEFAULT_QUARANTINE_LATENCY);
TrackerClientUpdater updater2 = new TrackerClientUpdater(trackerClients.get(2), DEFAULT_PARTITION_ID);
quarantine = new DegraderLoadBalancerQuarantine(updater2, config2, "abc0");
healthCheck = (TransportHealthCheck) quarantine.getHealthCheckClient();
restRequest = healthCheck.getRestRequest();
Assert.assertTrue(restRequest.getURI().equals(URI.create("http://test.linkedin.com:10010/abc2")));
Assert.assertTrue(restRequest.getMethod().equals("OPTIONS"));
}
use of com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyConfig in project rest.li by linkedin.
the class DegraderLoadBalancerTest method clusterRecovery1TC.
/**
* helper method to test DegraderLoadBalancerStrategy recovery with 1 TrackerClient.
*
* We want to test DegraderV2 and V3 with 2 different strategies : LoadBalacing and Call Dropping.
* So this method needs to able to handle all 4 permutations.
*
* @param myMap
* @param clock
* @param stepsToFullRecovery
* @param timeInterval
* @param strategy
*/
public void clusterRecovery1TC(Map<String, Object> myMap, TestClock clock, int stepsToFullRecovery, Long timeInterval, DegraderLoadBalancerStrategyAdapter strategy, DegraderLoadBalancerStrategyV2_1.DegraderLoadBalancerState.Strategy strategyV2, DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy strategyV3) {
final int NUM_CHECKS = 5;
final Long TIME_INTERVAL = timeInterval;
int localStepsToFullRecovery = stepsToFullRecovery;
DegraderLoadBalancerStrategyConfig config = DegraderLoadBalancerStrategyConfig.createHttpConfigFromMap(myMap);
List<TrackerClient> clients = new ArrayList<TrackerClient>();
URI uri1 = URI.create("http://test.linkedin.com:3242/fdsaf");
URIRequest request = new URIRequest(uri1);
TrackerClient client1 = new TrackerClient(uri1, getDefaultPartitionData(1d), new TestLoadBalancerClient(uri1), clock, null);
clients.add(client1);
// force client1 to be disabled
DegraderControl dcClient1Default = client1.getDegraderControl(DEFAULT_PARTITION_ID);
dcClient1Default.setOverrideMinCallCount(5);
dcClient1Default.setMinCallCount(5);
dcClient1Default.setMaxDropRate(1d);
dcClient1Default.setUpStep(1.0d);
List<CallCompletion> ccList = new ArrayList<CallCompletion>();
CallCompletion cc;
for (int j = 0; j < NUM_CHECKS; j++) {
cc = client1.getCallTracker().startCall();
ccList.add(cc);
}
// add high latency and errors to shut off traffic to this tracker client.
// note: the default values for highError and lowError in the degrader are 1.1,
// which means we don't use errorRates when deciding when to lb/degrade.
// In addition, because we changed to use the
clock.addMs(3500);
//for (int j = 0; j < NUM_CHECKS; j++)
for (Iterator<CallCompletion> iter = ccList.listIterator(); iter.hasNext(); ) {
cc = iter.next();
cc.endCallWithError();
iter.remove();
}
// go to next time interval.
clock.addMs(TIME_INTERVAL);
Assert.assertEquals(dcClient1Default.getCurrentComputedDropRate(), 1.0);
// trigger a state update
TrackerClient resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
if (config.getInitialRecoveryLevel() < 0.01) {
//the returned TrackerClient should be null
assertNull(resultTC, "expected null trackerclient");
// tracker client, so it's time to try it out. We need to enter this code at least once.
do {
// go to next time interval.
clock.addMs(TIME_INTERVAL);
// try adjusting the hash ring on this updateState
if (strategyV3 != null) {
strategy.setStrategyV3(DEFAULT_PARTITION_ID, strategyV3);
} else if (strategyV2 != null) {
strategy.setStrategyV2(strategyV2);
} else {
fail("should set strategy (either LoadBalance or Degrader");
}
resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
localStepsToFullRecovery--;
} while (localStepsToFullRecovery > 0);
}
assertNotNull(resultTC, "expected non-null trackerclient");
// make calls to the tracker client to verify that it's on the road to healthy status.
for (int j = 0; j < NUM_CHECKS; j++) {
cc = resultTC.getCallTracker().startCall();
ccList.add(cc);
}
clock.addMs(10);
for (Iterator<CallCompletion> iter = ccList.listIterator(); iter.hasNext(); ) {
cc = iter.next();
cc.endCall();
iter.remove();
}
// go to next time interval.
clock.addMs(TIME_INTERVAL);
Assert.assertTrue(dcClient1Default.getCurrentComputedDropRate() < 1d);
resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
assertNotNull(resultTC, "expected non-null trackerclient");
}
use of com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyConfig in project rest.li by linkedin.
the class DegraderLoadBalancerTest method testLowTrafficHighLatency100Clients.
@Test(groups = { "small", "back-end" })
public void testLowTrafficHighLatency100Clients() {
Map<String, Object> myMap = new HashMap<String, Object>();
Long timeInterval = 5000L;
TestClock clock = new TestClock();
myMap.put(PropertyKeys.CLOCK, clock);
myMap.put(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_INTERVAL_MS, timeInterval);
//we need to override the min call count to 0 because we're testing a service with low traffic.
//if we don't do this, the computedDropRate will not change and we will never be able to recover
//after we degraded the cluster.
Map<String, String> degraderImplProperties = new HashMap<String, String>();
degraderImplProperties.put(PropertyKeys.DEGRADER_MIN_CALL_COUNT, "1");
degraderImplProperties.put(PropertyKeys.DEGRADER_HIGH_ERROR_RATE, "0.5");
degraderImplProperties.put(PropertyKeys.DEGRADER_LOW_ERROR_RATE, "0.2");
DegraderImpl.Config degraderConfig = DegraderConfigFactory.toDegraderConfig(degraderImplProperties);
double qps = 0.3;
//test Strategy V3
List<TrackerClient> clients = createTrackerClient(100, clock, degraderConfig);
DegraderLoadBalancerStrategyConfig config = DegraderLoadBalancerStrategyConfig.createHttpConfigFromMap(myMap);
DegraderLoadBalancerStrategyV3 strategyV3 = new DegraderLoadBalancerStrategyV3(config, "DegraderLoadBalancerTest", null);
DegraderLoadBalancerStrategyAdapter strategy = new DegraderLoadBalancerStrategyAdapter(strategyV3);
testDegraderLoadBalancerSimulator(strategy, clock, timeInterval, clients, qps, degraderConfig);
//test Strategy V2
clients = createTrackerClient(100, clock, degraderConfig);
config = DegraderLoadBalancerStrategyConfig.createHttpConfigFromMap(myMap);
DegraderLoadBalancerStrategyV2_1 strategyV2 = new DegraderLoadBalancerStrategyV2_1(config, "DegraderLoadBalancerTest", null);
strategy = new DegraderLoadBalancerStrategyAdapter(strategyV2);
testDegraderLoadBalancerSimulator(strategy, clock, timeInterval, clients, qps, degraderConfig);
}
use of com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyConfig in project rest.li by linkedin.
the class DegraderLoadBalancerTest method testStateIsNullAndCallCountIsGreaterThanZero.
@Test(groups = { "small", "back-end" })
public void testStateIsNullAndCallCountIsGreaterThanZero() throws URISyntaxException, InterruptedException {
// check for average cluster latency < max latency
// max so we don't time out from lag on testing machine
DegraderLoadBalancerStrategyV3 strategy = new DegraderLoadBalancerStrategyV3(new DegraderLoadBalancerStrategyConfig(5000), "DegraderLoadBalancerTest", null);
List<TrackerClient> clients = new ArrayList<TrackerClient>();
TestClock clock1 = new TestClock();
TestClock clock2 = new TestClock();
clients.add(getClient(URI.create("http://test.linkedin.com:3242/fdsaf"), clock1));
clients.add(getClient(URI.create("http://test.linkedin.com:3243/fdsaf"), clock2));
for (int i = 0; i < 1000; ++i) {
clients.get(i % 2).getCallTracker().startCall().endCall();
}
clock1.addMs(5000);
// this should trigger setting _state (state is null and count > 0) with an override
// of 0d
getTrackerClient(strategy, null, new RequestContext(), -1, clients);
for (TrackerClient client : clients) {
assertEquals(client.getDegraderControl(DEFAULT_PARTITION_ID).getOverrideDropRate(), 0d);
}
}
Aggregations