Examples with DegraderControl com.linkedin.util.degrader.DegraderControl used on opensource projects

Example 11 with DegraderControl

use of com.linkedin.util.degrader.DegraderControl in project rest.li by linkedin.

the class DegraderLoadBalancerTest method testDegraderLoadBalancerHandlingExceptionInUpdate.

@Test(groups = { "small", "back-end" })
public void testDegraderLoadBalancerHandlingExceptionInUpdate() {
    Map<String, Object> myMap = new HashMap<String, Object>();
    Long timeInterval = 5000L;
    TestClock clock = new TestClock();
    myMap.put(PropertyKeys.CLOCK, clock);
    myMap.put(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_INTERVAL_MS, timeInterval);
    Map<String, String> degraderProperties = new HashMap<String, String>();
    degraderProperties.put(PropertyKeys.DEGRADER_HIGH_ERROR_RATE, "0.5");
    degraderProperties.put(PropertyKeys.DEGRADER_LOW_ERROR_RATE, "0.2");
    DegraderImpl.Config degraderConfig = DegraderConfigFactory.toDegraderConfig(degraderProperties);
    final List<TrackerClient> clients = createTrackerClient(3, clock, degraderConfig);
    DegraderLoadBalancerStrategyConfig unbrokenConfig = DegraderLoadBalancerStrategyConfig.createHttpConfigFromMap(myMap);
    DegraderLoadBalancerStrategyConfig brokenConfig = new MockDegraderLoadBalancerStrategyConfig(unbrokenConfig);
    URI uri4 = URI.create("http://test.linkedin.com:10010/abc4");
    //this client will throw exception when getDegraderControl is called hence triggering a failed state update
    BrokenTrackerClient brokenClient = new BrokenTrackerClient(uri4, getDefaultPartitionData(1d), new TestLoadBalancerClient(uri4), clock, null);
    clients.add(brokenClient);
    //test DegraderLoadBalancerStrategyV2_1 when the strategy is LOAD_BALANCE
    final DegraderLoadBalancerStrategyV2_1 strategyV2 = new DegraderLoadBalancerStrategyV2_1(brokenConfig, "testStrategyV2", null);
    DegraderLoadBalancerStrategyAdapter strategyAdapterV2 = new DegraderLoadBalancerStrategyAdapter(strategyV2);
    //simulate 100 threads trying to get client at the same time. Make sure that they won't be blocked if an exception
    //occurs during updateState()
    runMultiThreadedTest(strategyAdapterV2, clients, 100, true);
    DegraderLoadBalancerStrategyV2_1.DegraderLoadBalancerState stateV2 = strategyV2.getState();
    // only one exception would occur and other thread would succeed in initializing immediately after
    assertTrue(stateV2.isInitialized());
    assertEquals(stateV2.getStrategy(), DegraderLoadBalancerStrategyV2_1.DegraderLoadBalancerState.Strategy.CALL_DROPPING);
    brokenClient.reset();
    //test DegraderLoadBalancerStrategyV3 when the strategy is LOAD_BALANCE
    DegraderLoadBalancerStrategyV3 strategyV3 = new DegraderLoadBalancerStrategyV3(brokenConfig, "testStrategyV3", null);
    DegraderLoadBalancerStrategyAdapter strategyAdapterV3 = new DegraderLoadBalancerStrategyAdapter(strategyV3);
    //simulate 100 threads trying to get client at the same time. Make sure that they won't be blocked if an exception
    //occurs during updateState()
    runMultiThreadedTest(strategyAdapterV3, clients, 100, true);
    DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState stateV3 = strategyV3.getState().getPartitionState(0);
    // only one exception would occur and other thread would succeed in initializing immediately after
    assertTrue(stateV3.isInitialized());
    assertEquals(stateV3.getStrategy(), DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy.CALL_DROPPING);
    brokenClient.reset();
    // test DegraderLoadBalancerStrategy when the strategy is CALL_DROPPING. We have to make some prepare the
    // environment by simulating lots of high latency calls to the tracker client
    int numberOfCallsPerClient = 10;
    List<CallCompletion> callCompletions = new ArrayList<CallCompletion>();
    for (TrackerClient client : clients) {
        for (int i = 0; i < numberOfCallsPerClient; i++) {
            callCompletions.add(client.getCallTracker().startCall());
        }
    }
    clock.addMs(brokenConfig.getUpdateIntervalMs() - 1000);
    for (CallCompletion cc : callCompletions) {
        for (int i = 0; i < numberOfCallsPerClient; i++) {
            cc.endCall();
        }
    }
    clock.addMs(1000);
    Map<TrackerClient, TrackerClientMetrics> beforeStateUpdate = getTrackerClientMetrics(clients);
    //test DegraderLoadBalancerStrategyV2_1 when the strategy is CALL_DROPPING
    strategyV2.setStrategy(DegraderLoadBalancerStrategyV2_1.DegraderLoadBalancerState.Strategy.CALL_DROPPING);
    strategyV3.setStrategy(DEFAULT_PARTITION_ID, DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy.CALL_DROPPING);
    runMultiThreadedTest(strategyAdapterV2, clients, 100, true);
    stateV2 = strategyV2.getState();
    //MockDegraderLoadBalancerStrategyConfig getHighWaterMark should have been called and throw an exception every time and update would fail for any thread
    // no side-effects on state when update fails
    assertEquals(stateV2.getStrategy(), DegraderLoadBalancerStrategyV2_1.DegraderLoadBalancerState.Strategy.CALL_DROPPING);
    // no side-effects on tracker clients when update fails
    Map<TrackerClient, TrackerClientMetrics> afterFailedV2StateUpdate = getTrackerClientMetrics(clients);
    for (TrackerClient client : clients) {
        assertEquals(beforeStateUpdate.get(client), afterFailedV2StateUpdate.get(client));
    }
    runMultiThreadedTest(strategyAdapterV3, clients, 100, true);
    stateV3 = strategyV3.getState().getPartitionState(0);
    // no side-effects on state when update fails
    assertEquals(stateV3.getStrategy(), DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy.CALL_DROPPING);
    // no side-effects on tracker clients when update fails
    Map<TrackerClient, TrackerClientMetrics> afterFailedV3StateUpdate = getTrackerClientMetrics(clients);
    for (TrackerClient client : clients) {
        assertEquals(beforeStateUpdate.get(client), afterFailedV3StateUpdate.get(client));
    }
    brokenClient.reset();
    //this time we'll change the config to the correct one so it won't throw exception when strategy is CALL_DROPPING
    // update would succeed and state and trackerclients are expected to be mutated
    callCompletions.clear();
    for (TrackerClient client : clients) {
        for (int i = 0; i < numberOfCallsPerClient; i++) {
            callCompletions.add(client.getCallTracker().startCall());
        }
    }
    clock.addMs(brokenConfig.getUpdateIntervalMs() - 1000);
    for (CallCompletion cc : callCompletions) {
        for (int i = 0; i < numberOfCallsPerClient; i++) {
            cc.endCall();
        }
    }
    clock.addMs(1000);
    strategyV2.setConfig(unbrokenConfig);
    beforeStateUpdate = getTrackerClientMetrics(clients);
    // when we run this, the strategy is CALL_DROPPING, and our clients' latency is 4000 MS so our current override
    // drop rate is going to be 0.2 That means occasionally some tracker client will be null
    runMultiThreadedTest(strategyAdapterV2, clients, 100, false);
    stateV2 = strategyV2.getState();
    // This time update should succeed, and both state and trackerclients are updated
    Map<TrackerClient, TrackerClientMetrics> afterV2StateUpdate = getTrackerClientMetrics(clients);
    for (TrackerClient client : clients) {
        assertNotEquals(beforeStateUpdate.get(client), afterV2StateUpdate.get(client));
    }
    assertEquals(stateV2.getStrategy(), DegraderLoadBalancerStrategyV2_1.DegraderLoadBalancerState.Strategy.LOAD_BALANCE);
    brokenClient.reset();
    // reset metrics on tracker client's degrader control
    for (TrackerClient client : clients) {
        TrackerClientMetrics originalMetrics = beforeStateUpdate.get(client);
        DegraderControl degraderControl = client.getDegraderControl(DEFAULT_PARTITION_ID);
        degraderControl.setOverrideDropRate(originalMetrics._overrideDropRate);
        degraderControl.setMaxDropRate(originalMetrics._maxDropRate);
        degraderControl.setOverrideMinCallCount(originalMetrics._overrideMinCallCount);
    }
    callCompletions.clear();
    for (TrackerClient client : clients) {
        for (int i = 0; i < numberOfCallsPerClient; i++) {
            callCompletions.add(client.getCallTracker().startCall());
        }
    }
    clock.addMs(brokenConfig.getUpdateIntervalMs() - 1000);
    for (CallCompletion cc : callCompletions) {
        for (int i = 0; i < numberOfCallsPerClient; i++) {
            cc.endCall();
        }
    }
    clock.addMs(1000);
    strategyV3.setConfig(unbrokenConfig);
    beforeStateUpdate = getTrackerClientMetrics(clients);
    runMultiThreadedTest(strategyAdapterV3, clients, 100, false);
    stateV3 = strategyV3.getState().getPartitionState(0);
    // This time update should succeed, and both state and trackerclients are updated
    Map<TrackerClient, TrackerClientMetrics> afterV3StateUpdate = getTrackerClientMetrics(clients);
    for (TrackerClient client : clients) {
        assertNotEquals(beforeStateUpdate.get(client), afterV3StateUpdate.get(client));
    }
    assertEquals(stateV3.getStrategy(), DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy.LOAD_BALANCE);
}

Example 12 with DegraderControl

use of com.linkedin.util.degrader.DegraderControl in project rest.li by linkedin.

the class DegraderLoadBalancerTest method getTrackerClientMetrics.

private static Map<TrackerClient, TrackerClientMetrics> getTrackerClientMetrics(List<TrackerClient> clients) {
    Map<TrackerClient, TrackerClientMetrics> map = new HashMap<TrackerClient, TrackerClientMetrics>();
    for (TrackerClient client : clients) {
        DegraderControl degraderControl = client.getDegraderControl(DEFAULT_PARTITION_ID);
        map.put(client, new TrackerClientMetrics(degraderControl.getOverrideDropRate(), degraderControl.getMaxDropRate(), degraderControl.getOverrideMinCallCount()));
    }
    return map;
}

Example 13 with DegraderControl

use of com.linkedin.util.degrader.DegraderControl in project rest.li by linkedin.

the class DegraderLoadBalancerTest method clusterTotalRecovery1TC.

/**
   * simulates the situation where a cluster latency gets so high that we will reduce the number of
   * points in hashring to 0 and then increase the call drop rate to 1.0
   * This will causes the cluster to receive no traffic and we want to see if the cluster can recover
   * from such situation.
   * @param myMap
   * @param clock
   * @param timeInterval
   * @param strategy
   */
public void clusterTotalRecovery1TC(Map<String, Object> myMap, TestClock clock, Long timeInterval, DegraderLoadBalancerStrategyAdapter strategy) {
    final int NUM_CHECKS = 5;
    final Long TIME_INTERVAL = timeInterval;
    DegraderLoadBalancerStrategyConfig config = DegraderLoadBalancerStrategyConfig.createHttpConfigFromMap(myMap);
    List<TrackerClient> clients = new ArrayList<TrackerClient>();
    URI uri1 = URI.create("http://test.linkedin.com:3242/fdsaf");
    URIRequest request = new URIRequest(uri1);
    TrackerClient client1 = new TrackerClient(uri1, getDefaultPartitionData(1d), new TestLoadBalancerClient(uri1), clock, null);
    clients.add(client1);
    // force client1 to be disabled
    DegraderControl dcClient1Default = client1.getDegraderControl(DEFAULT_PARTITION_ID);
    dcClient1Default.setOverrideMinCallCount(5);
    dcClient1Default.setMinCallCount(5);
    dcClient1Default.setMaxDropRate(1d);
    dcClient1Default.setUpStep(1.0d);
    List<CallCompletion> ccList = new ArrayList<CallCompletion>();
    CallCompletion cc;
    for (int j = 0; j < NUM_CHECKS; j++) {
        cc = client1.getCallTracker().startCall();
        ccList.add(cc);
    }
    // add high latency and errors to shut off traffic to this tracker client.
    clock.addMs(3500);
    for (Iterator<CallCompletion> iter = ccList.listIterator(); iter.hasNext(); ) {
        cc = iter.next();
        cc.endCallWithError();
        iter.remove();
    }
    // go to next time interval.
    clock.addMs(TIME_INTERVAL);
    Assert.assertEquals(dcClient1Default.getCurrentComputedDropRate(), 1.0);
    // trigger a state update
    TrackerClient resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    // now we mimic the high latency and force the state to drop all calls so to make
    // the overrideClusterDropRate to 1.0
    ccList = new ArrayList<CallCompletion>();
    for (int j = 0; j < NUM_CHECKS; j++) {
        cc = client1.getCallTracker().startCall();
        ccList.add(cc);
    }
    //make sure that the latency is really high
    clock.addMs(3500);
    for (Iterator<CallCompletion> iter = ccList.listIterator(); iter.hasNext(); ) {
        cc = iter.next();
        cc.endCallWithError();
        iter.remove();
    }
    // go to next time interval.
    clock.addMs(TIME_INTERVAL);
    // trigger a state update
    resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    //this time the cluster override drop rate is set to 1.0 so resultTC should be null because we drop the client
    assertNull(resultTC);
    assertEquals(strategy.getCurrentOverrideDropRate(), config.getGlobalStepUp());
    // add another time interval
    clock.addMs(TIME_INTERVAL);
    // usually we alternate between LoadBalancing and CallDropping strategy but we want to test
    // call dropping strategy
    strategy.setStrategyToCallDrop();
    // we simulate call drop by not calling callCompletion endCall() or endCallWithEror() like we did above
    // because override drop rate is set to 1.0 that means all call will be dropped so resultTc should be null
    resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    // this time the cluster override drop rate is set to 0.2 because we're recovering
    assertEquals(strategy.getCurrentOverrideDropRate(), 1 - config.getGlobalStepDown());
    // add another time interval
    clock.addMs(TIME_INTERVAL);
    // set the strategy to callDropping again
    strategy.setStrategyToCallDrop();
    // because override drop rate is set to 0.2 and we simulate as if we still don't get any call
    // this cycle we will set the override drop rate to 0
    resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    assertEquals(strategy.getCurrentOverrideDropRate(), 0.0);
}

Example 14 with DegraderControl

use of com.linkedin.util.degrader.DegraderControl in project rest.li by linkedin.

the class DegraderLoadBalancerTest method testWeightedBalancingWithDeadClient.

@Test(groups = { "small", "back-end" })
public void testWeightedBalancingWithDeadClient() throws URISyntaxException {
    Map<String, Object> myMap = new HashMap<String, Object>();
    myMap.put(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_INTERVAL_MS, 5000L);
    myMap.put(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_MAX_CLUSTER_LATENCY_WITHOUT_DEGRADING, 100.0);
    // this test expected the dead tracker client to not recover through the
    // getTrackerClient mechanism. It only recovered through explicit calls to client1/client2.
    // While we have fixed this problem, keeping this testcase to show how we can completely disable
    // a tracker client through the getTrackerClient method.
    myMap.put(PropertyKeys.HTTP_LB_INITIAL_RECOVERY_LEVEL, 0.0);
    DegraderLoadBalancerStrategyV3 strategy = getStrategy(myMap);
    List<TrackerClient> clients = new ArrayList<TrackerClient>();
    URI uri1 = URI.create("http://test.linkedin.com:3242/fdsaf");
    URI uri2 = URI.create("http://test.linkedin.com:3243/fdsaf");
    TestClock clock1 = new TestClock();
    TestClock clock2 = new TestClock();
    TrackerClient client1 = getClient(uri1, clock1);
    TrackerClient client2 = getClient(uri2, clock2);
    clients.add(client1);
    clients.add(client2);
    // force client2 to be disabled
    DegraderControl dcClient2Default = client2.getDegraderControl(DEFAULT_PARTITION_ID);
    dcClient2Default.setMinCallCount(1);
    dcClient2Default.setOverrideMinCallCount(1);
    dcClient2Default.setMaxDropRate(1d);
    dcClient2Default.setUpStep(1d);
    dcClient2Default.setHighErrorRate(0);
    CallCompletion cc = client2.getCallTracker().startCall();
    clock2.addMs(10000);
    cc.endCallWithError();
    clock1.addMs(15000);
    clock2.addMs(5000);
    System.err.println(dcClient2Default.getCurrentComputedDropRate());
    System.err.println(dcClient2Default.getCurrentComputedDropRate());
    // now verify that we only get client1
    for (int i = 0; i < 1000; ++i) {
        assertEquals(getTrackerClient(strategy, null, new RequestContext(), 0, clients), client1);
    }
    // now force client1 to be disabled
    DegraderControl dcClient1Default = client1.getDegraderControl(DEFAULT_PARTITION_ID);
    dcClient1Default.setMinCallCount(1);
    dcClient1Default.setOverrideMinCallCount(1);
    dcClient1Default.setMaxDropRate(1d);
    dcClient1Default.setUpStep(1d);
    dcClient1Default.setHighErrorRate(0);
    cc = client1.getCallTracker().startCall();
    clock1.addMs(10000);
    cc.endCallWithError();
    clock1.addMs(5000);
    // now verify that we never get a client back
    for (int i = 0; i < 1000; ++i) {
        assertNull(getTrackerClient(strategy, null, new RequestContext(), 1, clients));
    }
    // now enable client1 and client2
    clock1.addMs(15000);
    clock2.addMs(15000);
    client1.getCallTracker().startCall().endCall();
    client2.getCallTracker().startCall().endCall();
    clock1.addMs(5000);
    clock2.addMs(5000);
    // now verify that we get client 1 or 2
    for (int i = 0; i < 1000; ++i) {
        assertTrue(clients.contains(getTrackerClient(strategy, null, new RequestContext(), 2, clients)));
    }
}

Example 15 with DegraderControl

use of com.linkedin.util.degrader.DegraderControl in project rest.li by linkedin.

the class DegraderLoadBalancerTest method clusterRecovery1TC.

/**
   * helper method to test DegraderLoadBalancerStrategy recovery with 1 TrackerClient.
   *
   * We want to test DegraderV2 and V3 with 2 different strategies : LoadBalacing and Call Dropping.
   * So this method needs to able to handle all 4 permutations.
   *
   * @param myMap
   * @param clock
   * @param stepsToFullRecovery
   * @param timeInterval
   * @param strategy
   */
public void clusterRecovery1TC(Map<String, Object> myMap, TestClock clock, int stepsToFullRecovery, Long timeInterval, DegraderLoadBalancerStrategyAdapter strategy, DegraderLoadBalancerStrategyV2_1.DegraderLoadBalancerState.Strategy strategyV2, DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy strategyV3) {
    final int NUM_CHECKS = 5;
    final Long TIME_INTERVAL = timeInterval;
    int localStepsToFullRecovery = stepsToFullRecovery;
    DegraderLoadBalancerStrategyConfig config = DegraderLoadBalancerStrategyConfig.createHttpConfigFromMap(myMap);
    List<TrackerClient> clients = new ArrayList<TrackerClient>();
    URI uri1 = URI.create("http://test.linkedin.com:3242/fdsaf");
    URIRequest request = new URIRequest(uri1);
    TrackerClient client1 = new TrackerClient(uri1, getDefaultPartitionData(1d), new TestLoadBalancerClient(uri1), clock, null);
    clients.add(client1);
    // force client1 to be disabled
    DegraderControl dcClient1Default = client1.getDegraderControl(DEFAULT_PARTITION_ID);
    dcClient1Default.setOverrideMinCallCount(5);
    dcClient1Default.setMinCallCount(5);
    dcClient1Default.setMaxDropRate(1d);
    dcClient1Default.setUpStep(1.0d);
    List<CallCompletion> ccList = new ArrayList<CallCompletion>();
    CallCompletion cc;
    for (int j = 0; j < NUM_CHECKS; j++) {
        cc = client1.getCallTracker().startCall();
        ccList.add(cc);
    }
    // add high latency and errors to shut off traffic to this tracker client.
    // note: the default values for highError and lowError in the degrader are 1.1,
    // which means we don't use errorRates when deciding when to lb/degrade.
    // In addition, because we changed to use the
    clock.addMs(3500);
    //for (int j = 0; j < NUM_CHECKS; j++)
    for (Iterator<CallCompletion> iter = ccList.listIterator(); iter.hasNext(); ) {
        cc = iter.next();
        cc.endCallWithError();
        iter.remove();
    }
    // go to next time interval.
    clock.addMs(TIME_INTERVAL);
    Assert.assertEquals(dcClient1Default.getCurrentComputedDropRate(), 1.0);
    // trigger a state update
    TrackerClient resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    if (config.getInitialRecoveryLevel() < 0.01) {
        //the returned TrackerClient should be null
        assertNull(resultTC, "expected null trackerclient");
        // tracker client, so it's time to try it out. We need to enter this code at least once.
        do {
            // go to next time interval.
            clock.addMs(TIME_INTERVAL);
            // try adjusting the hash ring on this updateState
            if (strategyV3 != null) {
                strategy.setStrategyV3(DEFAULT_PARTITION_ID, strategyV3);
            } else if (strategyV2 != null) {
                strategy.setStrategyV2(strategyV2);
            } else {
                fail("should set strategy (either LoadBalance or Degrader");
            }
            resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
            localStepsToFullRecovery--;
        } while (localStepsToFullRecovery > 0);
    }
    assertNotNull(resultTC, "expected non-null trackerclient");
    // make calls to the tracker client to verify that it's on the road to healthy status.
    for (int j = 0; j < NUM_CHECKS; j++) {
        cc = resultTC.getCallTracker().startCall();
        ccList.add(cc);
    }
    clock.addMs(10);
    for (Iterator<CallCompletion> iter = ccList.listIterator(); iter.hasNext(); ) {
        cc = iter.next();
        cc.endCall();
        iter.remove();
    }
    // go to next time interval.
    clock.addMs(TIME_INTERVAL);
    Assert.assertTrue(dcClient1Default.getCurrentComputedDropRate() < 1d);
    resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    assertNotNull(resultTC, "expected non-null trackerclient");
}