Examples with DegraderLoadBalancerState com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyV3.DegraderLoadBalancerState used on opensource projects

Example 11 with DegraderLoadBalancerState

use of com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyV3.DegraderLoadBalancerState in project rest.li by linkedin.

the class DegraderLoadBalancerTest method testDegraderLoadBalancerHandlingExceptionInUpdate.

@Test(groups = { "small", "back-end" })
public void testDegraderLoadBalancerHandlingExceptionInUpdate() {
    Map<String, Object> myMap = new HashMap<String, Object>();
    Long timeInterval = 5000L;
    TestClock clock = new TestClock();
    myMap.put(PropertyKeys.CLOCK, clock);
    myMap.put(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_INTERVAL_MS, timeInterval);
    Map<String, String> degraderProperties = new HashMap<String, String>();
    degraderProperties.put(PropertyKeys.DEGRADER_HIGH_ERROR_RATE, "0.5");
    degraderProperties.put(PropertyKeys.DEGRADER_LOW_ERROR_RATE, "0.2");
    DegraderImpl.Config degraderConfig = DegraderConfigFactory.toDegraderConfig(degraderProperties);
    final List<TrackerClient> clients = createTrackerClient(3, clock, degraderConfig);
    DegraderLoadBalancerStrategyConfig unbrokenConfig = DegraderLoadBalancerStrategyConfig.createHttpConfigFromMap(myMap);
    DegraderLoadBalancerStrategyConfig brokenConfig = new MockDegraderLoadBalancerStrategyConfig(unbrokenConfig);
    URI uri4 = URI.create("http://test.linkedin.com:10010/abc4");
    //this client will throw exception when getDegraderControl is called hence triggering a failed state update
    BrokenTrackerClient brokenClient = new BrokenTrackerClient(uri4, getDefaultPartitionData(1d), new TestLoadBalancerClient(uri4), clock, null);
    clients.add(brokenClient);
    //test DegraderLoadBalancerStrategyV2_1 when the strategy is LOAD_BALANCE
    final DegraderLoadBalancerStrategyV2_1 strategyV2 = new DegraderLoadBalancerStrategyV2_1(brokenConfig, "testStrategyV2", null);
    DegraderLoadBalancerStrategyAdapter strategyAdapterV2 = new DegraderLoadBalancerStrategyAdapter(strategyV2);
    //simulate 100 threads trying to get client at the same time. Make sure that they won't be blocked if an exception
    //occurs during updateState()
    runMultiThreadedTest(strategyAdapterV2, clients, 100, true);
    DegraderLoadBalancerStrategyV2_1.DegraderLoadBalancerState stateV2 = strategyV2.getState();
    // only one exception would occur and other thread would succeed in initializing immediately after
    assertTrue(stateV2.isInitialized());
    assertEquals(stateV2.getStrategy(), DegraderLoadBalancerStrategyV2_1.DegraderLoadBalancerState.Strategy.CALL_DROPPING);
    brokenClient.reset();
    //test DegraderLoadBalancerStrategyV3 when the strategy is LOAD_BALANCE
    DegraderLoadBalancerStrategyV3 strategyV3 = new DegraderLoadBalancerStrategyV3(brokenConfig, "testStrategyV3", null);
    DegraderLoadBalancerStrategyAdapter strategyAdapterV3 = new DegraderLoadBalancerStrategyAdapter(strategyV3);
    //simulate 100 threads trying to get client at the same time. Make sure that they won't be blocked if an exception
    //occurs during updateState()
    runMultiThreadedTest(strategyAdapterV3, clients, 100, true);
    DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState stateV3 = strategyV3.getState().getPartitionState(0);
    // only one exception would occur and other thread would succeed in initializing immediately after
    assertTrue(stateV3.isInitialized());
    assertEquals(stateV3.getStrategy(), DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy.CALL_DROPPING);
    brokenClient.reset();
    // test DegraderLoadBalancerStrategy when the strategy is CALL_DROPPING. We have to make some prepare the
    // environment by simulating lots of high latency calls to the tracker client
    int numberOfCallsPerClient = 10;
    List<CallCompletion> callCompletions = new ArrayList<CallCompletion>();
    for (TrackerClient client : clients) {
        for (int i = 0; i < numberOfCallsPerClient; i++) {
            callCompletions.add(client.getCallTracker().startCall());
        }
    }
    clock.addMs(brokenConfig.getUpdateIntervalMs() - 1000);
    for (CallCompletion cc : callCompletions) {
        for (int i = 0; i < numberOfCallsPerClient; i++) {
            cc.endCall();
        }
    }
    clock.addMs(1000);
    Map<TrackerClient, TrackerClientMetrics> beforeStateUpdate = getTrackerClientMetrics(clients);
    //test DegraderLoadBalancerStrategyV2_1 when the strategy is CALL_DROPPING
    strategyV2.setStrategy(DegraderLoadBalancerStrategyV2_1.DegraderLoadBalancerState.Strategy.CALL_DROPPING);
    strategyV3.setStrategy(DEFAULT_PARTITION_ID, DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy.CALL_DROPPING);
    runMultiThreadedTest(strategyAdapterV2, clients, 100, true);
    stateV2 = strategyV2.getState();
    //MockDegraderLoadBalancerStrategyConfig getHighWaterMark should have been called and throw an exception every time and update would fail for any thread
    // no side-effects on state when update fails
    assertEquals(stateV2.getStrategy(), DegraderLoadBalancerStrategyV2_1.DegraderLoadBalancerState.Strategy.CALL_DROPPING);
    // no side-effects on tracker clients when update fails
    Map<TrackerClient, TrackerClientMetrics> afterFailedV2StateUpdate = getTrackerClientMetrics(clients);
    for (TrackerClient client : clients) {
        assertEquals(beforeStateUpdate.get(client), afterFailedV2StateUpdate.get(client));
    }
    runMultiThreadedTest(strategyAdapterV3, clients, 100, true);
    stateV3 = strategyV3.getState().getPartitionState(0);
    // no side-effects on state when update fails
    assertEquals(stateV3.getStrategy(), DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy.CALL_DROPPING);
    // no side-effects on tracker clients when update fails
    Map<TrackerClient, TrackerClientMetrics> afterFailedV3StateUpdate = getTrackerClientMetrics(clients);
    for (TrackerClient client : clients) {
        assertEquals(beforeStateUpdate.get(client), afterFailedV3StateUpdate.get(client));
    }
    brokenClient.reset();
    //this time we'll change the config to the correct one so it won't throw exception when strategy is CALL_DROPPING
    // update would succeed and state and trackerclients are expected to be mutated
    callCompletions.clear();
    for (TrackerClient client : clients) {
        for (int i = 0; i < numberOfCallsPerClient; i++) {
            callCompletions.add(client.getCallTracker().startCall());
        }
    }
    clock.addMs(brokenConfig.getUpdateIntervalMs() - 1000);
    for (CallCompletion cc : callCompletions) {
        for (int i = 0; i < numberOfCallsPerClient; i++) {
            cc.endCall();
        }
    }
    clock.addMs(1000);
    strategyV2.setConfig(unbrokenConfig);
    beforeStateUpdate = getTrackerClientMetrics(clients);
    // when we run this, the strategy is CALL_DROPPING, and our clients' latency is 4000 MS so our current override
    // drop rate is going to be 0.2 That means occasionally some tracker client will be null
    runMultiThreadedTest(strategyAdapterV2, clients, 100, false);
    stateV2 = strategyV2.getState();
    // This time update should succeed, and both state and trackerclients are updated
    Map<TrackerClient, TrackerClientMetrics> afterV2StateUpdate = getTrackerClientMetrics(clients);
    for (TrackerClient client : clients) {
        assertNotEquals(beforeStateUpdate.get(client), afterV2StateUpdate.get(client));
    }
    assertEquals(stateV2.getStrategy(), DegraderLoadBalancerStrategyV2_1.DegraderLoadBalancerState.Strategy.LOAD_BALANCE);
    brokenClient.reset();
    // reset metrics on tracker client's degrader control
    for (TrackerClient client : clients) {
        TrackerClientMetrics originalMetrics = beforeStateUpdate.get(client);
        DegraderControl degraderControl = client.getDegraderControl(DEFAULT_PARTITION_ID);
        degraderControl.setOverrideDropRate(originalMetrics._overrideDropRate);
        degraderControl.setMaxDropRate(originalMetrics._maxDropRate);
        degraderControl.setOverrideMinCallCount(originalMetrics._overrideMinCallCount);
    }
    callCompletions.clear();
    for (TrackerClient client : clients) {
        for (int i = 0; i < numberOfCallsPerClient; i++) {
            callCompletions.add(client.getCallTracker().startCall());
        }
    }
    clock.addMs(brokenConfig.getUpdateIntervalMs() - 1000);
    for (CallCompletion cc : callCompletions) {
        for (int i = 0; i < numberOfCallsPerClient; i++) {
            cc.endCall();
        }
    }
    clock.addMs(1000);
    strategyV3.setConfig(unbrokenConfig);
    beforeStateUpdate = getTrackerClientMetrics(clients);
    runMultiThreadedTest(strategyAdapterV3, clients, 100, false);
    stateV3 = strategyV3.getState().getPartitionState(0);
    // This time update should succeed, and both state and trackerclients are updated
    Map<TrackerClient, TrackerClientMetrics> afterV3StateUpdate = getTrackerClientMetrics(clients);
    for (TrackerClient client : clients) {
        assertNotEquals(beforeStateUpdate.get(client), afterV3StateUpdate.get(client));
    }
    assertEquals(stateV3.getStrategy(), DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy.LOAD_BALANCE);
}

Example 12 with DegraderLoadBalancerState

use of com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyV3.DegraderLoadBalancerState in project rest.li by linkedin.

the class DegraderLoadBalancerStateTest method testConcurrentResizeAndSet.

/**
   * Resizing the array of partitions doesn't interfere with setting partition state.
   */
@Test(groups = { "small", "back-end" })
public void testConcurrentResizeAndSet() throws InterruptedException {
    // This test aims to reproduce a specific bug, which occurs when one thread sets a
    // partition state while another thread is in the middle of resizing the array of states.
    // To reproduce this, we inject a tricky Clock, which pauses execution of the latter
    // thread in the middle of resizing (when constructing the new partition state).
    // This depends on DegraderLoadBalancerState to call the clock at least once to initialize
    // partition 1. If that changes, you'll have to change clock-related constants below.
    final PauseClock clock = new PauseClock();
    final DegraderLoadBalancerState subject = new DegraderLoadBalancerStrategyV3(new DegraderLoadBalancerStrategyConfig(5000, true, 1, null, Collections.<String, Object>emptyMap(), clock, 1, 1, 1, 1, 1, 1, 1, 1, 0.2, null, 21, null, 0.1, null, null, null, null, 100), SERVICE_NAME, null).getState();
    Thread getPartition1 = new Thread() {

        @Override
        public void run() {
            // resize the array as a side-effect
            subject.getPartitionState(1);
        }
    };
    assertNotNull(subject.getPartitionState(0));
    final long clockCalled = clock._calls.get();
    // 1 partition initialized (so far)
    assertTrue(clockCalled > 0, "clock not called");
    clock._paused = new CountDownLatch(1);
    clock._resume = new CountDownLatch(1);
    getPartition1.start();
    assertTrue(clock._paused.await(60, TimeUnit.SECONDS));
    // Now getPartition1 has started resizing the array.
    final PartitionDegraderLoadBalancerState newState = newPartitionState(0, 0);
    assertNotSame(subject.getPartitionState(0), newState);
    subject.setPartitionState(0, newState);
    assertSame(subject.getPartitionState(0), newState);
    clock._resume.countDown();
    getPartition1.join(60000);
    assertFalse(getPartition1.isAlive());
    // Now getPartition1 has finished resizing the array.
    // as before
    assertSame(subject.getPartitionState(0), newState);
    // 2 partitions initialized
    assertTrue(clock._calls.get() > clockCalled, "clock not called again");
}

Example 13 with DegraderLoadBalancerState

use of com.linkedin.d2.balancer.strategies.degrader.DegraderLoadBalancerStrategyV3.DegraderLoadBalancerState in project rest.li by linkedin.

the class SimpleLoadBalancerStateTest method testRefreshWithConcurrentGetTC.

// This test is to verify a fix for a specific bug, where the d2 client receives a zookeeper
// update and concurrent getTrackerClient requests. In that case, all but the first concurrent
// requests got a null tracker client because the degraderLoadBalancerState was not fully initialized
// (hashring was empty), and this continued until the first request had atomically swamped a
// fully initialized state for other requests to use. This test failed on pre-fix code, it now
// succeeds.
@Test(groups = { "small", "back-end" })
public void testRefreshWithConcurrentGetTC() throws URISyntaxException, InterruptedException {
    reset();
    LinkedList<String> strategyList = new LinkedList<String>();
    URI uri = URI.create("http://cluster-1/test");
    final List<String> schemes = new ArrayList<String>();
    schemes.add("http");
    strategyList.add("degraderV3");
    // set up state
    _state.listenToService("service-1", new NullStateListenerCallback());
    _state.listenToCluster("cluster-1", new NullStateListenerCallback());
    assertNull(_state.getStrategy("service-1", "http"));
    // Use the _clusterRegistry.put to populate the _state.clusterProperties, used by
    // _state.refreshServiceStrategies
    _clusterRegistry.put("cluster-1", new ClusterProperties("cluster-1"));
    _serviceRegistry.put("service-1", new ServiceProperties("service-1", "cluster-1", "/test", strategyList, Collections.<String, Object>emptyMap(), Collections.<String, Object>emptyMap(), Collections.<String, String>emptyMap(), schemes, Collections.<URI>emptySet()));
    LoadBalancerStrategy strategy = _state.getStrategy("service-1", "http");
    assertNotNull(strategy, "got null strategy in setup");
    // test serial to make sure things are working before concurrent test
    TransportClient resultTC = _state.getClient("service-1", "http");
    assertNotNull(resultTC, "got null tracker client in non-concurrent env");
    ExecutorService myExecutor = Executors.newCachedThreadPool();
    ArrayList<TcCallable> cArray = new ArrayList<TcCallable>();
    List<TrackerClient> clients = new ArrayList<TrackerClient>();
    Map<Integer, PartitionData> partitionDataMap = new HashMap<Integer, PartitionData>(2);
    partitionDataMap.put(DefaultPartitionAccessor.DEFAULT_PARTITION_ID, new PartitionData(1d));
    clients.add(new TrackerClient(uri, partitionDataMap, new DegraderLoadBalancerTest.TestLoadBalancerClient(uri), SystemClock.instance(), null));
    for (int i = 0; i < 20; i++) {
        cArray.add(i, new TcCallable(clients, _state));
    }
    Runnable refreshTask = new Runnable() {

        @Override
        public void run() {
            while (true) {
                List<String> myStrategyList = new LinkedList<String>();
                myStrategyList.add("degraderV3");
                _state.refreshServiceStrategies(new ServiceProperties("service-1", "cluster-1", "/test", myStrategyList, Collections.<String, Object>emptyMap(), Collections.<String, Object>emptyMap(), Collections.<String, String>emptyMap(), schemes, Collections.<URI>emptySet()));
                if (Thread.interrupted()) {
                    return;
                }
            }
        }
    };
    myExecutor.execute(refreshTask);
    Integer badResults = 0;
    ArrayList<Future<Integer>> myList = new ArrayList<Future<Integer>>();
    for (int i = 0; i < cArray.size(); i++) {
        @SuppressWarnings("unchecked") Callable<Integer> c = (Callable) cArray.get(i);
        myList.add(i, myExecutor.submit(c));
    }
    try {
        for (int i = 0; i < cArray.size(); i++) {
            badResults += myList.get(i).get();
        }
    } catch (ExecutionException e) {
        Assert.assertFalse(true, "got ExecutionException");
    } finally {
        try {
            // call shutdownNow() to send an interrupt to the refreshTask
            myExecutor.shutdownNow();
            boolean status = myExecutor.awaitTermination(5, TimeUnit.SECONDS);
            if (status == false) {
                Assert.assertFalse(true, "failed to shutdown threads correctly");
            }
        } catch (InterruptedException ie) {
            // this thread was interrupted
            myExecutor.shutdownNow();
        }
    }
    Assert.assertTrue(badResults == 0, "getTrackerClients returned null");
}