Examples with LoadBalancerQuarantine com.linkedin.d2.balancer.strategies.LoadBalancerQuarantine used on opensource projects

Example 6 with LoadBalancerQuarantine

use of com.linkedin.d2.balancer.strategies.LoadBalancerQuarantine in project rest.li by linkedin.

the class QuarantineManagerTest method testEnrollOneQuarantine.

@Test
public void testEnrollOneQuarantine() {
    LoadBalancerQuarantine quarantine = Mockito.mock(LoadBalancerQuarantine.class);
    List<TrackerClient> trackerClients = TrackerClientMockHelper.mockTrackerClients(3);
    Map<TrackerClient, LoadBalancerQuarantine> existingQuarantineMap = new HashMap<>();
    setup(0.5, true, true);
    _quarantineManager.tryEnableQuarantine();
    PartitionState state = new PartitionStateTestDataBuilder().setTrackerClientStateMap(trackerClients, Arrays.asList(StateUpdater.MIN_HEALTH_SCORE, StateUpdater.MIN_HEALTH_SCORE, QuarantineManager.INITIAL_RECOVERY_HEALTH_SCORE), Arrays.asList(TrackerClientState.HealthState.UNHEALTHY, TrackerClientState.HealthState.NEUTRAL, TrackerClientState.HealthState.UNHEALTHY), Arrays.asList(20, 20, 20)).build();
    _quarantineManager.updateQuarantineState(state, state, 10000);
    assertEquals(state.getQuarantineMap().size(), 1, "No exception expected");
    assertTrue(state.getQuarantineMap().containsKey(trackerClients.get(0)));
}

Example 7 with LoadBalancerQuarantine

use of com.linkedin.d2.balancer.strategies.LoadBalancerQuarantine in project rest.li by linkedin.

the class QuarantineManagerTest method testQuarantineCheck.

@Test(dataProvider = "trueFalse")
public void testQuarantineCheck(boolean quarantineCheckResult) {
    setup(0.5, false, false);
    LoadBalancerQuarantine quarantine = Mockito.mock(LoadBalancerQuarantine.class);
    List<TrackerClient> trackerClients = TrackerClientMockHelper.mockTrackerClients(3);
    Map<TrackerClient, LoadBalancerQuarantine> existingQuarantineMap = new HashMap<>();
    existingQuarantineMap.put(trackerClients.get(0), quarantine);
    Mockito.when(quarantine.checkUpdateQuarantineState()).thenReturn(quarantineCheckResult);
    PartitionState state = new PartitionStateTestDataBuilder().setTrackerClientStateMap(trackerClients, Arrays.asList(StateUpdater.MIN_HEALTH_SCORE, 0.6, 0.6), Arrays.asList(TrackerClientState.HealthState.NEUTRAL, TrackerClientState.HealthState.UNHEALTHY, TrackerClientState.HealthState.UNHEALTHY), Arrays.asList(20, 20, 20)).setQuarantineMap(existingQuarantineMap).build();
    _quarantineManager.tryEnableQuarantine();
    _quarantineManager.updateQuarantineState(state, state, DEFAULT_AVG_CLUSTER_LATENCY);
    if (quarantineCheckResult) {
        assertTrue(state.getQuarantineMap().isEmpty());
        assertEquals(state.getRecoveryTrackerClients().size(), 1, "The quarantine should be over and the host should be put into recovery");
        assertEquals(state.getTrackerClientStateMap().get(trackerClients.get(0)).getHealthScore(), QuarantineManager.INITIAL_RECOVERY_HEALTH_SCORE);
    } else {
        assertEquals(state.getQuarantineMap().size(), 1, "Quarantine health check failed, the host should be kept in quarantine state");
        assertTrue(state.getRecoveryTrackerClients().isEmpty(), "No client should be in recovery state");
        assertTrue(state.getQuarantineMap().containsKey(trackerClients.get(0)));
    }
}

Example 8 with LoadBalancerQuarantine

use of com.linkedin.d2.balancer.strategies.LoadBalancerQuarantine in project rest.li by linkedin.

the class DegraderLoadBalancerTest method DegraderLoadBalancerQuarantineTest.

/**
 * DegraderLoadBalancerQuarantineTest
 */
@Test(groups = { "small", "back-end" })
public void DegraderLoadBalancerQuarantineTest() {
    DegraderLoadBalancerStrategyConfig config = new DegraderLoadBalancerStrategyConfig(1000);
    TestClock clock = new TestClock();
    DegraderImpl.Config degraderConfig = DegraderConfigFactory.toDegraderConfig(Collections.emptyMap());
    List<DegraderTrackerClient> trackerClients = createTrackerClient(3, clock, degraderConfig);
    DegraderTrackerClientUpdater degraderTrackerClientUpdater = new DegraderTrackerClientUpdater(trackerClients.get(0), DEFAULT_PARTITION_ID);
    LoadBalancerQuarantine quarantine = new LoadBalancerQuarantine(degraderTrackerClientUpdater.getTrackerClient(), config, "abc0");
    TransportHealthCheck healthCheck = (TransportHealthCheck) quarantine.getHealthCheckClient();
    RestRequest restRequest = healthCheck.getRestRequest();
    Assert.assertTrue(restRequest.getURI().equals(URI.create("http://test.linkedin.com:10010/abc0")));
    Assert.assertTrue(restRequest.getMethod().equals("OPTIONS"));
    DegraderLoadBalancerStrategyConfig config1 = new DegraderLoadBalancerStrategyConfig(1000, DegraderLoadBalancerStrategyConfig.DEFAULT_UPDATE_ONLY_AT_INTERVAL, 100, null, Collections.<String, Object>emptyMap(), DegraderLoadBalancerStrategyConfig.DEFAULT_CLOCK, DegraderLoadBalancerStrategyConfig.DEFAULT_INITIAL_RECOVERY_LEVEL, DegraderLoadBalancerStrategyConfig.DEFAULT_RAMP_FACTOR, DegraderLoadBalancerStrategyConfig.DEFAULT_HIGH_WATER_MARK, DegraderLoadBalancerStrategyConfig.DEFAULT_LOW_WATER_MARK, DegraderLoadBalancerStrategyConfig.DEFAULT_GLOBAL_STEP_UP, DegraderLoadBalancerStrategyConfig.DEFAULT_GLOBAL_STEP_DOWN, DegraderLoadBalancerStrategyConfig.DEFAULT_CLUSTER_MIN_CALL_COUNT_HIGH_WATER_MARK, DegraderLoadBalancerStrategyConfig.DEFAULT_CLUSTER_MIN_CALL_COUNT_LOW_WATER_MARK, DegraderLoadBalancerStrategyConfig.DEFAULT_HASHRING_POINT_CLEANUP_RATE, null, DegraderLoadBalancerStrategyConfig.DEFAULT_NUM_PROBES, DegraderLoadBalancerStrategyConfig.DEFAULT_POINTS_PER_HOST, DegraderLoadBalancerStrategyConfig.DEFAULT_BOUNDED_LOAD_BALANCING_FACTOR, null, DegraderLoadBalancerStrategyConfig.DEFAULT_QUARANTINE_MAXPERCENT, null, null, "GET", "/test/admin", DegraderImpl.DEFAULT_LOW_LATENCY, null, DegraderLoadBalancerStrategyConfig.DEFAULT_LOW_EVENT_EMITTING_INTERVAL, DegraderLoadBalancerStrategyConfig.DEFAULT_HIGH_EVENT_EMITTING_INTERVAL, DegraderLoadBalancerStrategyConfig.DEFAULT_CLUSTER_NAME);
    DegraderTrackerClientUpdater updater1 = new DegraderTrackerClientUpdater(trackerClients.get(1), DEFAULT_PARTITION_ID);
    quarantine = new LoadBalancerQuarantine(updater1.getTrackerClient(), config1, "abc0");
    healthCheck = (TransportHealthCheck) quarantine.getHealthCheckClient();
    restRequest = healthCheck.getRestRequest();
    Assert.assertTrue(restRequest.getURI().equals(URI.create("http://test.linkedin.com:10010/test/admin")));
    Assert.assertTrue(restRequest.getMethod().equals("GET"));
    DegraderLoadBalancerStrategyConfig config2 = new DegraderLoadBalancerStrategyConfig(1000, DegraderLoadBalancerStrategyConfig.DEFAULT_UPDATE_ONLY_AT_INTERVAL, 100, null, Collections.<String, Object>emptyMap(), DegraderLoadBalancerStrategyConfig.DEFAULT_CLOCK, DegraderLoadBalancerStrategyConfig.DEFAULT_INITIAL_RECOVERY_LEVEL, DegraderLoadBalancerStrategyConfig.DEFAULT_RAMP_FACTOR, DegraderLoadBalancerStrategyConfig.DEFAULT_HIGH_WATER_MARK, DegraderLoadBalancerStrategyConfig.DEFAULT_LOW_WATER_MARK, DegraderLoadBalancerStrategyConfig.DEFAULT_GLOBAL_STEP_UP, DegraderLoadBalancerStrategyConfig.DEFAULT_GLOBAL_STEP_DOWN, DegraderLoadBalancerStrategyConfig.DEFAULT_CLUSTER_MIN_CALL_COUNT_HIGH_WATER_MARK, DegraderLoadBalancerStrategyConfig.DEFAULT_CLUSTER_MIN_CALL_COUNT_LOW_WATER_MARK, DegraderLoadBalancerStrategyConfig.DEFAULT_HASHRING_POINT_CLEANUP_RATE, null, DegraderLoadBalancerStrategyConfig.DEFAULT_NUM_PROBES, DegraderLoadBalancerStrategyConfig.DEFAULT_POINTS_PER_HOST, DegraderLoadBalancerStrategyConfig.DEFAULT_BOUNDED_LOAD_BALANCING_FACTOR, null, DegraderLoadBalancerStrategyConfig.DEFAULT_QUARANTINE_MAXPERCENT, null, null, "OPTIONS", null, DegraderImpl.DEFAULT_LOW_LATENCY, null, DegraderLoadBalancerStrategyConfig.DEFAULT_LOW_EVENT_EMITTING_INTERVAL, DegraderLoadBalancerStrategyConfig.DEFAULT_HIGH_EVENT_EMITTING_INTERVAL, DegraderLoadBalancerStrategyConfig.DEFAULT_CLUSTER_NAME);
    DegraderTrackerClientUpdater updater2 = new DegraderTrackerClientUpdater(trackerClients.get(2), DEFAULT_PARTITION_ID);
    quarantine = new LoadBalancerQuarantine(updater2.getTrackerClient(), config2, "abc0");
    healthCheck = (TransportHealthCheck) quarantine.getHealthCheckClient();
    restRequest = healthCheck.getRestRequest();
    Assert.assertTrue(restRequest.getURI().equals(URI.create("http://test.linkedin.com:10010/abc2")));
    Assert.assertTrue(restRequest.getMethod().equals("OPTIONS"));
}

Example 9 with LoadBalancerQuarantine

use of com.linkedin.d2.balancer.strategies.LoadBalancerQuarantine in project rest.li by linkedin.

the class QuarantineManager method enrollNewQuarantineAndRecovery.

/**
 * Enroll new tracker client to quarantine or recovery state
 *
 * @param newPartitionState The new state of the partition
 * @param oldPartitionState The old state of the partition
 * @param quarantineLatency The latency threshold for D2 quarantine
 */
private void enrollNewQuarantineAndRecovery(PartitionState newPartitionState, PartitionState oldPartitionState, long quarantineLatency, long currentTime) {
    int partitionId = newPartitionState.getPartitionId();
    Map<TrackerClient, LoadBalancerQuarantine> quarantineMap = newPartitionState.getQuarantineMap();
    Map<TrackerClient, LoadBalancerQuarantine> quarantineHistory = newPartitionState.getQuarantineHistory();
    Set<TrackerClient> recoverySet = newPartitionState.getRecoveryTrackerClients();
    for (TrackerClient trackerClient : newPartitionState.getTrackerClients()) {
        TrackerClientState trackerClientState = newPartitionState.getTrackerClientStateMap().get(trackerClient);
        double serverWeight = trackerClient.getPartitionWeight(partitionId);
        // Check and enroll quarantine map
        boolean isQuarantined = enrollClientInQuarantineMap(trackerClient, trackerClientState, serverWeight, quarantineMap, quarantineHistory, newPartitionState.getTrackerClientStateMap().size(), quarantineLatency, currentTime);
        if (!isQuarantined) {
            if (!_fastRecoveryEnabled) {
                performNormalRecovery(trackerClientState);
            } else {
                // Only enroll the client into recovery state if fast recovery is enabled
                enrollSingleClientInRecoverySet(trackerClient, trackerClientState, serverWeight, recoverySet, oldPartitionState);
            }
        }
    }
}

Example 10 with LoadBalancerQuarantine

use of com.linkedin.d2.balancer.strategies.LoadBalancerQuarantine in project rest.li by linkedin.

the class DegraderLoadBalancerStrategyV3 method getUnhealthyTrackerClients.

private static List<DegraderTrackerClient> getUnhealthyTrackerClients(List<DegraderTrackerClientUpdater> degraderTrackerClientUpdaters, Map<URI, Integer> pointsMap, Map<DegraderTrackerClient, LoadBalancerQuarantine> quarantineMap, DegraderLoadBalancerStrategyConfig config, int partitionId) {
    List<DegraderTrackerClient> unhealthyClients = new ArrayList<>();
    for (DegraderTrackerClientUpdater clientUpdater : degraderTrackerClientUpdaters) {
        DegraderTrackerClient client = clientUpdater.getTrackerClient();
        int perfectHealth = (int) (client.getPartitionWeight(partitionId) * client.getSubsetWeight(partitionId) * config.getPointsPerWeight());
        URI uri = client.getUri();
        if (!pointsMap.containsKey(uri)) {
            _log.warn("Client with URI {} is absent in point map, pointMap={}, quarantineMap={}", new Object[] { uri, pointsMap, quarantineMap });
            continue;
        }
        Integer point = pointsMap.get(uri);
        if (point < perfectHealth) {
            unhealthyClients.add(client);
        }
    }
    return unhealthyClients;
}