Examples with CallCompletion com.linkedin.util.degrader.CallCompletion used on opensource projects

Example 1 with CallCompletion

use of com.linkedin.util.degrader.CallCompletion in project rest.li by linkedin.

the class DegraderLoadBalancerTest method testHighLowWatermarks.

@Test(groups = { "small", "back-end" })
public void testHighLowWatermarks() {
    final int NUM_CHECKS = 5;
    Map<String, Object> myMap = new HashMap<String, Object>();
    Long timeInterval = 5000L;
    double globalStepUp = 0.4;
    double globalStepDown = 0.4;
    double highWaterMark = 1000;
    double lowWaterMark = 50;
    TestClock clock = new TestClock();
    myMap.put(PropertyKeys.CLOCK, clock);
    myMap.put(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_INTERVAL_MS, timeInterval);
    myMap.put(PropertyKeys.HTTP_LB_GLOBAL_STEP_UP, globalStepUp);
    myMap.put(PropertyKeys.HTTP_LB_GLOBAL_STEP_DOWN, globalStepDown);
    myMap.put(PropertyKeys.HTTP_LB_HIGH_WATER_MARK, highWaterMark);
    myMap.put(PropertyKeys.HTTP_LB_LOW_WATER_MARK, lowWaterMark);
    myMap.put(PropertyKeys.HTTP_LB_CLUSTER_MIN_CALL_COUNT_HIGH_WATER_MARK, 1l);
    myMap.put(PropertyKeys.HTTP_LB_CLUSTER_MIN_CALL_COUNT_LOW_WATER_MARK, 1l);
    DegraderLoadBalancerStrategyConfig config = DegraderLoadBalancerStrategyConfig.createHttpConfigFromMap(myMap);
    DegraderLoadBalancerStrategyV3 strategy = new DegraderLoadBalancerStrategyV3(config, "DegraderLoadBalancerTest", null);
    List<TrackerClient> clients = new ArrayList<TrackerClient>();
    URI uri1 = URI.create("http://test.linkedin.com:3242/fdsaf");
    URIRequest request = new URIRequest(uri1);
    TrackerClient client1 = new TrackerClient(uri1, getDefaultPartitionData(1d), new TestLoadBalancerClient(uri1), clock, null);
    clients.add(client1);
    DegraderControl dcClient1Default = client1.getDegraderControl(DEFAULT_PARTITION_ID);
    dcClient1Default.setOverrideMinCallCount(5);
    dcClient1Default.setMinCallCount(5);
    List<CallCompletion> ccList = new ArrayList<CallCompletion>();
    CallCompletion cc;
    TrackerClient resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    // The override drop rate should be zero at this point.
    assertEquals(dcClient1Default.getOverrideDropRate(), 0.0);
    // make high latency calls to the tracker client, verify the override drop rate doesn't change
    for (int j = 0; j < NUM_CHECKS; j++) {
        cc = client1.getCallTracker().startCall();
        ccList.add(cc);
    }
    clock.addMs((long) highWaterMark);
    for (Iterator<CallCompletion> iter = ccList.listIterator(); iter.hasNext(); ) {
        cc = iter.next();
        cc.endCall();
        iter.remove();
    }
    // go to next time interval.
    clock.addMs(timeInterval);
    // try call dropping on the next updateState
    strategy.setStrategy(DEFAULT_PARTITION_ID, DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy.CALL_DROPPING);
    resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    // we now expect that the override drop rate stepped up because updateState
    // made that decision.
    assertEquals(dcClient1Default.getOverrideDropRate(), globalStepUp);
    // make mid latency calls to the tracker client, verify the override drop rate doesn't change
    for (int j = 0; j < NUM_CHECKS; j++) {
        // need to use client1 because the resultTC may be null
        cc = client1.getCallTracker().startCall();
        ccList.add(cc);
    }
    clock.addMs((long) highWaterMark - 1);
    for (Iterator<CallCompletion> iter = ccList.listIterator(); iter.hasNext(); ) {
        cc = iter.next();
        cc.endCall();
        iter.remove();
    }
    // go to next time interval.
    clock.addMs(timeInterval);
    double previousOverrideDropRate = dcClient1Default.getOverrideDropRate();
    // try call dropping on the next updateState
    strategy.setStrategy(DEFAULT_PARTITION_ID, DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy.CALL_DROPPING);
    resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    assertEquals(dcClient1Default.getOverrideDropRate(), previousOverrideDropRate);
    // make low latency calls to the tracker client, verify the override drop rate decreases
    for (int j = 0; j < NUM_CHECKS; j++) {
        cc = client1.getCallTracker().startCall();
        ccList.add(cc);
    }
    clock.addMs((long) lowWaterMark);
    for (Iterator<CallCompletion> iter = ccList.listIterator(); iter.hasNext(); ) {
        cc = iter.next();
        cc.endCall();
        iter.remove();
    }
    // go to next time interval.
    clock.addMs(timeInterval);
    // try Call dropping on this updateState
    strategy.setStrategy(DEFAULT_PARTITION_ID, DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy.CALL_DROPPING);
    resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    assertEquals(resultTC.getDegraderControl(DEFAULT_PARTITION_ID).getOverrideDropRate(), 0.0);
}

Example 2 with CallCompletion

use of com.linkedin.util.degrader.CallCompletion in project rest.li by linkedin.

the class DegraderLoadBalancerTest method callClients.

/**
   * simulates calling clients
   * @param milliseconds latency of the call
   * @param qps qps of traffic per client
   * @param clients list of clients being called
   * @param clock
   * @param timeInterval
   * @param withError calling client with error that we don't use for load balancing (any generic error)
   * @param withQualifiedDegraderError calling client with error that we use for load balancing
   */
private void callClients(long milliseconds, double qps, List<TrackerClient> clients, TestClock clock, long timeInterval, boolean withError, boolean withQualifiedDegraderError) {
    LinkedList<CallCompletion> callCompletions = new LinkedList<CallCompletion>();
    int callHowManyTimes = (int) ((qps * timeInterval) / 1000);
    for (int i = 0; i < callHowManyTimes; i++) {
        for (TrackerClient client : clients) {
            CallCompletion cc = client.getCallTracker().startCall();
            callCompletions.add(cc);
        }
    }
    Random random = new Random();
    clock.addMs(milliseconds);
    for (CallCompletion cc : callCompletions) {
        if (withError) {
            cc.endCallWithError();
        } else if (withQualifiedDegraderError) {
            //choose a random error type
            if (random.nextBoolean()) {
                cc.endCallWithError(ErrorType.CLOSED_CHANNEL_EXCEPTION);
            } else {
                cc.endCallWithError(ErrorType.CONNECT_EXCEPTION);
            }
        } else {
            cc.endCall();
        }
    }
    //complete a full interval cycle
    clock.addMs(timeInterval - (milliseconds % timeInterval));
}

Example 3 with CallCompletion

use of com.linkedin.util.degrader.CallCompletion in project rest.li by linkedin.

the class DegraderLoadBalancerTest method testClusterRecovery2TC.

@Test(groups = { "small", "back-end" })
public void testClusterRecovery2TC() {
    final int NUM_CHECKS = 5;
    final Long TIME_INTERVAL = 5000L;
    Map<String, Object> myMap = new HashMap<String, Object>();
    // 1,2,4,8,16,32,64,100% steps, given a 2x recovery step coefficient
    int localStepsToFullRecovery = 8;
    myMap.put(PropertyKeys.HTTP_LB_INITIAL_RECOVERY_LEVEL, 0.005);
    myMap.put(PropertyKeys.HTTP_LB_RING_RAMP_FACTOR, 2d);
    myMap.put(PropertyKeys.HTTP_LB_STRATEGY_PROPERTIES_UPDATE_INTERVAL_MS, TIME_INTERVAL);
    TestClock clock = new TestClock();
    myMap.put(PropertyKeys.CLOCK, clock);
    DegraderLoadBalancerStrategyConfig config = DegraderLoadBalancerStrategyConfig.createHttpConfigFromMap(myMap);
    DegraderLoadBalancerStrategyV3 strategy = new DegraderLoadBalancerStrategyV3(config, "DegraderLoadBalancerTest", null);
    List<TrackerClient> clients = new ArrayList<TrackerClient>();
    URI uri1 = URI.create("http://test.linkedin.com:3242/fdsaf");
    URI uri2 = URI.create("http://test.linkedin.com:3243/fdsaf");
    URIRequest request = new URIRequest(uri1);
    List<CallCompletion> ccList = new ArrayList<CallCompletion>();
    CallCompletion cc;
    TrackerClient client1 = new TrackerClient(uri1, getDefaultPartitionData(1d), new TestLoadBalancerClient(uri1), clock, null);
    TrackerClient client2 = new TrackerClient(uri2, getDefaultPartitionData(1d), new TestLoadBalancerClient(uri2), clock, null);
    clients.add(client1);
    clients.add(client2);
    // force client1 to be disabled if we encounter errors/high latency
    DegraderControl dcClient1Default = client1.getDegraderControl(DEFAULT_PARTITION_ID);
    dcClient1Default.setMinCallCount(5);
    dcClient1Default.setOverrideMinCallCount(5);
    dcClient1Default.setUpStep(1.0);
    // force client2 to be disabled if we encounter errors/high latency
    DegraderControl dcClient2Default = client2.getDegraderControl(DEFAULT_PARTITION_ID);
    dcClient2Default.setOverrideMinCallCount(5);
    dcClient2Default.setMinCallCount(5);
    dcClient2Default.setUpStep(0.4);
    // Have one cycle of successful calls to verify valid tracker clients returned.
    // try load balancing on this updateState, need to updateState before forcing the strategy.
    TrackerClient resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    strategy.setStrategy(DEFAULT_PARTITION_ID, DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy.LOAD_BALANCE);
    resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    assertNotNull(resultTC, "expected non-null trackerclient");
    for (int j = 0; j < NUM_CHECKS; j++) {
        ccList.add(client1.getCallTracker().startCall());
        ccList.add(client2.getCallTracker().startCall());
    }
    clock.addMs(1);
    for (Iterator<CallCompletion> iter = ccList.listIterator(); iter.hasNext(); ) {
        cc = iter.next();
        cc.endCall();
    }
    // bump to next interval, and get stats.
    clock.addMs(5000);
    // try Load balancing on this updateState
    strategy.setStrategy(DEFAULT_PARTITION_ID, DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy.LOAD_BALANCE);
    resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    assertNotNull(resultTC, "expected non-null trackerclient");
    Assert.assertEquals(dcClient1Default.getCurrentComputedDropRate(), 0.0);
    Assert.assertEquals(dcClient2Default.getCurrentComputedDropRate(), 0.0);
    // now simulate a bad cluster state with high error and high latency
    for (int j = 0; j < NUM_CHECKS; j++) {
        ccList.add(client1.getCallTracker().startCall());
        ccList.add(client2.getCallTracker().startCall());
    }
    clock.addMs(3500);
    for (Iterator<CallCompletion> iter = ccList.listIterator(); iter.hasNext(); ) {
        cc = iter.next();
        cc.endCallWithError();
    }
    // go to next interval
    clock.addMs(5000);
    Assert.assertEquals(dcClient1Default.getCurrentComputedDropRate(), 1.0);
    Assert.assertEquals(dcClient2Default.getCurrentComputedDropRate(), 0.4);
    // trigger a state update, the returned TrackerClient should be client2
    // because client 1 should have gone up to a 1.0 drop rate, and the cluster should
    // be unhealthy
    strategy.setStrategy(DEFAULT_PARTITION_ID, DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy.LOAD_BALANCE);
    resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    assertEquals(resultTC, client2);
    // it's current drop rate.
    do {
        // go to next time interval.
        clock.addMs(TIME_INTERVAL);
        // adjust the hash ring this time.
        strategy.setStrategy(DEFAULT_PARTITION_ID, DegraderLoadBalancerStrategyV3.PartitionDegraderLoadBalancerState.Strategy.LOAD_BALANCE);
        resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
        localStepsToFullRecovery--;
    } while (localStepsToFullRecovery > 0);
    assertNotNull(resultTC, "expected non-null trackerclient");
    assertTrue(strategy.getState().getPartitionState(DEFAULT_PARTITION_ID).getPointsMap().get(client1.getUri()) == client1.getPartitionWeight(DEFAULT_PARTITION_ID) * config.getPointsPerWeight(), "client1 did not recover to full weight in hash map.");
    Assert.assertEquals(dcClient2Default.getCurrentComputedDropRate(), 0.4, "client2 drop rate not as expected");
    cc = client1.getCallTracker().startCall();
    clock.addMs(10);
    cc.endCall();
    clock.addMs(TIME_INTERVAL);
    resultTC = getTrackerClient(strategy, request, new RequestContext(), 1, clients);
    assertNotNull(resultTC, "expected non-null trackerclient");
}

Example 4 with CallCompletion

use of com.linkedin.util.degrader.CallCompletion in project rest.li by linkedin.

the class DegraderLoadBalancerTest method testWeightedAndLatencyDegradationBalancingRing.

@Test(groups = { "small", "back-end" }, dataProvider = "consistentHashAlgorithms")
public void testWeightedAndLatencyDegradationBalancingRing(String consistentHashAlgorithm) throws URISyntaxException {
    DegraderLoadBalancerStrategyV3 strategy = getStrategy(consistentHashAlgorithm);
    List<TrackerClient> clients = new ArrayList<TrackerClient>();
    URI uri1 = URI.create("http://test.linkedin.com:3242/fdsaf");
    URI uri2 = URI.create("http://test.linkedin.com:3243/fdsaf");
    TestClock clock1 = new TestClock();
    TestClock clock2 = new TestClock();
    TrackerClient client1 = new TrackerClient(uri1, getDefaultPartitionData(1d), new TestLoadBalancerClient(uri1), clock1, null);
    TrackerClient client2 = new TrackerClient(uri2, getDefaultPartitionData(0.8d), new TestLoadBalancerClient(uri2), clock2, null);
    clients.add(client1);
    clients.add(client2);
    DegraderControl dcClient2Default = client2.getDegraderControl(DEFAULT_PARTITION_ID);
    dcClient2Default.setOverrideMinCallCount(1);
    dcClient2Default.setMinCallCount(1);
    dcClient2Default.setMaxDropRate(1d);
    dcClient2Default.setUpStep(0.4d);
    dcClient2Default.setHighErrorRate(0);
    CallCompletion cc = client2.getCallTracker().startCall();
    clock2.addMs(1);
    cc.endCallWithError();
    clock1.addMs(15000);
    clock2.addMs(5000);
    // trigger a state update
    assertNotNull(getTrackerClient(strategy, null, new RequestContext(), 1, clients));
    // now do a basic verification to verify getTrackerClient is properly weighting things
    double calls = 10000d;
    int client1Count = 0;
    int client2Count = 0;
    double tolerance = 0.05d;
    for (int i = 0; i < calls; ++i) {
        TrackerClient client = getTrackerClient(strategy, null, new RequestContext(), 1, clients);
        assertNotNull(client);
        if (client.getUri().equals(uri1)) {
            ++client1Count;
        } else {
            ++client2Count;
        }
    }
    assertTrue(Math.abs((client1Count / calls) - (100 / 148d)) < tolerance);
    assertTrue(Math.abs((client2Count / calls) - (48 / 148d)) < tolerance);
}

Example 5 with CallCompletion

use of com.linkedin.util.degrader.CallCompletion in project rest.li by linkedin.

the class DegraderLoadBalancerTest method testWeightedAndLatencyDegradationBalancingRingWithPartitions.

@Test(groups = { "small", "back-end" }, dataProvider = "consistentHashAlgorithms")
public void testWeightedAndLatencyDegradationBalancingRingWithPartitions(String consistentHashAlgorithm) throws URISyntaxException {
    DegraderLoadBalancerStrategyV3 strategy = getStrategy(consistentHashAlgorithm);
    List<TrackerClient> clientsForPartition0 = new ArrayList<TrackerClient>();
    List<TrackerClient> clientsForPartition1 = new ArrayList<TrackerClient>();
    URI uri1 = URI.create("http://someTestService/someTestUrl");
    URI uri2 = URI.create("http://abcxfweuoeueoueoueoukeueoueoueoueoueouo/2354");
    URI uri3 = URI.create("http://slashdot/blah");
    URI uri4 = URI.create("http://idle/server");
    TestClock clock1 = new TestClock();
    TestClock clock2 = new TestClock();
    TestClock clock3 = new TestClock();
    @SuppressWarnings("serial") TrackerClient client1 = new TrackerClient(uri1, new HashMap<Integer, PartitionData>() {

        {
            put(0, new PartitionData(1d));
        }
    }, new TestLoadBalancerClient(uri1), clock1, null);
    @SuppressWarnings("serial") TrackerClient client2 = new TrackerClient(uri2, new HashMap<Integer, PartitionData>() {

        {
            put(0, new PartitionData(0.5d));
            put(1, new PartitionData(0.5d));
        }
    }, new TestLoadBalancerClient(uri2), clock2, null);
    @SuppressWarnings("serial") TrackerClient client3 = new TrackerClient(uri3, new HashMap<Integer, PartitionData>() {

        {
            put(1, new PartitionData(1d));
        }
    }, new TestLoadBalancerClient(uri3), clock3, null);
    final int partitionId0 = 0;
    clientsForPartition0.add(client1);
    clientsForPartition0.add(client2);
    final int partitionId1 = 1;
    clientsForPartition1.add(client2);
    clientsForPartition1.add(client3);
    // force client2 to be disabled
    DegraderControl dcClient2Partition0 = client2.getDegraderControl(0);
    DegraderControl dcClient2Partition1 = client2.getDegraderControl(1);
    dcClient2Partition0.setOverrideMinCallCount(1);
    dcClient2Partition0.setMinCallCount(1);
    dcClient2Partition0.setMaxDropRate(1d);
    dcClient2Partition0.setUpStep(0.4d);
    dcClient2Partition0.setHighErrorRate(0);
    dcClient2Partition1.setOverrideMinCallCount(1);
    dcClient2Partition1.setMinCallCount(1);
    dcClient2Partition1.setMaxDropRate(1d);
    dcClient2Partition1.setUpStep(0.4d);
    dcClient2Partition1.setHighErrorRate(0);
    CallCompletion cc = client2.getCallTracker().startCall();
    clock2.addMs(1);
    cc.endCallWithError();
    // force client3 to be disabled
    DegraderControl dcClient3Partition1 = client3.getDegraderControl(1);
    dcClient3Partition1.setOverrideMinCallCount(1);
    dcClient3Partition1.setMinCallCount(1);
    dcClient3Partition1.setMaxDropRate(1d);
    dcClient3Partition1.setHighErrorRate(0);
    dcClient3Partition1.setUpStep(0.2d);
    CallCompletion cc3 = client3.getCallTracker().startCall();
    clock3.addMs(1);
    cc3.endCallWithError();
    clock1.addMs(15000);
    clock2.addMs(5000);
    clock3.addMs(5000);
    // trigger a state update
    assertNotNull(strategy.getTrackerClient(null, new RequestContext(), 1, partitionId0, clientsForPartition0));
    assertNotNull(strategy.getTrackerClient(null, new RequestContext(), 1, partitionId1, clientsForPartition1));
    assertNotNull(strategy.getRing(1, partitionId0, clientsForPartition0));
    assertNotNull(strategy.getRing(1, partitionId1, clientsForPartition1));
    // now do a basic verification to verify getTrackerClient is properly weighting things
    int calls = 10000;
    int client1Count = 0;
    int client2Count = 0;
    double tolerance = 0.05d;
    for (int i = 0; i < calls; ++i) {
        TrackerClient client = strategy.getTrackerClient(null, new RequestContext(), 1, partitionId0, clientsForPartition0);
        assertNotNull(client);
        if (client.getUri().equals(uri1)) {
            ++client1Count;
        } else {
            ++client2Count;
        }
    }
    assertTrue(Math.abs((client1Count / (double) calls) - (100 / 130d)) < tolerance);
    assertTrue(Math.abs((client2Count / (double) calls) - (30 / 130d)) < tolerance);
    client2Count = 0;
    int client3Count = 0;
    int client4Count = 0;
    for (int i = 0; i < calls; ++i) {
        TrackerClient client = strategy.getTrackerClient(null, new RequestContext(), 1, partitionId1, clientsForPartition1);
        assertNotNull(client);
        if (client.getUri().equals(uri3)) {
            ++client3Count;
        } else if (client.getUri().equals(uri2)) {
            ++client2Count;
        } else {
            ++client4Count;
        }
    }
    assertTrue(Math.abs((client3Count / (double) calls) - (80 / 110d)) < tolerance);
    assertTrue(Math.abs((client2Count / (double) calls) - (30 / 110d)) < tolerance);
    assertTrue(client4Count == 0);
}