Examples with ConsistentHashKeyMapper com.linkedin.d2.balancer.util.hashing.ConsistentHashKeyMapper used on opensource projects

Example 6 with ConsistentHashKeyMapper

use of com.linkedin.d2.balancer.util.hashing.ConsistentHashKeyMapper in project rest.li by linkedin.

the class ConsistentHashKeyMapperTest method getConsistentHashKeyMapper.

private ConsistentHashKeyMapper getConsistentHashKeyMapper(RingFactory<URI> ringFactory) throws URISyntaxException {
    String serviceName = "articles";
    String clusterName = "cluster";
    String path = "path";
    String strategyName = "degrader";
    //setup partition
    Map<URI, Map<Integer, PartitionData>> partitionDescriptions = new HashMap<URI, Map<Integer, PartitionData>>();
    final URI foo1 = new URI("http://foo1.com");
    Map<Integer, PartitionData> foo1Data = new HashMap<Integer, PartitionData>();
    foo1Data.put(0, new PartitionData(1.0));
    partitionDescriptions.put(foo1, foo1Data);
    final URI foo2 = new URI("http://foo2.com");
    Map<Integer, PartitionData> foo2Data = new HashMap<Integer, PartitionData>();
    foo2Data.put(3, new PartitionData(1.0));
    foo2Data.put(4, new PartitionData(1.0));
    partitionDescriptions.put(foo2, foo2Data);
    final URI foo3 = new URI("http://foo3.com");
    Map<Integer, PartitionData> foo3Data = new HashMap<Integer, PartitionData>();
    foo3Data.put(0, new PartitionData(1.0));
    partitionDescriptions.put(foo3, foo3Data);
    final URI foo4 = new URI("http://foo4.com");
    Map<Integer, PartitionData> foo4Data = new HashMap<Integer, PartitionData>();
    foo4Data.put(1, new PartitionData(1.0));
    partitionDescriptions.put(foo4, foo4Data);
    final URI foo5 = new URI("http://foo5.com");
    Map<Integer, PartitionData> foo5Data = new HashMap<Integer, PartitionData>();
    foo5Data.put(1, new PartitionData(1.0));
    partitionDescriptions.put(foo5, foo5Data);
    final URI foo6 = new URI("http://foo6.com");
    Map<Integer, PartitionData> foo6Data = new HashMap<Integer, PartitionData>();
    foo6Data.put(1, new PartitionData(1.0));
    partitionDescriptions.put(foo6, foo6Data);
    //setup strategy which involves tweaking the hash ring to get partitionId -> URI host
    List<LoadBalancerState.SchemeStrategyPair> orderedStrategies = new ArrayList<LoadBalancerState.SchemeStrategyPair>();
    LoadBalancerStrategy strategy = new TestLoadBalancerStrategy(partitionDescriptions, ringFactory);
    orderedStrategies.add(new LoadBalancerState.SchemeStrategyPair("http", strategy));
    //setup the partition accessor which is used to get partitionId -> keys
    PartitionAccessor accessor = new TestPartitionAccessor();
    URI serviceURI = new URI("d2://" + serviceName);
    SimpleLoadBalancer balancer = new SimpleLoadBalancer(new PartitionedLoadBalancerTestState(clusterName, serviceName, path, strategyName, partitionDescriptions, orderedStrategies, accessor));
    ConsistentHashKeyMapper mapper = new ConsistentHashKeyMapper(balancer, balancer);
    return mapper;
}

Example 7 with ConsistentHashKeyMapper

use of com.linkedin.d2.balancer.util.hashing.ConsistentHashKeyMapper in project rest.li by linkedin.

the class ConsistentHashKeyMapperTest method testMapKeysConcurrency.

@SuppressWarnings("rawtypes")
@Test
public void testMapKeysConcurrency() throws Exception {
    String serviceName = "articles";
    String clusterName = "cluster";
    String path = "path";
    String strategyName = "degrader";
    int numPartitions = 500;
    // setup partition
    Map<URI, Map<Integer, PartitionData>> partitionDescriptions = new HashMap<URI, Map<Integer, PartitionData>>();
    final URI foo1 = new URI("http://foo1.com");
    Map<Integer, PartitionData> foo1Data = new HashMap<Integer, PartitionData>();
    for (int i = 0; i < numPartitions; i++) {
        foo1Data.put(i, new PartitionData(1.0));
    }
    partitionDescriptions.put(foo1, foo1Data);
    DegraderLoadBalancerStrategyV3 strategy = new DegraderLoadBalancerStrategyV3(new DegraderLoadBalancerStrategyConfig(5000), serviceName, null);
    List<LoadBalancerState.SchemeStrategyPair> orderedStrategies = new ArrayList<LoadBalancerState.SchemeStrategyPair>();
    orderedStrategies.add(new LoadBalancerState.SchemeStrategyPair("http", strategy));
    PartitionAccessor accessor = new TestDeadlockPartitionAccessor(numPartitions);
    SimpleLoadBalancer balancer = new SimpleLoadBalancer(new PartitionedLoadBalancerTestState(clusterName, serviceName, path, strategyName, partitionDescriptions, orderedStrategies, accessor));
    ConsistentHashKeyMapper mapper = new ConsistentHashKeyMapper(balancer, balancer);
    CountDownLatch latch = new CountDownLatch(numPartitions);
    List<Runnable> runnables = createRunnables(numPartitions, mapper, serviceName, latch);
    final ExecutorService executor = Executors.newFixedThreadPool(numPartitions);
    List<Future> futures = new ArrayList<Future>();
    for (int i = 0; i < numPartitions; i++) {
        futures.add(executor.submit(runnables.get(i)));
    }
    for (Future future : futures) {
        future.get(30, TimeUnit.SECONDS);
    }
}

Example 8 with ConsistentHashKeyMapper

use of com.linkedin.d2.balancer.util.hashing.ConsistentHashKeyMapper in project rest.li by linkedin.

the class TestScatterGather method getKeyToHostMapper.

private static ConsistentHashKeyMapper getKeyToHostMapper(int n, int partitionNum) throws URISyntaxException {
    Map<URI, Integer> endpoints = new HashMap<URI, Integer>();
    for (int ii = 0; ii < n; ++ii) {
        endpoints.put(new URI("test" + String.valueOf(ii)), 100);
    }
    final int partitionSize = endpoints.size() / partitionNum;
    List<Map<URI, Integer>> mapList = new ArrayList<Map<URI, Integer>>();
    int count = 0;
    for (final URI uri : endpoints.keySet()) {
        final int index = count / partitionSize;
        if (index == mapList.size()) {
            mapList.add(new HashMap<URI, Integer>());
        }
        Map<URI, Integer> map = mapList.get(index);
        map.put(uri, endpoints.get(uri));
        count++;
    }
    List<Ring<URI>> rings = new ArrayList<Ring<URI>>();
    for (final Map<URI, Integer> map : mapList) {
        final ConsistentHashRing<URI> ring = new ConsistentHashRing<URI>(map);
        rings.add(ring);
    }
    return new ConsistentHashKeyMapper(new StaticRingProvider(rings), new TestPartitionInfoProvider());
}

Example 9 with ConsistentHashKeyMapper

use of com.linkedin.d2.balancer.util.hashing.ConsistentHashKeyMapper in project rest.li by linkedin.

the class ConsistentHashKeyMapperTest method createRunnables.

/**
   * Create tasks for the deadlock test
   */
private List<Runnable> createRunnables(int num, final ConsistentHashKeyMapper mapper, String serviceName, final CountDownLatch latch) throws URISyntaxException {
    final URI serviceURI = new URI("d2://" + serviceName);
    List<Runnable> runnables = new ArrayList<Runnable>();
    for (int i = 0; i < num; i++) {
        // since i < numPartitions, the keys will be distributed to different partitions
        final List<String> keys = generateKeys(i);
        Runnable runnable = new Runnable() {

            @Override
            public void run() {
                // wait until all jobs submitted
                latch.countDown();
                try {
                    latch.await();
                    mapper.mapKeysV3(serviceURI, keys, 1);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                } catch (ServiceUnavailableException e) {
                    e.printStackTrace();
                }
            }
        };
        runnables.add(runnable);
    }
    return runnables;
}

Example 10 with ConsistentHashKeyMapper

use of com.linkedin.d2.balancer.util.hashing.ConsistentHashKeyMapper in project rest.li by linkedin.

the class TestAllPartitionsRequestBuilder method testSendAllPartitionsRequests.

@Test(dataProvider = com.linkedin.restli.internal.common.TestConstants.RESTLI_PROTOCOL_1_2_PREFIX + "restliRequestOptions")
public void testSendAllPartitionsRequests(RestliRequestOptions options, RingFactory<URI> ringFactory) throws ServiceUnavailableException, URISyntaxException, RestException, InterruptedException {
    final int PARTITION_NUM = 5;
    List<URI> expectedUris = new ArrayList<URI>();
    ConsistentHashKeyMapper mapper = getKeyToHostMapper(PARTITION_NUM, expectedUris, ringFactory);
    AllPartitionsRequestBuilder<Greeting> searchRB = new AllPartitionsRequestBuilder<Greeting>(mapper);
    ActionRequestBuilder<Long, Greeting> builder = new ActionRequestBuilder<Long, Greeting>(TEST_URI, Greeting.class, _COLL_SPEC, options);
    ActionRequest<Greeting> request = builder.name("updateTone").id(1L).setParam(new FieldDef<Tone>("newTone", Tone.class, DataTemplateUtil.getSchema(Tone.class)), Tone.FRIENDLY).build();
    final Map<String, Greeting> results = new ConcurrentHashMap<String, Greeting>();
    final CountDownLatch latch = new CountDownLatch(PARTITION_NUM);
    final List<Throwable> errors = new ArrayList<Throwable>();
    final List<Greeting> responses = new ArrayList<Greeting>();
    Callback<Response<Greeting>> cb = new Callback<Response<Greeting>>() {

        @Override
        public void onError(Throwable e) {
            synchronized (errors) {
                errors.add(e);
            }
            latch.countDown();
        }

        @Override
        public void onSuccess(Response<Greeting> response) {
            results.put(response.getEntity().toString(), response.getEntity());
            synchronized (responses) {
                responses.add(response.getEntity());
            }
            latch.countDown();
        }
    };
    HostSet hostsResult = searchRB.sendRequests(getClient(), request, new RequestContext(), cb);
    List<URI> uris = hostsResult.getAllHosts();
    Assert.assertTrue(uris.containsAll(expectedUris));
    Assert.assertTrue(expectedUris.containsAll(uris));
    latch.await();
    if (!errors.isEmpty()) {
        Assert.fail("I knew it: " + errors.toString());
    }
    Assert.assertEquals(PARTITION_NUM, responses.size());
}