Search in sources :

Example 6 with RingBasedUriMapper

use of com.linkedin.d2.balancer.util.hashing.RingBasedUriMapper in project rest.li by linkedin.

the class RingBasedURIMapperTest method testMapUrisStickyRoutingOnly.

@Test
public void testMapUrisStickyRoutingOnly() throws ServiceUnavailableException, PartitionAccessException {
    int partitionCount = 1;
    int requestPerPartition = 1000;
    int totalHostCount = 100;
    HashRingProvider ringProvider = createStaticHashRingProvider(totalHostCount, partitionCount, getHashFunction(true));
    PartitionInfoProvider infoProvider = createRangeBasedPartitionInfoProvider(partitionCount);
    URIMapper mapper = new RingBasedUriMapper(ringProvider, infoProvider);
    List<URIKeyPair<Integer>> requests = testUtil.generateRequests(partitionCount, requestPerPartition);
    URIMappingResult<Integer> results1 = mapper.mapUris(requests);
    URIMappingResult<Integer> results2 = mapper.mapUris(requests);
    // Sticky routing between two runs
    Assert.assertEquals(results1.getMappedKeys(), results2.getMappedKeys());
    Assert.assertEquals(results1.getUnmappedKeys(), results2.getUnmappedKeys());
    Map<URI, Set<Integer>> mapping = results1.getMappedKeys();
    // Testing universal stickiness, take out 50 requests randomly and make sure they would be resolved to the same host as does URIMapper
    Collections.shuffle(requests);
    HashFunction<Request> hashFunction = ringProvider.getRequestHashFunction(TEST_SERVICE);
    for (int i = 0; i < 50; i++) {
        URIKeyPair<Integer> request = requests.get(i);
        int partitionId = infoProvider.getPartitionAccessor(TEST_SERVICE).getPartitionId(request.getRequestUri());
        Ring<URI> ring = ringProvider.getRings(request.getRequestUri()).get(partitionId);
        URI uri = ring.get(hashFunction.hash(new URIRequest(request.getRequestUri())));
        Assert.assertTrue(mapping.keySet().contains(uri));
    }
    // Only one partition
    Assert.assertEquals(1, new HashSet<>(results1.getHostPartitionInfo().values()).size());
    Assert.assertEquals(1, new HashSet<>(results2.getHostPartitionInfo().values()).size());
}
Also used : Set(java.util.Set) HashSet(java.util.HashSet) PartitionInfoProvider(com.linkedin.d2.balancer.util.partitions.PartitionInfoProvider) URIRequest(com.linkedin.d2.balancer.util.URIRequest) Request(com.linkedin.r2.message.Request) URIRequest(com.linkedin.d2.balancer.util.URIRequest) URI(java.net.URI) URIKeyPair(com.linkedin.d2.balancer.util.URIKeyPair) URIMapper(com.linkedin.d2.balancer.URIMapper) HashSet(java.util.HashSet) Test(org.testng.annotations.Test)

Example 7 with RingBasedUriMapper

use of com.linkedin.d2.balancer.util.hashing.RingBasedUriMapper in project rest.li by linkedin.

the class RingBasedURIMapperTest method testSameHostSupportingMultiplePartitions.

/**
 * If one host supports multiple partitions and for those partitions, this same one host happens to be picked, URIMapper should
 * merge the key entries for those partitions.
 */
@Test
public void testSameHostSupportingMultiplePartitions() throws ServiceUnavailableException {
    int partitionCount = 10;
    int requestPerPartition = 100;
    // one host supporting 10 partitions
    URI host = createHostURI(0, 0);
    List<Ring<URI>> rings = IntStream.range(0, partitionCount).boxed().map(i -> new MPConsistentHashRing<>(Collections.singletonMap(host, 100))).collect(Collectors.toList());
    StaticRingProvider ringProvider = new StaticRingProvider(rings);
    ringProvider.setHashFunction(new RandomHash());
    PartitionInfoProvider infoProvider = createRangeBasedPartitionInfoProvider(partitionCount);
    URIMapper mapper = new RingBasedUriMapper(ringProvider, infoProvider);
    List<URIKeyPair<Integer>> requests = testUtil.generateRequests(partitionCount, requestPerPartition);
    URIMappingResult<Integer> results = mapper.mapUris(requests);
    Map<URI, Set<Integer>> mapping = results.getMappedKeys();
    Map<Integer, Set<Integer>> unmappedKeys = results.getUnmappedKeys();
    Assert.assertTrue(unmappedKeys.isEmpty());
    Assert.assertEquals(1, mapping.size());
    Assert.assertEquals(1000, mapping.values().iterator().next().size());
}
Also used : IntStream(java.util.stream.IntStream) PartitionAccessException(com.linkedin.d2.balancer.util.partitions.PartitionAccessException) Arrays(java.util.Arrays) DataProvider(org.testng.annotations.DataProvider) URIKeyPair(com.linkedin.d2.balancer.util.URIKeyPair) URISyntaxException(java.net.URISyntaxException) Set(java.util.Set) HashMap(java.util.HashMap) Test(org.testng.annotations.Test) URIMappingResult(com.linkedin.d2.balancer.util.URIMappingResult) URIMapper(com.linkedin.d2.balancer.URIMapper) Collectors(java.util.stream.Collectors) ArrayList(java.util.ArrayList) URIRequest(com.linkedin.d2.balancer.util.URIRequest) HashSet(java.util.HashSet) List(java.util.List) Request(com.linkedin.r2.message.Request) Assert(org.testng.Assert) Map(java.util.Map) URI(java.net.URI) PartitionInfoProvider(com.linkedin.d2.balancer.util.partitions.PartitionInfoProvider) ServiceUnavailableException(com.linkedin.d2.balancer.ServiceUnavailableException) Collections(java.util.Collections) URIMapperTestUtil(com.linkedin.d2.balancer.util.hashing.URIMapperTestUtil) Set(java.util.Set) HashSet(java.util.HashSet) PartitionInfoProvider(com.linkedin.d2.balancer.util.partitions.PartitionInfoProvider) URI(java.net.URI) URIKeyPair(com.linkedin.d2.balancer.util.URIKeyPair) URIMapper(com.linkedin.d2.balancer.URIMapper) Test(org.testng.annotations.Test)

Example 8 with RingBasedUriMapper

use of com.linkedin.d2.balancer.util.hashing.RingBasedUriMapper in project rest.li by linkedin.

the class RingBasedURIMapperTest method testNonStickyAndNonPartitioning.

@Test
public void testNonStickyAndNonPartitioning() throws ServiceUnavailableException {
    int partitionCount = 1;
    int requestPerPartition = 1000;
    int totalHostCount = 100;
    HashRingProvider ringProvider = createStaticHashRingProvider(totalHostCount, partitionCount, getHashFunction(false));
    PartitionInfoProvider infoProvider = createRangeBasedPartitionInfoProvider(partitionCount);
    URIMapper mapper = new RingBasedUriMapper(ringProvider, infoProvider);
    List<URIKeyPair<Integer>> requests = testUtil.generateRequests(partitionCount, requestPerPartition);
    URIMappingResult<Integer> results = mapper.mapUris(requests);
    Map<URI, Set<Integer>> mapping = results.getMappedKeys();
    Map<Integer, Set<Integer>> unmappedKeys = results.getUnmappedKeys();
    Map<URI, Integer> hostToPartitionId = results.getHostPartitionInfo();
    Assert.assertTrue(unmappedKeys.isEmpty());
    Assert.assertEquals(1, mapping.size());
    Assert.assertEquals(1, hostToPartitionId.size());
    Assert.assertEquals(1000, mapping.values().iterator().next().size());
}
Also used : Set(java.util.Set) HashSet(java.util.HashSet) PartitionInfoProvider(com.linkedin.d2.balancer.util.partitions.PartitionInfoProvider) URI(java.net.URI) URIKeyPair(com.linkedin.d2.balancer.util.URIKeyPair) URIMapper(com.linkedin.d2.balancer.URIMapper) Test(org.testng.annotations.Test)

Example 9 with RingBasedUriMapper

use of com.linkedin.d2.balancer.util.hashing.RingBasedUriMapper in project rest.li by linkedin.

the class RingBasedURIMapperTest method testMapUrisPartitionedOnly.

@Test
public void testMapUrisPartitionedOnly() throws ServiceUnavailableException {
    int partitionCount = 10;
    int requestPerPartition = 100;
    int totalHostCount = 100;
    HashRingProvider ringProvider = createStaticHashRingProvider(totalHostCount, partitionCount, getHashFunction(false));
    PartitionInfoProvider infoProvider = createRangeBasedPartitionInfoProvider(partitionCount);
    URIMapper mapper = new RingBasedUriMapper(ringProvider, infoProvider);
    List<URIKeyPair<Integer>> requests = testUtil.generateRequests(partitionCount, requestPerPartition);
    URIMappingResult<Integer> results = mapper.mapUris(requests);
    Map<URI, Set<Integer>> mapping = results.getMappedKeys();
    Map<URI, Integer> hostToPartitionId = results.getHostPartitionInfo();
    // No unmapped keys
    Assert.assertTrue(results.getUnmappedKeys().isEmpty());
    // Without sticky routing, one host should be returned for each partition
    Assert.assertEquals(10, mapping.size());
    Assert.assertEquals(10, hostToPartitionId.size());
    for (Map.Entry<URI, Integer> entry : hostToPartitionId.entrySet()) {
        // partition ids are correctly assigned for each URI
        Assert.assertTrue(entry.getKey().toString().contains(String.valueOf(entry.getValue())));
    }
    Set<Integer> mappedKeys = mapping.values().stream().reduce(new HashSet<>(), (e1, e2) -> {
        e1.addAll(e2);
        return e1;
    });
    int mappedKeyCount = mapping.values().stream().map(Set::size).reduce(Integer::sum).get();
    // Collective exhaustiveness and mutual exclusiveness
    Assert.assertEquals(partitionCount * requestPerPartition, mappedKeys.size());
    Assert.assertEquals(partitionCount * requestPerPartition, mappedKeyCount);
}
Also used : Set(java.util.Set) HashSet(java.util.HashSet) PartitionInfoProvider(com.linkedin.d2.balancer.util.partitions.PartitionInfoProvider) URI(java.net.URI) URIKeyPair(com.linkedin.d2.balancer.util.URIKeyPair) URIMapper(com.linkedin.d2.balancer.URIMapper) HashMap(java.util.HashMap) Map(java.util.Map) Test(org.testng.annotations.Test)

Example 10 with RingBasedUriMapper

use of com.linkedin.d2.balancer.util.hashing.RingBasedUriMapper in project rest.li by linkedin.

the class RingBasedURIMapperTest method testUniversalStickiness.

@Test
public void testUniversalStickiness() throws ServiceUnavailableException, URISyntaxException {
    int partitionCount = 4;
    int totalHostCount = 200;
    HashRingProvider ringProvider = createStaticHashRingProvider(totalHostCount, partitionCount, getHashFunction(true));
    HashFunction<Request> hashFunction = ringProvider.getRequestHashFunction(TEST_SERVICE);
    PartitionInfoProvider infoProvider = createRangeBasedPartitionInfoProvider(partitionCount);
    URIMapper mapper = new RingBasedUriMapper(ringProvider, infoProvider);
    // no partition, will be unmapped
    URIKeyPair<Integer> request1 = new URIKeyPair<>(1, new URI("d2://testService/1"));
    // partition 0
    URIKeyPair<Integer> request2 = new URIKeyPair<>(2, new URI("d2://testService/2?partition=0"));
    // partition 1
    URIKeyPair<Integer> request3 = new URIKeyPair<>(3, new URI("d2://testService/3?partition=1"));
    // partition 2
    URIKeyPair<Integer> request4 = new URIKeyPair<>(4, new URI("d2://testService/4?partition=2"));
    // partition 3
    URIKeyPair<Integer> request5 = new URIKeyPair<>(5, new URI("d2://testService/5?partition=3"));
    URIKeyPair<Integer> request6 = // partition 0 with different sticky key
    new URIKeyPair<>(6, new URI("d2://testService/6?partition=0"));
    URIKeyPair<Integer> request7 = // partition 1 with different sticky key
    new URIKeyPair<>(7, new URI("d2://testService/7?partition=1"));
    URIKeyPair<Integer> request8 = // partition 2 with different sticky key
    new URIKeyPair<>(8, new URI("d2://testService/8?partition=2"));
    URIKeyPair<Integer> request9 = // partition 3 with different sticky key
    new URIKeyPair<>(9, new URI("d2://testService/9?partition=3"));
    URIKeyPair<Integer> request10 = // with extra parameters
    new URIKeyPair<>(10, new URI("d2://testService/10?partition=0&uuid=1"));
    List<URIKeyPair<Integer>> requests = Arrays.asList(request1, request2, request3, request4, request5, request6, request7, request8, request9, request10);
    // uriMapper mapping
    URIMappingResult<Integer> uriMapperResult = mapper.mapUris(requests);
    // normal mapping
    Map<Integer, Set<Integer>> normalUnmapped = new HashMap<>();
    Map<URI, Set<Integer>> normalHostToKeySet = new HashMap<>();
    for (URIKeyPair<Integer> request : requests) {
        int partitionId = 0;
        try {
            partitionId = infoProvider.getPartitionAccessor(TEST_SERVICE).getPartitionId(request.getRequestUri());
        } catch (PartitionAccessException e) {
            normalUnmapped.computeIfAbsent(-1, k -> new HashSet<>()).add(request.getKey());
        }
        Ring<URI> ring = ringProvider.getRings(request.getRequestUri()).get(partitionId);
        URI uri = ring.get(hashFunction.hash(new URIRequest(request.getRequestUri())));
        normalHostToKeySet.computeIfAbsent(uri, k -> new HashSet<>());
        normalHostToKeySet.get(uri).add(request.getKey());
    }
    // they should have the same results
    Assert.assertEquals(uriMapperResult.getUnmappedKeys(), normalUnmapped);
    for (Map.Entry<URI, Set<Integer>> resolvedKeys : uriMapperResult.getMappedKeys().entrySet()) {
        Set<Integer> uriMapperKeySet = resolvedKeys.getValue();
        Assert.assertTrue(normalHostToKeySet.containsKey(resolvedKeys.getKey()));
        Set<Integer> normalKeySet = normalHostToKeySet.get(resolvedKeys.getKey());
        Assert.assertEquals(uriMapperKeySet, normalKeySet);
    }
}
Also used : Set(java.util.Set) HashSet(java.util.HashSet) PartitionInfoProvider(com.linkedin.d2.balancer.util.partitions.PartitionInfoProvider) HashMap(java.util.HashMap) URIRequest(com.linkedin.d2.balancer.util.URIRequest) Request(com.linkedin.r2.message.Request) URIRequest(com.linkedin.d2.balancer.util.URIRequest) URI(java.net.URI) URIKeyPair(com.linkedin.d2.balancer.util.URIKeyPair) URIMapper(com.linkedin.d2.balancer.URIMapper) PartitionAccessException(com.linkedin.d2.balancer.util.partitions.PartitionAccessException) HashMap(java.util.HashMap) Map(java.util.Map) Test(org.testng.annotations.Test)

Aggregations

PartitionInfoProvider (com.linkedin.d2.balancer.util.partitions.PartitionInfoProvider)11 URIMapper (com.linkedin.d2.balancer.URIMapper)10 Test (org.testng.annotations.Test)10 URIKeyPair (com.linkedin.d2.balancer.util.URIKeyPair)9 URI (java.net.URI)9 HashSet (java.util.HashSet)7 Set (java.util.Set)7 URIRequest (com.linkedin.d2.balancer.util.URIRequest)3 Request (com.linkedin.r2.message.Request)3 HashMap (java.util.HashMap)3 Map (java.util.Map)3 PartitionAccessException (com.linkedin.d2.balancer.util.partitions.PartitionAccessException)2 ArrayList (java.util.ArrayList)2 ServiceUnavailableException (com.linkedin.d2.balancer.ServiceUnavailableException)1 URIMappingResult (com.linkedin.d2.balancer.util.URIMappingResult)1 HashRingProvider (com.linkedin.d2.balancer.util.hashing.HashRingProvider)1 RingBasedUriMapper (com.linkedin.d2.balancer.util.hashing.RingBasedUriMapper)1 URIMapperTestUtil (com.linkedin.d2.balancer.util.hashing.URIMapperTestUtil)1 URIMapperTestUtil.createHashBasedPartitionInfoProvider (com.linkedin.d2.balancer.util.hashing.URIMapperTestUtil.createHashBasedPartitionInfoProvider)1 URIMapperTestUtil.createStaticHashRingProvider (com.linkedin.d2.balancer.util.hashing.URIMapperTestUtil.createStaticHashRingProvider)1