Examples with HashFunction com.linkedin.d2.balancer.util.hashing.HashFunction used on opensource projects

Example 1 with HashFunction

use of com.linkedin.d2.balancer.util.hashing.HashFunction in project rest.li by linkedin.

the class RelativeLoadBalancerStrategyFactory method getRequestHashFunction.

private HashFunction<Request> getRequestHashFunction(D2RelativeStrategyProperties relativeStrategyProperties) {
    if (relativeStrategyProperties.hasRingProperties() && relativeStrategyProperties.getRingProperties().hasHashConfig()) {
        HashMethod hashMethod = relativeStrategyProperties.getRingProperties().getHashMethod();
        HashConfig hashConfig = relativeStrategyProperties.getRingProperties().getHashConfig();
        switch(hashMethod) {
            case URI_REGEX:
                return new URIRegexHash(RelativeStrategyPropertiesConverter.toHashConfigMap(hashConfig));
            case RANDOM:
            default:
                return new RandomHash();
        }
    }
    // Fall back to RandomHash if not specified
    return new RandomHash();
}

Example 2 with HashFunction

use of com.linkedin.d2.balancer.util.hashing.HashFunction in project rest.li by linkedin.

the class RingBasedUriMapper method distributeToHosts.

private <KEY> Map<URI, Set<KEY>> distributeToHosts(Map<Integer, List<URIKeyPair<KEY>>> requestsByParititonId, Map<Integer, Ring<URI>> rings, HashFunction<Request> hashFunction, Map<URI, Integer> hostToPartitionId, Map<Integer, Set<KEY>> unmapped) {
    if (hashFunction instanceof RandomHash) {
        return distributeToHostNonSticky(requestsByParititonId, rings, hostToPartitionId, unmapped);
    }
    Map<URI, Set<KEY>> hostToKeySet = new HashMap<>();
    for (Map.Entry<Integer, List<URIKeyPair<KEY>>> entry : requestsByParititonId.entrySet()) {
        int partitionId = entry.getKey();
        for (URIKeyPair<KEY> request : entry.getValue()) {
            int hashcode = hashFunction.hash(new URIRequest(request.getRequestUri()));
            URI resolvedHost = rings.get(partitionId).get(hashcode);
            if (resolvedHost == null) {
                // under custom use case, key will be null, in which case we will just return a map from partition id to empty set
                // Users should be able to understand what partitions do not have available hosts by examining the keys in "unmapped"
                Set<KEY> unmappedKeys = convertURIKeyPairListToKeySet(entry.getValue());
                unmapped.computeIfAbsent(entry.getKey(), k -> new HashSet<>()).addAll(unmappedKeys);
                break;
            } else {
                // under custom use case, key will be null, in which case we will just return a map from uri to empty set
                hostToPartitionId.putIfAbsent(resolvedHost, entry.getKey());
                Set<KEY> newSet = hostToKeySet.computeIfAbsent(resolvedHost, host -> new HashSet<>());
                if (request.getKey() != null) {
                    newSet.add(request.getKey());
                }
            }
        }
    }
    return hostToKeySet;
}

Example 3 with HashFunction

use of com.linkedin.d2.balancer.util.hashing.HashFunction in project rest.li by linkedin.

the class RingBasedUriMapper method mapUris.

/**
 * To achieve scatter-gather, there will be two passes.
 *
 * Pass 1: All requests are assigned a partitionId based on partition properties
 *         If partitioning is not enabled, all requests will have default partitionId of 0;
 *
 * Pass 2: All requests in the same partition will be routed based on the ring of that partition.
 *         If sticky routing is not specified, ONE host on the ring of that partition will be assigned for all hosts assigned to that partition.
 *
 * Unmapped key in either step will be collected in the unmapped keySet in the result.
 *
 * @param <KEY> type of provided key
 * @param requestUriKeyPairs a list of URIKeyPair, each contains a d2 request uri and a unique resource key.
 * @return {@link URIMappingResult} that contains host to keySet mapping as well as unmapped keys.
 * @throws ServiceUnavailableException when the requested service is not available
 */
@Override
public <KEY> URIMappingResult<KEY> mapUris(List<URIKeyPair<KEY>> requestUriKeyPairs) throws ServiceUnavailableException {
    if (requestUriKeyPairs == null || requestUriKeyPairs.isEmpty()) {
        return new URIMappingResult<>(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    }
    // API assumes that all requests will be made to the same service, just use the first request to get the service name and act as sample uri
    URI sampleURI = requestUriKeyPairs.get(0).getRequestUri();
    String serviceName = LoadBalancerUtil.getServiceNameFromUri(sampleURI);
    // To achieve scatter-gather, we require the following information
    PartitionAccessor accessor = _partitionInfoProvider.getPartitionAccessor(serviceName);
    Map<Integer, Ring<URI>> rings = _hashRingProvider.getRings(sampleURI);
    HashFunction<Request> hashFunction = _hashRingProvider.getRequestHashFunction(serviceName);
    Map<Integer, Set<KEY>> unmapped = new HashMap<>();
    // Pass One
    Map<Integer, List<URIKeyPair<KEY>>> requestsByPartition = distributeToPartitions(requestUriKeyPairs, accessor, unmapped);
    // Pass Two
    Map<URI, Integer> hostToParitionId = new HashMap<>();
    Map<URI, Set<KEY>> hostToKeySet = distributeToHosts(requestsByPartition, rings, hashFunction, hostToParitionId, unmapped);
    return new URIMappingResult<>(hostToKeySet, unmapped, hostToParitionId);
}

Example 4 with HashFunction

use of com.linkedin.d2.balancer.util.hashing.HashFunction in project rest.li by linkedin.

the class RingBasedURIMapperTest method testMapUrisStickyRoutingOnly.

@Test
public void testMapUrisStickyRoutingOnly() throws ServiceUnavailableException, PartitionAccessException {
    int partitionCount = 1;
    int requestPerPartition = 1000;
    int totalHostCount = 100;
    HashRingProvider ringProvider = createStaticHashRingProvider(totalHostCount, partitionCount, getHashFunction(true));
    PartitionInfoProvider infoProvider = createRangeBasedPartitionInfoProvider(partitionCount);
    URIMapper mapper = new RingBasedUriMapper(ringProvider, infoProvider);
    List<URIKeyPair<Integer>> requests = testUtil.generateRequests(partitionCount, requestPerPartition);
    URIMappingResult<Integer> results1 = mapper.mapUris(requests);
    URIMappingResult<Integer> results2 = mapper.mapUris(requests);
    // Sticky routing between two runs
    Assert.assertEquals(results1.getMappedKeys(), results2.getMappedKeys());
    Assert.assertEquals(results1.getUnmappedKeys(), results2.getUnmappedKeys());
    Map<URI, Set<Integer>> mapping = results1.getMappedKeys();
    // Testing universal stickiness, take out 50 requests randomly and make sure they would be resolved to the same host as does URIMapper
    Collections.shuffle(requests);
    HashFunction<Request> hashFunction = ringProvider.getRequestHashFunction(TEST_SERVICE);
    for (int i = 0; i < 50; i++) {
        URIKeyPair<Integer> request = requests.get(i);
        int partitionId = infoProvider.getPartitionAccessor(TEST_SERVICE).getPartitionId(request.getRequestUri());
        Ring<URI> ring = ringProvider.getRings(request.getRequestUri()).get(partitionId);
        URI uri = ring.get(hashFunction.hash(new URIRequest(request.getRequestUri())));
        Assert.assertTrue(mapping.keySet().contains(uri));
    }
    // Only one partition
    Assert.assertEquals(1, new HashSet<>(results1.getHostPartitionInfo().values()).size());
    Assert.assertEquals(1, new HashSet<>(results2.getHostPartitionInfo().values()).size());
}

Example 5 with HashFunction

use of com.linkedin.d2.balancer.util.hashing.HashFunction in project rest.li by linkedin.

the class RingBasedURIMapperTest method testUniversalStickiness.

@Test
public void testUniversalStickiness() throws ServiceUnavailableException, URISyntaxException {
    int partitionCount = 4;
    int totalHostCount = 200;
    HashRingProvider ringProvider = createStaticHashRingProvider(totalHostCount, partitionCount, getHashFunction(true));
    HashFunction<Request> hashFunction = ringProvider.getRequestHashFunction(TEST_SERVICE);
    PartitionInfoProvider infoProvider = createRangeBasedPartitionInfoProvider(partitionCount);
    URIMapper mapper = new RingBasedUriMapper(ringProvider, infoProvider);
    // no partition, will be unmapped
    URIKeyPair<Integer> request1 = new URIKeyPair<>(1, new URI("d2://testService/1"));
    // partition 0
    URIKeyPair<Integer> request2 = new URIKeyPair<>(2, new URI("d2://testService/2?partition=0"));
    // partition 1
    URIKeyPair<Integer> request3 = new URIKeyPair<>(3, new URI("d2://testService/3?partition=1"));
    // partition 2
    URIKeyPair<Integer> request4 = new URIKeyPair<>(4, new URI("d2://testService/4?partition=2"));
    // partition 3
    URIKeyPair<Integer> request5 = new URIKeyPair<>(5, new URI("d2://testService/5?partition=3"));
    URIKeyPair<Integer> request6 = // partition 0 with different sticky key
    new URIKeyPair<>(6, new URI("d2://testService/6?partition=0"));
    URIKeyPair<Integer> request7 = // partition 1 with different sticky key
    new URIKeyPair<>(7, new URI("d2://testService/7?partition=1"));
    URIKeyPair<Integer> request8 = // partition 2 with different sticky key
    new URIKeyPair<>(8, new URI("d2://testService/8?partition=2"));
    URIKeyPair<Integer> request9 = // partition 3 with different sticky key
    new URIKeyPair<>(9, new URI("d2://testService/9?partition=3"));
    URIKeyPair<Integer> request10 = // with extra parameters
    new URIKeyPair<>(10, new URI("d2://testService/10?partition=0&uuid=1"));
    List<URIKeyPair<Integer>> requests = Arrays.asList(request1, request2, request3, request4, request5, request6, request7, request8, request9, request10);
    // uriMapper mapping
    URIMappingResult<Integer> uriMapperResult = mapper.mapUris(requests);
    // normal mapping
    Map<Integer, Set<Integer>> normalUnmapped = new HashMap<>();
    Map<URI, Set<Integer>> normalHostToKeySet = new HashMap<>();
    for (URIKeyPair<Integer> request : requests) {
        int partitionId = 0;
        try {
            partitionId = infoProvider.getPartitionAccessor(TEST_SERVICE).getPartitionId(request.getRequestUri());
        } catch (PartitionAccessException e) {
            normalUnmapped.computeIfAbsent(-1, k -> new HashSet<>()).add(request.getKey());
        }
        Ring<URI> ring = ringProvider.getRings(request.getRequestUri()).get(partitionId);
        URI uri = ring.get(hashFunction.hash(new URIRequest(request.getRequestUri())));
        normalHostToKeySet.computeIfAbsent(uri, k -> new HashSet<>());
        normalHostToKeySet.get(uri).add(request.getKey());
    }
    // they should have the same results
    Assert.assertEquals(uriMapperResult.getUnmappedKeys(), normalUnmapped);
    for (Map.Entry<URI, Set<Integer>> resolvedKeys : uriMapperResult.getMappedKeys().entrySet()) {
        Set<Integer> uriMapperKeySet = resolvedKeys.getValue();
        Assert.assertTrue(normalHostToKeySet.containsKey(resolvedKeys.getKey()));
        Set<Integer> normalKeySet = normalHostToKeySet.get(resolvedKeys.getKey());
        Assert.assertEquals(uriMapperKeySet, normalKeySet);
    }
}