Examples with PartitionAccessor com.linkedin.d2.balancer.util.partitions.PartitionAccessor used on opensource projects

Example 6 with PartitionAccessor

use of com.linkedin.d2.balancer.util.partitions.PartitionAccessor in project rest.li by linkedin.

the class SimpleLoadBalancerTest method testGetAllPartitionMultipleHostsOrdering.

/**
   * This tests the getPartitionInfo() when keys are null (actually a test for KeyMapper.getAllPartitionMultipleHosts()).
   */
@Test
public void testGetAllPartitionMultipleHostsOrdering() throws Exception {
    String serviceName = "articles";
    String clusterName = "cluster";
    String path = "path";
    String strategyName = "degrader";
    //setup partition
    Map<URI, Map<Integer, PartitionData>> partitionDescriptions = new HashMap<URI, Map<Integer, PartitionData>>();
    final URI server1 = new URI("http://foo1.com");
    Map<Integer, PartitionData> server1Data = new HashMap<Integer, PartitionData>();
    server1Data.put(1, new PartitionData(1.0));
    server1Data.put(2, new PartitionData(1.0));
    server1Data.put(3, new PartitionData(1.0));
    partitionDescriptions.put(server1, server1Data);
    final URI server2 = new URI("http://foo2.com");
    Map<Integer, PartitionData> server2Data = new HashMap<Integer, PartitionData>();
    server2Data.put(1, new PartitionData(1.0));
    server2Data.put(2, new PartitionData(1.0));
    //server2Data.put(3, new PartitionData(1.0));
    partitionDescriptions.put(server2, server2Data);
    final URI server3 = new URI("http://foo3.com");
    Map<Integer, PartitionData> server3Data = new HashMap<Integer, PartitionData>();
    server3Data.put(1, new PartitionData(1.0));
    server3Data.put(2, new PartitionData(1.0));
    //server3Data.put(3, new PartitionData(1.0));
    partitionDescriptions.put(server3, server3Data);
    //setup strategy which involves tweaking the hash ring to get partitionId -> URI host
    List<LoadBalancerState.SchemeStrategyPair> orderedStrategies = new ArrayList<LoadBalancerState.SchemeStrategyPair>();
    LoadBalancerStrategy strategy = new TestLoadBalancerStrategy(partitionDescriptions);
    orderedStrategies.add(new LoadBalancerState.SchemeStrategyPair("http", strategy));
    //setup the partition accessor which is used to get partitionId -> keys
    PartitionAccessor accessor = new TestPartitionAccessor();
    URI serviceURI = new URI("d2://" + serviceName);
    SimpleLoadBalancer balancer = new SimpleLoadBalancer(new PartitionedLoadBalancerTestState(clusterName, serviceName, path, strategyName, partitionDescriptions, orderedStrategies, accessor));
    HostToKeyMapper<URI> result = balancer.getPartitionInformation(serviceURI, null, 3, 123);
    Assert.assertEquals(result.getPartitionInfoMap().size(), 4);
    Assert.assertEquals(4, result.getPartitionCount());
    // partition 0 should be empty
    Assert.assertTrue(result.getPartitionInfoMap().get(0).getHosts().isEmpty());
    // partition 1 should have server1, server2 and server3.
    List<URI> ordering1 = result.getPartitionInfoMap().get(1).getHosts();
    List<URI> allServers = new ArrayList<>();
    allServers.add(server1);
    allServers.add(server2);
    allServers.add(server3);
    Assert.assertTrue(ordering1.size() == 3);
    Assert.assertTrue(ordering1.containsAll(allServers));
    // partition 2 should be the same as partition 1
    List<URI> ordering2 = result.getPartitionInfoMap().get(2).getHosts();
    Assert.assertEquals(ordering1, ordering2);
    // partition 3 should only contain server1
    List<URI> ordering3 = result.getPartitionInfoMap().get(3).getHosts();
    Assert.assertEquals(ordering3.get(0), server1);
    // partition 0 and partition 3 should not have enough hosts: lacking 3 and 2 respectively.
    Assert.assertTrue(result.getPartitionsWithoutEnoughHosts().containsKey(3));
    Assert.assertTrue(result.getPartitionsWithoutEnoughHosts().containsKey(0));
    Assert.assertEquals((int) result.getPartitionsWithoutEnoughHosts().get(3), 2);
    Assert.assertEquals((int) result.getPartitionsWithoutEnoughHosts().get(0), 3);
}

Example 7 with PartitionAccessor

use of com.linkedin.d2.balancer.util.partitions.PartitionAccessor in project rest.li by linkedin.

the class TestD2Config method testSingleClusterHashPartitions.

// preliminary test for partitioning cluster
@Test
public static void testSingleClusterHashPartitions() throws IOException, InterruptedException, URISyntaxException, Exception {
    @SuppressWarnings("serial") final Map<String, List<String>> clustersData = new HashMap<String, List<String>>() {

        {
            put("partitioned-cluster", Arrays.asList(new String[] { "partitioned-service-1", "partitioned-service-2" }));
        }
    };
    final Map<String, Object> partitionProperties = new HashMap<String, Object>();
    Map<String, Object> hashBased = new HashMap<String, Object>();
    hashBased.put("partitionKeyRegex", "\\bid\\b=(\\d+)");
    hashBased.put("partitionCount", "10");
    hashBased.put("hashAlgorithm", "modulo");
    hashBased.put("partitionType", "HASH");
    partitionProperties.put("partitionProperties", hashBased);
    D2ConfigTestUtil d2Conf = new D2ConfigTestUtil(clustersData, partitionProperties);
    assertEquals(d2Conf.runDiscovery(_zkHosts), 0);
    verifyPartitionProperties("partitioned-cluster", partitionProperties);
    final ClusterProperties clusterprops = getClusterProperties(_zkclient, "partitioned-cluster");
    final PartitionAccessor accessor = PartitionAccessorFactory.getPartitionAccessor(clusterprops.getPartitionProperties());
    assertEquals(0, accessor.getPartitionId(0 + ""));
    assertEquals(9, accessor.getPartitionId(99 + ""));
    assertEquals(6, accessor.getPartitionId(176 + ""));
    assertEquals(3, accessor.getPartitionId(833 + ""));
}

Example 8 with PartitionAccessor

use of com.linkedin.d2.balancer.util.partitions.PartitionAccessor in project rest.li by linkedin.

the class TestD2Config method testSingleClusterRangePartitions.

// preliminary test for partitioning cluster
@Test
public static void testSingleClusterRangePartitions() throws IOException, InterruptedException, URISyntaxException, Exception {
    @SuppressWarnings("serial") final Map<String, List<String>> clustersData = new HashMap<String, List<String>>() {

        {
            put("partitioned-cluster", Arrays.asList(new String[] { "partitioned-service-1", "partitioned-service-2" }));
        }
    };
    final Map<String, Object> partitionProperties = new HashMap<String, Object>();
    Map<String, Object> rangeBased = new HashMap<String, Object>();
    rangeBased.put("partitionKeyRegex", "\\bid\\b=(\\d+)");
    rangeBased.put("keyRangeStart", "0");
    rangeBased.put("partitionCount", "10");
    rangeBased.put("partitionSize", "100");
    rangeBased.put("partitionType", "RANGE");
    partitionProperties.put("partitionProperties", rangeBased);
    D2ConfigTestUtil d2Conf = new D2ConfigTestUtil(clustersData, partitionProperties);
    assertEquals(d2Conf.runDiscovery(_zkHosts), 0);
    verifyPartitionProperties("partitioned-cluster", partitionProperties);
    final ClusterProperties clusterprops = getClusterProperties(_zkclient, "partitioned-cluster");
    final PartitionAccessor accessor = PartitionAccessorFactory.getPartitionAccessor(clusterprops.getPartitionProperties());
    try {
        accessor.getPartitionId(-1 + "");
        fail("Exception expected");
    } catch (PartitionAccessException e) {
    }
    try {
        accessor.getPartitionId(1000 + "");
        fail("Exception expected");
    } catch (PartitionAccessException e) {
    }
    assertEquals(0, accessor.getPartitionId(0 + ""));
    assertEquals(0, accessor.getPartitionId(99 + ""));
    assertEquals(1, accessor.getPartitionId(176 + ""));
    assertEquals(8, accessor.getPartitionId(833 + ""));
    final String legalUri1 = "/profiles?field=position&id=100";
    final String legalUri2 = "/profiles?wid=99&id=176&randid=301";
    final String illegalUri1 = "/profiles?wid=99";
    final String illegalUri2 = "/profiles?id=1000000000000000000000000000000000000000000000111111111";
    try {
        accessor.getPartitionId(URI.create(illegalUri1));
        fail("Exception expected");
    } catch (PartitionAccessException e) {
    }
    try {
        accessor.getPartitionId(URI.create(illegalUri2));
        fail("Exception expected");
    } catch (PartitionAccessException e) {
    }
    assertEquals(1, accessor.getPartitionId(URI.create(legalUri1)));
    assertEquals(1, accessor.getPartitionId(URI.create(legalUri2)));
    // Start Echo server on cluster-1
    Map<Integer, Double> serverConfig1 = new HashMap<Integer, Double>();
    serverConfig1.put(0, 0.5d);
    serverConfig1.put(3, 0.5d);
    Map<Integer, Double> serverConfig2 = new HashMap<Integer, Double>();
    serverConfig2.put(0, 0.25d);
    serverConfig2.put(1, 0.5d);
    serverConfig2.put(2, 0.5d);
    final int echoServerPort1 = 2346;
    final int echoServerPort2 = 2347;
    _echoServerList.add(startEchoServer(echoServerPort1, "partitioned-cluster", serverConfig1));
    _echoServerList.add(startEchoServer(echoServerPort2, "partitioned-cluster", serverConfig2));
    Map<URI, Map<Integer, Double>> partitionWeights = new HashMap<URI, Map<Integer, Double>>();
    partitionWeights.put(URI.create("http://127.0.0.1:" + echoServerPort1 + "/partitioned-cluster"), serverConfig1);
    partitionWeights.put(URI.create("http://127.0.0.1:" + echoServerPort2 + "/partitioned-cluster"), serverConfig2);
    verifyPartitionedUriProperties("partitioned-cluster", partitionWeights);
}

Example 9 with PartitionAccessor

use of com.linkedin.d2.balancer.util.partitions.PartitionAccessor in project rest.li by linkedin.

the class ConsistentHashKeyMapperTest method getConsistentHashKeyMapper.

private ConsistentHashKeyMapper getConsistentHashKeyMapper(RingFactory<URI> ringFactory) throws URISyntaxException {
    String serviceName = "articles";
    String clusterName = "cluster";
    String path = "path";
    String strategyName = "degrader";
    //setup partition
    Map<URI, Map<Integer, PartitionData>> partitionDescriptions = new HashMap<URI, Map<Integer, PartitionData>>();
    final URI foo1 = new URI("http://foo1.com");
    Map<Integer, PartitionData> foo1Data = new HashMap<Integer, PartitionData>();
    foo1Data.put(0, new PartitionData(1.0));
    partitionDescriptions.put(foo1, foo1Data);
    final URI foo2 = new URI("http://foo2.com");
    Map<Integer, PartitionData> foo2Data = new HashMap<Integer, PartitionData>();
    foo2Data.put(3, new PartitionData(1.0));
    foo2Data.put(4, new PartitionData(1.0));
    partitionDescriptions.put(foo2, foo2Data);
    final URI foo3 = new URI("http://foo3.com");
    Map<Integer, PartitionData> foo3Data = new HashMap<Integer, PartitionData>();
    foo3Data.put(0, new PartitionData(1.0));
    partitionDescriptions.put(foo3, foo3Data);
    final URI foo4 = new URI("http://foo4.com");
    Map<Integer, PartitionData> foo4Data = new HashMap<Integer, PartitionData>();
    foo4Data.put(1, new PartitionData(1.0));
    partitionDescriptions.put(foo4, foo4Data);
    final URI foo5 = new URI("http://foo5.com");
    Map<Integer, PartitionData> foo5Data = new HashMap<Integer, PartitionData>();
    foo5Data.put(1, new PartitionData(1.0));
    partitionDescriptions.put(foo5, foo5Data);
    final URI foo6 = new URI("http://foo6.com");
    Map<Integer, PartitionData> foo6Data = new HashMap<Integer, PartitionData>();
    foo6Data.put(1, new PartitionData(1.0));
    partitionDescriptions.put(foo6, foo6Data);
    //setup strategy which involves tweaking the hash ring to get partitionId -> URI host
    List<LoadBalancerState.SchemeStrategyPair> orderedStrategies = new ArrayList<LoadBalancerState.SchemeStrategyPair>();
    LoadBalancerStrategy strategy = new TestLoadBalancerStrategy(partitionDescriptions, ringFactory);
    orderedStrategies.add(new LoadBalancerState.SchemeStrategyPair("http", strategy));
    //setup the partition accessor which is used to get partitionId -> keys
    PartitionAccessor accessor = new TestPartitionAccessor();
    URI serviceURI = new URI("d2://" + serviceName);
    SimpleLoadBalancer balancer = new SimpleLoadBalancer(new PartitionedLoadBalancerTestState(clusterName, serviceName, path, strategyName, partitionDescriptions, orderedStrategies, accessor));
    ConsistentHashKeyMapper mapper = new ConsistentHashKeyMapper(balancer, balancer);
    return mapper;
}

Example 10 with PartitionAccessor

use of com.linkedin.d2.balancer.util.partitions.PartitionAccessor in project rest.li by linkedin.

the class ConsistentHashKeyMapperTest method testMapKeysConcurrency.

@SuppressWarnings("rawtypes")
@Test
public void testMapKeysConcurrency() throws Exception {
    String serviceName = "articles";
    String clusterName = "cluster";
    String path = "path";
    String strategyName = "degrader";
    int numPartitions = 500;
    // setup partition
    Map<URI, Map<Integer, PartitionData>> partitionDescriptions = new HashMap<URI, Map<Integer, PartitionData>>();
    final URI foo1 = new URI("http://foo1.com");
    Map<Integer, PartitionData> foo1Data = new HashMap<Integer, PartitionData>();
    for (int i = 0; i < numPartitions; i++) {
        foo1Data.put(i, new PartitionData(1.0));
    }
    partitionDescriptions.put(foo1, foo1Data);
    DegraderLoadBalancerStrategyV3 strategy = new DegraderLoadBalancerStrategyV3(new DegraderLoadBalancerStrategyConfig(5000), serviceName, null);
    List<LoadBalancerState.SchemeStrategyPair> orderedStrategies = new ArrayList<LoadBalancerState.SchemeStrategyPair>();
    orderedStrategies.add(new LoadBalancerState.SchemeStrategyPair("http", strategy));
    PartitionAccessor accessor = new TestDeadlockPartitionAccessor(numPartitions);
    SimpleLoadBalancer balancer = new SimpleLoadBalancer(new PartitionedLoadBalancerTestState(clusterName, serviceName, path, strategyName, partitionDescriptions, orderedStrategies, accessor));
    ConsistentHashKeyMapper mapper = new ConsistentHashKeyMapper(balancer, balancer);
    CountDownLatch latch = new CountDownLatch(numPartitions);
    List<Runnable> runnables = createRunnables(numPartitions, mapper, serviceName, latch);
    final ExecutorService executor = Executors.newFixedThreadPool(numPartitions);
    List<Future> futures = new ArrayList<Future>();
    for (int i = 0; i < numPartitions; i++) {
        futures.add(executor.submit(runnables.get(i)));
    }
    for (Future future : futures) {
        future.get(30, TimeUnit.SECONDS);
    }
}