Examples with Cluster org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster used on opensource projects

Example 1 with Cluster

use of org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster in project hbase by apache.

the class TestBaseLoadBalancer method testRegionAvailabilityWithRegionMoves.

@Test(timeout = 180000)
public void testRegionAvailabilityWithRegionMoves() throws Exception {
    List<HRegionInfo> list0 = new ArrayList<>();
    List<HRegionInfo> list1 = new ArrayList<>();
    List<HRegionInfo> list2 = new ArrayList<>();
    // create a region (region1)
    HRegionInfo hri1 = new HRegionInfo(TableName.valueOf(name.getMethodName()), "key1".getBytes(), "key2".getBytes(), false, 100);
    // create a replica of the region (replica_of_region1)
    HRegionInfo hri2 = RegionReplicaUtil.getRegionInfoForReplica(hri1, 1);
    // create a second region (region2)
    HRegionInfo hri3 = new HRegionInfo(TableName.valueOf(name.getMethodName()), "key2".getBytes(), "key3".getBytes(), false, 101);
    //only region1
    list0.add(hri1);
    //only replica_of_region1
    list1.add(hri2);
    //only region2
    list2.add(hri3);
    Map<ServerName, List<HRegionInfo>> clusterState = new LinkedHashMap<>();
    //servers[0] hosts region1
    clusterState.put(servers[0], list0);
    //servers[1] hosts replica_of_region1
    clusterState.put(servers[1], list1);
    //servers[2] hosts region2
    clusterState.put(servers[2], list2);
    // create a cluster with the above clusterState. The way in which the
    // cluster is created (constructor code) would make sure the indices of
    // the servers are in the order in which it is inserted in the clusterState
    // map (linkedhashmap is important).
    Cluster cluster = new Cluster(clusterState, null, null, rackManager);
    // check whether moving region1 from servers[1] to servers[2] would lower availability
    assertTrue(!cluster.wouldLowerAvailability(hri1, servers[2]));
    // now move region1 from servers[0] to servers[2]
    cluster.doAction(new MoveRegionAction(0, 0, 2));
    // check that the numMaxRegionsPerTable for "table" has increased to 2
    assertEquals(2, cluster.numMaxRegionsPerTable[0]);
    // now repeat check whether moving region1 from servers[1] to servers[2]
    // would lower availability
    assertTrue(cluster.wouldLowerAvailability(hri1, servers[2]));
    // start over again
    clusterState.clear();
    List<HRegionInfo> list3 = new ArrayList<>();
    HRegionInfo hri4 = RegionReplicaUtil.getRegionInfoForReplica(hri3, 1);
    list3.add(hri4);
    //servers[0], rack1 hosts region1
    clusterState.put(servers[0], list0);
    //servers[5], rack2 hosts replica_of_region1
    clusterState.put(servers[5], list1);
    //servers[6], rack2 hosts region2
    clusterState.put(servers[6], list2);
    //servers[12], rack3 hosts replica_of_region2
    clusterState.put(servers[12], list3);
    // create a cluster with the above clusterState
    cluster = new Cluster(clusterState, null, null, rackManager);
    // check whether a move of replica_of_region2 from servers[12],rack3 to servers[0],rack1 would
    // lower the availability
    assertTrue(!cluster.wouldLowerAvailability(hri4, servers[0]));
    // now move region2 from servers[6],rack2 to servers[0],rack1
    cluster.doAction(new MoveRegionAction(2, 2, 0));
    // now repeat check if replica_of_region2 from servers[12],rack3 to servers[0],rack1 would
    // lower the availability
    assertTrue(cluster.wouldLowerAvailability(hri3, servers[0]));
}

Example 2 with Cluster

use of org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster in project hbase by apache.

the class TestBaseLoadBalancer method testClusterRegionLocations.

@Test(timeout = 180000)
public void testClusterRegionLocations() {
    // tests whether region locations are handled correctly in Cluster
    List<ServerName> servers = getListOfServerNames(randomServers(10, 10));
    List<HRegionInfo> regions = randomRegions(101);
    Map<ServerName, List<HRegionInfo>> clusterState = new HashMap<>();
    assignRegions(regions, servers, clusterState);
    // mock block locality for some regions
    RegionLocationFinder locationFinder = mock(RegionLocationFinder.class);
    // block locality: region:0   => {server:0}
    //                 region:1   => {server:0, server:1}
    //                 region:42 => {server:4, server:9, server:5}
    when(locationFinder.getTopBlockLocations(regions.get(0))).thenReturn(Lists.newArrayList(servers.get(0)));
    when(locationFinder.getTopBlockLocations(regions.get(1))).thenReturn(Lists.newArrayList(servers.get(0), servers.get(1)));
    when(locationFinder.getTopBlockLocations(regions.get(42))).thenReturn(Lists.newArrayList(servers.get(4), servers.get(9), servers.get(5)));
    when(locationFinder.getTopBlockLocations(regions.get(43))).thenReturn(// this server does not exists in clusterStatus
    Lists.newArrayList(ServerName.valueOf("foo", 0, 0)));
    BaseLoadBalancer.Cluster cluster = new Cluster(clusterState, null, locationFinder, null);
    // this is ok, it is just a test
    int r0 = ArrayUtils.indexOf(cluster.regions, regions.get(0));
    int r1 = ArrayUtils.indexOf(cluster.regions, regions.get(1));
    int r10 = ArrayUtils.indexOf(cluster.regions, regions.get(10));
    int r42 = ArrayUtils.indexOf(cluster.regions, regions.get(42));
    int r43 = ArrayUtils.indexOf(cluster.regions, regions.get(43));
    int s0 = cluster.serversToIndex.get(servers.get(0).getHostAndPort());
    int s1 = cluster.serversToIndex.get(servers.get(1).getHostAndPort());
    int s4 = cluster.serversToIndex.get(servers.get(4).getHostAndPort());
    int s5 = cluster.serversToIndex.get(servers.get(5).getHostAndPort());
    int s9 = cluster.serversToIndex.get(servers.get(9).getHostAndPort());
    // region 0 locations
    assertEquals(1, cluster.regionLocations[r0].length);
    assertEquals(s0, cluster.regionLocations[r0][0]);
    // region 1 locations
    assertEquals(2, cluster.regionLocations[r1].length);
    assertEquals(s0, cluster.regionLocations[r1][0]);
    assertEquals(s1, cluster.regionLocations[r1][1]);
    // region 10 locations
    assertEquals(0, cluster.regionLocations[r10].length);
    // region 42 locations
    assertEquals(3, cluster.regionLocations[r42].length);
    assertEquals(s4, cluster.regionLocations[r42][0]);
    assertEquals(s9, cluster.regionLocations[r42][1]);
    assertEquals(s5, cluster.regionLocations[r42][2]);
    // region 43 locations
    assertEquals(1, cluster.regionLocations[r43].length);
    assertEquals(-1, cluster.regionLocations[r43][0]);
}

Example 3 with Cluster

use of org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster in project hbase by apache.

the class TestBaseLoadBalancer method testRegionAvailability.

@Test(timeout = 180000)
public void testRegionAvailability() throws Exception {
    // Create a cluster with a few servers, assign them to specific racks
    // then assign some regions. The tests should check whether moving a
    // replica from one node to a specific other node or rack lowers the
    // availability of the region or not
    List<HRegionInfo> list0 = new ArrayList<>();
    List<HRegionInfo> list1 = new ArrayList<>();
    List<HRegionInfo> list2 = new ArrayList<>();
    // create a region (region1)
    HRegionInfo hri1 = new HRegionInfo(TableName.valueOf(name.getMethodName()), "key1".getBytes(), "key2".getBytes(), false, 100);
    // create a replica of the region (replica_of_region1)
    HRegionInfo hri2 = RegionReplicaUtil.getRegionInfoForReplica(hri1, 1);
    // create a second region (region2)
    HRegionInfo hri3 = new HRegionInfo(TableName.valueOf(name.getMethodName()), "key2".getBytes(), "key3".getBytes(), false, 101);
    //only region1
    list0.add(hri1);
    //only replica_of_region1
    list1.add(hri2);
    //only region2
    list2.add(hri3);
    Map<ServerName, List<HRegionInfo>> clusterState = new LinkedHashMap<>();
    //servers[0] hosts region1
    clusterState.put(servers[0], list0);
    //servers[1] hosts replica_of_region1
    clusterState.put(servers[1], list1);
    //servers[2] hosts region2
    clusterState.put(servers[2], list2);
    // create a cluster with the above clusterState. The way in which the
    // cluster is created (constructor code) would make sure the indices of
    // the servers are in the order in which it is inserted in the clusterState
    // map (linkedhashmap is important). A similar thing applies to the region lists
    Cluster cluster = new Cluster(clusterState, null, null, rackManager);
    // check whether a move of region1 from servers[0] to servers[1] would lower
    // the availability of region1
    assertTrue(cluster.wouldLowerAvailability(hri1, servers[1]));
    // check whether a move of region1 from servers[0] to servers[2] would lower
    // the availability of region1
    assertTrue(!cluster.wouldLowerAvailability(hri1, servers[2]));
    // check whether a move of replica_of_region1 from servers[0] to servers[2] would lower
    // the availability of replica_of_region1
    assertTrue(!cluster.wouldLowerAvailability(hri2, servers[2]));
    // check whether a move of region2 from servers[0] to servers[1] would lower
    // the availability of region2
    assertTrue(!cluster.wouldLowerAvailability(hri3, servers[1]));
    // now lets have servers[1] host replica_of_region2
    list1.add(RegionReplicaUtil.getRegionInfoForReplica(hri3, 1));
    // create a new clusterState with the above change
    cluster = new Cluster(clusterState, null, null, rackManager);
    // now check whether a move of a replica from servers[0] to servers[1] would lower
    // the availability of region2
    assertTrue(cluster.wouldLowerAvailability(hri3, servers[1]));
    // start over again
    clusterState.clear();
    //servers[0], rack1 hosts region1
    clusterState.put(servers[0], list0);
    //servers[5], rack2 hosts replica_of_region1 and replica_of_region2
    clusterState.put(servers[5], list1);
    //servers[6], rack2 hosts region2
    clusterState.put(servers[6], list2);
    //servers[10], rack3 hosts no region
    clusterState.put(servers[10], new ArrayList<>());
    // create a cluster with the above clusterState
    cluster = new Cluster(clusterState, null, null, rackManager);
    // check whether a move of region1 from servers[0],rack1 to servers[6],rack2 would
    // lower the availability
    assertTrue(cluster.wouldLowerAvailability(hri1, servers[0]));
    // now create a cluster without the rack manager
    cluster = new Cluster(clusterState, null, null, null);
    // now repeat check whether a move of region1 from servers[0] to servers[6] would
    // lower the availability
    assertTrue(!cluster.wouldLowerAvailability(hri1, servers[6]));
}

Example 4 with Cluster

use of org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster in project hbase by apache.

the class TestStochasticLoadBalancer method testNeedsBalanceForColocatedReplicas.

@Test
public void testNeedsBalanceForColocatedReplicas() {
    // check for the case where there are two hosts and with one rack, and where
    // both the replicas are hosted on the same server
    List<HRegionInfo> regions = randomRegions(1);
    ServerName s1 = ServerName.valueOf("host1", 1000, 11111);
    ServerName s2 = ServerName.valueOf("host11", 1000, 11111);
    Map<ServerName, List<HRegionInfo>> map = new HashMap<>();
    map.put(s1, regions);
    regions.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(0), 1));
    // until the step above s1 holds two replicas of a region
    regions = randomRegions(1);
    map.put(s2, regions);
    assertTrue(loadBalancer.needsBalance(new Cluster(map, null, null, null)));
    // check for the case where there are two hosts on the same rack and there are two racks
    // and both the replicas are on the same rack
    map.clear();
    regions = randomRegions(1);
    List<HRegionInfo> regionsOnS2 = new ArrayList<>(1);
    regionsOnS2.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(0), 1));
    map.put(s1, regions);
    map.put(s2, regionsOnS2);
    // add another server so that the cluster has some host on another rack
    map.put(ServerName.valueOf("host2", 1000, 11111), randomRegions(1));
    assertTrue(loadBalancer.needsBalance(new Cluster(map, null, null, new ForTestRackManagerOne())));
}

Example 5 with Cluster

use of org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster in project hbase by apache.

the class TestStochasticLoadBalancer method testTableSkewCost.

@Test
public void testTableSkewCost() {
    Configuration conf = HBaseConfiguration.create();
    StochasticLoadBalancer.CostFunction costFunction = new StochasticLoadBalancer.TableSkewCostFunction(conf);
    for (int[] mockCluster : clusterStateMocks) {
        BaseLoadBalancer.Cluster cluster = mockCluster(mockCluster);
        costFunction.init(cluster);
        double cost = costFunction.cost();
        assertTrue(cost >= 0);
        assertTrue(cost <= 1.01);
    }
}