Examples with DiskBalancerCluster org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerCluster used on opensource projects

Example 16 with DiskBalancerCluster

use of org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerCluster in project hadoop by apache.

the class TestPlanner method testGreedyPlannerEqualizeData.

/**
   * In this test we pass 3 volumes with 30, 20 and 10 GB of data used. We
   * expect the planner to print out 20 GB on each volume.
   * <p/>
   * That is the plan should say move 10 GB from volume30 to volume10.
   */
@Test
public void testGreedyPlannerEqualizeData() throws Exception {
    NullConnector nullConnector = new NullConnector();
    DiskBalancerCluster cluster = new DiskBalancerCluster(nullConnector);
    DiskBalancerDataNode node = new DiskBalancerDataNode(UUID.randomUUID().toString());
    DiskBalancerVolume volume30 = createVolume("volume30", 100, 30);
    DiskBalancerVolume volume20 = createVolume("volume20", 100, 20);
    DiskBalancerVolume volume10 = createVolume("volume10", 100, 10);
    node.addVolume(volume10);
    node.addVolume(volume20);
    node.addVolume(volume30);
    nullConnector.addNode(node);
    cluster.readClusterInfo();
    Assert.assertEquals(1, cluster.getNodes().size());
    GreedyPlanner planner = new GreedyPlanner(5.0f, node);
    NodePlan plan = new NodePlan(node.getDataNodeUUID(), node.getDataNodePort());
    planner.balanceVolumeSet(node, node.getVolumeSets().get("SSD"), plan);
    // We should have only one planned move from
    // volume30 to volume10 of 10 GB Size.
    assertEquals(1, plan.getVolumeSetPlans().size());
    Step step = plan.getVolumeSetPlans().get(0);
    assertEquals("volume30", step.getSourceVolume().getPath());
    assertEquals("volume10", step.getDestinationVolume().getPath());
    assertEquals("10 G", step.getSizeString(step.getBytesToMove()));
}

Example 17 with DiskBalancerCluster

use of org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerCluster in project hadoop by apache.

the class TestPlanner method testGreedyPlannerBalanceVolumeSet.

@Test
public void testGreedyPlannerBalanceVolumeSet() throws Exception {
    URI clusterJson = getClass().getResource("/diskBalancer/data-cluster-3node-3disk.json").toURI();
    ClusterConnector jsonConnector = ConnectorFactory.getCluster(clusterJson, null);
    DiskBalancerCluster cluster = new DiskBalancerCluster(jsonConnector);
    cluster.readClusterInfo();
    Assert.assertEquals(3, cluster.getNodes().size());
    cluster.setNodesToProcess(cluster.getNodes());
    DiskBalancerDataNode node = cluster.getNodes().get(0);
    GreedyPlanner planner = new GreedyPlanner(10.0f, node);
    NodePlan plan = new NodePlan(node.getDataNodeName(), node.getDataNodePort());
    planner.balanceVolumeSet(node, node.getVolumeSets().get("SSD"), plan);
}

Example 18 with DiskBalancerCluster

use of org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerCluster in project hadoop by apache.

the class TestPlanner method testGreedyPlannerPlanWithDifferentDiskSizes.

@Test
public void testGreedyPlannerPlanWithDifferentDiskSizes() throws Exception {
    NullConnector nullConnector = new NullConnector();
    DiskBalancerCluster cluster = new DiskBalancerCluster(nullConnector);
    DiskBalancerDataNode node = new DiskBalancerDataNode(UUID.randomUUID().toString());
    DiskBalancerVolume volume1 = createVolume("volume100", 1000, 100);
    DiskBalancerVolume volume2 = createVolume("volume0-1", 500, 0);
    DiskBalancerVolume volume3 = createVolume("volume0-2", 250, 0);
    node.addVolume(volume1);
    node.addVolume(volume2);
    node.addVolume(volume3);
    nullConnector.addNode(node);
    cluster.readClusterInfo();
    Assert.assertEquals(1, cluster.getNodes().size());
    GreedyPlanner newPlanner = new GreedyPlanner(01.0f, node);
    NodePlan newPlan = new NodePlan(node.getDataNodeName(), node.getDataNodePort());
    newPlanner.balanceVolumeSet(node, node.getVolumeSets().get("SSD"), newPlan);
    assertEquals(2, newPlan.getVolumeSetPlans().size());
    for (Step step : newPlan.getVolumeSetPlans()) {
        if (step.getDestinationVolume().getPath().equals("volume0-1")) {
            assertEquals("volume100", step.getSourceVolume().getPath());
            assertEquals("28.5 G", step.getSizeString(step.getBytesToMove()));
        }
        if (step.getDestinationVolume().getPath().equals("volume0-2")) {
            assertEquals("volume100", step.getSourceVolume().getPath());
            assertEquals("14.3 G", step.getSizeString(step.getBytesToMove()));
        }
    }
    Step step = newPlan.getVolumeSetPlans().get(0);
    assertEquals(0.05714f, step.getIdealStorage(), 0.001f);
}

Example 19 with DiskBalancerCluster

use of org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerCluster in project hadoop by apache.

the class TestPlanner method testGreedyPlannerThresholdTest.

@Test
public void testGreedyPlannerThresholdTest() throws Exception {
    NullConnector nullConnector = new NullConnector();
    DiskBalancerCluster cluster = new DiskBalancerCluster(nullConnector);
    DiskBalancerDataNode node = new DiskBalancerDataNode(UUID.randomUUID().toString());
    DiskBalancerVolume volume1 = createVolume("volume100", 1000, 100);
    DiskBalancerVolume volume2 = createVolume("volume0-1", 300, 0);
    DiskBalancerVolume volume3 = createVolume("volume0-2", 300, 0);
    node.addVolume(volume1);
    node.addVolume(volume2);
    node.addVolume(volume3);
    nullConnector.addNode(node);
    cluster.readClusterInfo();
    Assert.assertEquals(1, cluster.getNodes().size());
    GreedyPlanner planner = new GreedyPlanner(10.0f, node);
    NodePlan plan = new NodePlan(node.getDataNodeName(), node.getDataNodePort());
    planner.balanceVolumeSet(node, node.getVolumeSets().get("SSD"), plan);
    //We should see NO moves since the total data on the volume100
    // is less than or equal to threashold value that we pass, which is 10%
    assertEquals(0, plan.getVolumeSetPlans().size());
    // for this new planner we are passing 1% as as threshold value
    // hence planner must move data if possible.
    GreedyPlanner newPlanner = new GreedyPlanner(01.0f, node);
    NodePlan newPlan = new NodePlan(node.getDataNodeName(), node.getDataNodePort());
    newPlanner.balanceVolumeSet(node, node.getVolumeSets().get("SSD"), newPlan);
    assertEquals(2, newPlan.getVolumeSetPlans().size());
    // Move size should say move 19 GB
    // Here is how the math works out.
    // TotalCapacity = 1000 + 300 + 300 = 1600 GB
    // TotolUsed = 100
    // Expected data% on each disk = 0.0625
    // On Disk (volume0-1) = 300 * 0.0625 - 18.75 -- We round it up
    // in the display string -- hence 18.8 GB, it will be same on volume 2 too.
    // since they are equal sized disks with same used capacity
    Step step = newPlan.getVolumeSetPlans().get(0);
    assertEquals("volume100", step.getSourceVolume().getPath());
    assertTrue(step.getSizeString(step.getBytesToMove()).matches("18.[6|7|8] G"));
    step = newPlan.getVolumeSetPlans().get(1);
    assertEquals("volume100", step.getSourceVolume().getPath());
    assertTrue(step.getSizeString(step.getBytesToMove()).matches("18.[6|7|8] G"));
}

Example 20 with DiskBalancerCluster

use of org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerCluster in project hadoop by apache.

the class TestPlanner method testGreedyPlannerNoVolumeTest.

@Test
public void testGreedyPlannerNoVolumeTest() throws Exception {
    NullConnector nullConnector = new NullConnector();
    DiskBalancerCluster cluster = new DiskBalancerCluster(nullConnector);
    List<NodePlan> planList = cluster.computePlan(10.0f);
    assertNotNull(planList);
}