Examples with SCMContainerPlacementCapacity org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity used on opensource projects

Example 1 with SCMContainerPlacementCapacity

use of org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity in project ozone by apache.

the class TestContainerPlacement method testCapacityPlacementYieldsBetterDataDistribution.

/**
 * This test simulates lots of Cluster I/O and updates the metadata in SCM.
 * We simulate adding and removing containers from the cluster. It asserts
 * that our placement algorithm has taken the capacity of nodes into
 * consideration by asserting that standard deviation of used space on these
 * has improved.
 */
@Test
public void testCapacityPlacementYieldsBetterDataDistribution() throws SCMException {
    final int opsCount = 200 * 1000;
    final int nodesRequired = 3;
    Random random = new Random();
    // The nature of init code in MockNodeManager yields similar clusters.
    MockNodeManager nodeManagerCapacity = new MockNodeManager(true, 100);
    MockNodeManager nodeManagerRandom = new MockNodeManager(true, 100);
    DescriptiveStatistics beforeCapacity = computeStatistics(nodeManagerCapacity);
    DescriptiveStatistics beforeRandom = computeStatistics(nodeManagerRandom);
    // Assert that our initial layout of clusters are similar.
    assertEquals(beforeCapacity.getStandardDeviation(), beforeRandom.getStandardDeviation(), 0.001);
    SCMContainerPlacementCapacity capacityPlacer = new SCMContainerPlacementCapacity(nodeManagerCapacity, new OzoneConfiguration(), null, true, null);
    SCMContainerPlacementRandom randomPlacer = new SCMContainerPlacementRandom(nodeManagerRandom, new OzoneConfiguration(), null, true, null);
    for (int x = 0; x < opsCount; x++) {
        long containerSize = random.nextInt(10) * OzoneConsts.GB;
        long metadataSize = random.nextInt(10) * OzoneConsts.GB;
        List<DatanodeDetails> nodesCapacity = capacityPlacer.chooseDatanodes(new ArrayList<>(), null, nodesRequired, metadataSize, containerSize);
        assertEquals(nodesRequired, nodesCapacity.size());
        List<DatanodeDetails> nodesRandom = randomPlacer.chooseDatanodes(nodesCapacity, null, nodesRequired, metadataSize, containerSize);
        // One fifth of all calls are delete
        if (x % 5 == 0) {
            deleteContainer(nodeManagerCapacity, nodesCapacity, containerSize);
            deleteContainer(nodeManagerRandom, nodesRandom, containerSize);
        } else {
            createContainer(nodeManagerCapacity, nodesCapacity, containerSize);
            createContainer(nodeManagerRandom, nodesRandom, containerSize);
        }
    }
    DescriptiveStatistics postCapacity = computeStatistics(nodeManagerCapacity);
    DescriptiveStatistics postRandom = computeStatistics(nodeManagerRandom);
    // This is a very bold claim, and needs large number of I/O operations.
    // The claim in this assertion is that we improved the data distribution
    // of this cluster in relation to the start state of the cluster.
    Assert.assertTrue(beforeCapacity.getStandardDeviation() > postCapacity.getStandardDeviation());
    // This asserts that Capacity placement yields a better placement
    // algorithm than random placement, since both cluster started at an
    // identical state.
    Assert.assertTrue(postRandom.getStandardDeviation() > postCapacity.getStandardDeviation());
}