use of org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom in project ozone by apache.
the class TestContainerPlacement method testCapacityPlacementYieldsBetterDataDistribution.
/**
* This test simulates lots of Cluster I/O and updates the metadata in SCM.
* We simulate adding and removing containers from the cluster. It asserts
* that our placement algorithm has taken the capacity of nodes into
* consideration by asserting that standard deviation of used space on these
* has improved.
*/
@Test
public void testCapacityPlacementYieldsBetterDataDistribution() throws SCMException {
final int opsCount = 200 * 1000;
final int nodesRequired = 3;
Random random = new Random();
// The nature of init code in MockNodeManager yields similar clusters.
MockNodeManager nodeManagerCapacity = new MockNodeManager(true, 100);
MockNodeManager nodeManagerRandom = new MockNodeManager(true, 100);
DescriptiveStatistics beforeCapacity = computeStatistics(nodeManagerCapacity);
DescriptiveStatistics beforeRandom = computeStatistics(nodeManagerRandom);
// Assert that our initial layout of clusters are similar.
assertEquals(beforeCapacity.getStandardDeviation(), beforeRandom.getStandardDeviation(), 0.001);
SCMContainerPlacementCapacity capacityPlacer = new SCMContainerPlacementCapacity(nodeManagerCapacity, new OzoneConfiguration(), null, true, null);
SCMContainerPlacementRandom randomPlacer = new SCMContainerPlacementRandom(nodeManagerRandom, new OzoneConfiguration(), null, true, null);
for (int x = 0; x < opsCount; x++) {
long containerSize = random.nextInt(10) * OzoneConsts.GB;
long metadataSize = random.nextInt(10) * OzoneConsts.GB;
List<DatanodeDetails> nodesCapacity = capacityPlacer.chooseDatanodes(new ArrayList<>(), null, nodesRequired, metadataSize, containerSize);
assertEquals(nodesRequired, nodesCapacity.size());
List<DatanodeDetails> nodesRandom = randomPlacer.chooseDatanodes(nodesCapacity, null, nodesRequired, metadataSize, containerSize);
// One fifth of all calls are delete
if (x % 5 == 0) {
deleteContainer(nodeManagerCapacity, nodesCapacity, containerSize);
deleteContainer(nodeManagerRandom, nodesRandom, containerSize);
} else {
createContainer(nodeManagerCapacity, nodesCapacity, containerSize);
createContainer(nodeManagerRandom, nodesRandom, containerSize);
}
}
DescriptiveStatistics postCapacity = computeStatistics(nodeManagerCapacity);
DescriptiveStatistics postRandom = computeStatistics(nodeManagerRandom);
// This is a very bold claim, and needs large number of I/O operations.
// The claim in this assertion is that we improved the data distribution
// of this cluster in relation to the start state of the cluster.
Assert.assertTrue(beforeCapacity.getStandardDeviation() > postCapacity.getStandardDeviation());
// This asserts that Capacity placement yields a better placement
// algorithm than random placement, since both cluster started at an
// identical state.
Assert.assertTrue(postRandom.getStandardDeviation() > postCapacity.getStandardDeviation());
}
Aggregations