Examples with MockNodeManager org.apache.hadoop.hdds.scm.container.MockNodeManager used on opensource projects

Example 6 with MockNodeManager

use of org.apache.hadoop.hdds.scm.container.MockNodeManager in project ozone by apache.

the class TestPipelinePlacementPolicy method init.

@Before
public void init() throws Exception {
    cluster = initTopology();
    // start with nodes with rack awareness.
    nodeManager = new MockNodeManager(cluster, getNodesWithRackAwareness(), false, PIPELINE_PLACEMENT_MAX_NODES_COUNT);
    conf = SCMTestUtils.getConf();
    conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, PIPELINE_LOAD_LIMIT);
    conf.setStorageSize(OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN, 10, StorageUnit.MB);
    nodeManager.setNumPipelinePerDatanode(PIPELINE_LOAD_LIMIT);
    testDir = GenericTestUtils.getTestDir(TestContainerManagerImpl.class.getSimpleName() + UUID.randomUUID());
    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
    dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition());
    scmhaManager = MockSCMHAManager.getInstance(true);
    stateManager = PipelineStateManagerImpl.newBuilder().setPipelineStore(SCMDBDefinition.PIPELINES.getTable(dbStore)).setRatisServer(scmhaManager.getRatisServer()).setNodeManager(nodeManager).setSCMDBTransactionBuffer(scmhaManager.getDBTransactionBuffer()).build();
    placementPolicy = new PipelinePlacementPolicy(nodeManager, stateManager, conf);
}

Example 7 with MockNodeManager

use of org.apache.hadoop.hdds.scm.container.MockNodeManager in project ozone by apache.

the class TestPipelinePlacementPolicy method testValidatePlacementPolicyOK.

@Test
public void testValidatePlacementPolicyOK() {
    cluster = initTopology();
    nodeManager = new MockNodeManager(cluster, getNodesWithRackAwareness(), false, PIPELINE_PLACEMENT_MAX_NODES_COUNT);
    placementPolicy = new PipelinePlacementPolicy(nodeManager, stateManager, conf);
    List<DatanodeDetails> dns = new ArrayList<>();
    dns.add(MockDatanodeDetails.createDatanodeDetails("host1", "/rack1"));
    dns.add(MockDatanodeDetails.createDatanodeDetails("host2", "/rack1"));
    dns.add(MockDatanodeDetails.createDatanodeDetails("host3", "/rack2"));
    for (DatanodeDetails dn : dns) {
        cluster.add(dn);
    }
    ContainerPlacementStatus status = placementPolicy.validateContainerPlacement(dns, 3);
    assertTrue(status.isPolicySatisfied());
    assertEquals(0, status.misReplicationCount());
    List<DatanodeDetails> subSet = new ArrayList<>();
    // Cut it down to two nodes, two racks
    subSet.add(dns.get(0));
    subSet.add(dns.get(2));
    status = placementPolicy.validateContainerPlacement(subSet, 3);
    assertTrue(status.isPolicySatisfied());
    assertEquals(0, status.misReplicationCount());
    // Cut it down to two nodes, one racks
    subSet = new ArrayList<>();
    subSet.add(dns.get(0));
    subSet.add(dns.get(1));
    status = placementPolicy.validateContainerPlacement(subSet, 3);
    assertFalse(status.isPolicySatisfied());
    assertEquals(1, status.misReplicationCount());
    // One node, but only one replica
    subSet = new ArrayList<>();
    subSet.add(dns.get(0));
    status = placementPolicy.validateContainerPlacement(subSet, 1);
    assertTrue(status.isPolicySatisfied());
}

Example 8 with MockNodeManager

use of org.apache.hadoop.hdds.scm.container.MockNodeManager in project ozone by apache.

the class TestPipelinePlacementPolicy method testChooseNodeWithSingleNodeRack.

@Test
public void testChooseNodeWithSingleNodeRack() throws IOException {
    // There is only one node on 3 racks altogether.
    List<DatanodeDetails> datanodes = new ArrayList<>();
    for (Node node : SINGLE_NODE_RACK) {
        DatanodeDetails datanode = overwriteLocationInNode(MockDatanodeDetails.randomDatanodeDetails(), node);
        datanodes.add(datanode);
    }
    MockNodeManager localNodeManager = new MockNodeManager(initTopology(), datanodes, false, datanodes.size());
    PipelineStateManager tempPipelineStateManager = PipelineStateManagerImpl.newBuilder().setNodeManager(localNodeManager).setRatisServer(scmhaManager.getRatisServer()).setPipelineStore(SCMDBDefinition.PIPELINES.getTable(dbStore)).setSCMDBTransactionBuffer(scmhaManager.getDBTransactionBuffer()).build();
    PipelinePlacementPolicy localPlacementPolicy = new PipelinePlacementPolicy(localNodeManager, tempPipelineStateManager, conf);
    int nodesRequired = HddsProtos.ReplicationFactor.THREE.getNumber();
    List<DatanodeDetails> results = localPlacementPolicy.chooseDatanodes(new ArrayList<>(datanodes.size()), new ArrayList<>(datanodes.size()), nodesRequired, 0, 0);
    Assert.assertEquals(nodesRequired, results.size());
    // 3 nodes should be on different racks.
    Assert.assertNotEquals(results.get(0).getNetworkLocation(), results.get(1).getNetworkLocation());
    Assert.assertNotEquals(results.get(0).getNetworkLocation(), results.get(2).getNetworkLocation());
    Assert.assertNotEquals(results.get(1).getNetworkLocation(), results.get(2).getNetworkLocation());
}

Example 9 with MockNodeManager

use of org.apache.hadoop.hdds.scm.container.MockNodeManager in project ozone by apache.

the class TestPipelinePlacementPolicy method testValidatePlacementPolicySingleRackInCluster.

@Test
public void testValidatePlacementPolicySingleRackInCluster() {
    cluster = initTopology();
    nodeManager = new MockNodeManager(cluster, new ArrayList<>(), false, PIPELINE_PLACEMENT_MAX_NODES_COUNT);
    placementPolicy = new PipelinePlacementPolicy(nodeManager, stateManager, conf);
    List<DatanodeDetails> dns = new ArrayList<>();
    dns.add(MockDatanodeDetails.createDatanodeDetails("host1", "/rack1"));
    dns.add(MockDatanodeDetails.createDatanodeDetails("host2", "/rack1"));
    dns.add(MockDatanodeDetails.createDatanodeDetails("host3", "/rack1"));
    for (DatanodeDetails dn : dns) {
        cluster.add(dn);
    }
    ContainerPlacementStatus status = placementPolicy.validateContainerPlacement(dns, 3);
    assertTrue(status.isPolicySatisfied());
    assertEquals(0, status.misReplicationCount());
}

Example 10 with MockNodeManager

use of org.apache.hadoop.hdds.scm.container.MockNodeManager in project ozone by apache.

the class TestPipelinePlacementPolicy method testChooseNodeNotEnoughSpace.

@Test
public void testChooseNodeNotEnoughSpace() throws IOException {
    // There is only one node on 3 racks altogether.
    List<DatanodeDetails> datanodes = new ArrayList<>();
    for (Node node : SINGLE_NODE_RACK) {
        DatanodeDetails datanode = overwriteLocationInNode(MockDatanodeDetails.randomDatanodeDetails(), node);
        datanodes.add(datanode);
    }
    MockNodeManager localNodeManager = new MockNodeManager(initTopology(), datanodes, false, datanodes.size());
    PipelineStateManager tempPipelineStateManager = PipelineStateManagerImpl.newBuilder().setNodeManager(localNodeManager).setRatisServer(scmhaManager.getRatisServer()).setPipelineStore(SCMDBDefinition.PIPELINES.getTable(dbStore)).setSCMDBTransactionBuffer(scmhaManager.getDBTransactionBuffer()).build();
    PipelinePlacementPolicy localPlacementPolicy = new PipelinePlacementPolicy(localNodeManager, tempPipelineStateManager, conf);
    int nodesRequired = HddsProtos.ReplicationFactor.THREE.getNumber();
    String expectedMessageSubstring = "Unable to find enough nodes that meet " + "the space requirement";
    try {
        // A huge container size
        localPlacementPolicy.chooseDatanodes(new ArrayList<>(datanodes.size()), new ArrayList<>(datanodes.size()), nodesRequired, 0, 10 * OzoneConsts.TB);
        Assert.fail("SCMException should have been thrown.");
    } catch (SCMException ex) {
        Assert.assertTrue(ex.getMessage().contains(expectedMessageSubstring));
    }
    try {
        // a huge free space min configured
        localPlacementPolicy.chooseDatanodes(new ArrayList<>(datanodes.size()), new ArrayList<>(datanodes.size()), nodesRequired, 10 * OzoneConsts.TB, 0);
        Assert.fail("SCMException should have been thrown.");
    } catch (SCMException ex) {
        Assert.assertTrue(ex.getMessage().contains(expectedMessageSubstring));
    }
}