Examples with NodeSchemaManager org.apache.hadoop.hdds.scm.net.NodeSchemaManager used on opensource projects

Example 1 with NodeSchemaManager

use of org.apache.hadoop.hdds.scm.net.NodeSchemaManager in project ozone by apache.

the class TestKeyManagerImpl method setUp.

@BeforeClass
public static void setUp() throws Exception {
    conf = new OzoneConfiguration();
    dir = GenericTestUtils.getRandomizedTestDir();
    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, dir.toString());
    conf.set(OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY, "true");
    mockScmBlockLocationProtocol = mock(ScmBlockLocationProtocol.class);
    nodeManager = new MockNodeManager(true, 10);
    NodeSchema[] schemas = new NodeSchema[] { ROOT_SCHEMA, RACK_SCHEMA, LEAF_SCHEMA };
    NodeSchemaManager schemaManager = NodeSchemaManager.getInstance();
    schemaManager.init(schemas, false);
    NetworkTopology clusterMap = new NetworkTopologyImpl(schemaManager);
    nodeManager.getAllNodes().stream().forEach(node -> {
        node.setNetworkName(node.getUuidString());
        clusterMap.add(node);
    });
    ((MockNodeManager) nodeManager).setNetworkTopology(clusterMap);
    SCMConfigurator configurator = new SCMConfigurator();
    configurator.setScmNodeManager(nodeManager);
    configurator.setNetworkTopology(clusterMap);
    configurator.setSCMHAManager(MockSCMHAManager.getInstance(true));
    configurator.setScmContext(SCMContext.emptyContext());
    scm = HddsTestUtils.getScm(conf, configurator);
    scm.start();
    scm.exitSafeMode();
    scmBlockSize = (long) conf.getStorageSize(OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
    conf.setLong(OZONE_KEY_PREALLOCATION_BLOCKS_MAX, 10);
    mockScmContainerClient = Mockito.mock(StorageContainerLocationProtocol.class);
    OmTestManagers omTestManagers = new OmTestManagers(conf, scm.getBlockProtocolServer(), mockScmContainerClient);
    om = omTestManagers.getOzoneManager();
    metadataManager = omTestManagers.getMetadataManager();
    keyManager = (KeyManagerImpl) omTestManagers.getKeyManager();
    prefixManager = omTestManagers.getPrefixManager();
    writeClient = omTestManagers.getWriteClient();
    mockContainerClient();
    Mockito.when(mockScmBlockLocationProtocol.allocateBlock(Mockito.anyLong(), Mockito.anyInt(), any(ReplicationConfig.class), Mockito.anyString(), any(ExcludeList.class))).thenThrow(new SCMException("SafeModePrecheck failed for allocateBlock", ResultCodes.SAFE_MODE_EXCEPTION));
    createVolume(VOLUME_NAME);
    createBucket(VOLUME_NAME, BUCKET_NAME, false);
    createBucket(VOLUME_NAME, VERSIONED_BUCKET_NAME, true);
}

Example 2 with NodeSchemaManager

use of org.apache.hadoop.hdds.scm.net.NodeSchemaManager in project ozone by apache.

the class TestFindTargetStrategy method testFindTargetGreedyByNetworkTopology.

/**
 * Checks whether FindTargetGreedyByNetworkTopology always choose target
 * for a given source by network topology distance.
 */
@Test
public void testFindTargetGreedyByNetworkTopology() {
    // network topology with default cost
    List<NodeSchema> schemas = new ArrayList<>();
    schemas.add(ROOT_SCHEMA);
    schemas.add(RACK_SCHEMA);
    schemas.add(NODEGROUP_SCHEMA);
    schemas.add(LEAF_SCHEMA);
    NodeSchemaManager manager = NodeSchemaManager.getInstance();
    manager.init(schemas.toArray(new NodeSchema[0]), true);
    NetworkTopology newCluster = new NetworkTopologyImpl(manager);
    DatanodeDetails source = MockDatanodeDetails.createDatanodeDetails("1.1.1.1", "/r1/ng1");
    // create one target in the same rack and same node group
    DatanodeDetails target1 = MockDatanodeDetails.createDatanodeDetails("2.2.2.2", "/r1/ng1");
    // create tree targets in the same rack but different node group
    DatanodeDetails target2 = MockDatanodeDetails.createDatanodeDetails("3.3.3.3", "/r1/ng2");
    DatanodeDetails target3 = MockDatanodeDetails.createDatanodeDetails("4.4.4.4", "/r1/ng2");
    DatanodeDetails target4 = MockDatanodeDetails.createDatanodeDetails("5.5.5.5", "/r1/ng2");
    // create one target in different rack
    DatanodeDetails target5 = MockDatanodeDetails.createDatanodeDetails("6.6.6.6", "/r2/ng1");
    // add all datanode to cluster map
    newCluster.add(source);
    newCluster.add(target1);
    newCluster.add(target2);
    newCluster.add(target3);
    newCluster.add(target4);
    newCluster.add(target5);
    // make sure targets have different network topology distance to source
    assertEquals(2, newCluster.getDistanceCost(source, target1));
    assertEquals(4, newCluster.getDistanceCost(source, target2));
    assertEquals(4, newCluster.getDistanceCost(source, target3));
    assertEquals(4, newCluster.getDistanceCost(source, target4));
    assertEquals(6, newCluster.getDistanceCost(source, target5));
    // insert in ascending order of network topology distance
    List<DatanodeUsageInfo> overUtilizedDatanodes = new ArrayList<>();
    // set the farthest target with the lowest usage info
    overUtilizedDatanodes.add(new DatanodeUsageInfo(target5, new SCMNodeStat(100, 0, 90)));
    // set the tree targets, which have the same network topology distance
    // to source , with different usage info
    overUtilizedDatanodes.add(new DatanodeUsageInfo(target2, new SCMNodeStat(100, 0, 20)));
    overUtilizedDatanodes.add(new DatanodeUsageInfo(target3, new SCMNodeStat(100, 0, 40)));
    overUtilizedDatanodes.add(new DatanodeUsageInfo(target4, new SCMNodeStat(100, 0, 60)));
    // set the nearest target with the highest usage info
    overUtilizedDatanodes.add(new DatanodeUsageInfo(target1, new SCMNodeStat(100, 0, 10)));
    FindTargetGreedyByNetworkTopology findTargetGreedyByNetworkTopology = new FindTargetGreedyByNetworkTopology(null, null, null, newCluster);
    findTargetGreedyByNetworkTopology.reInitialize(overUtilizedDatanodes, null, null);
    findTargetGreedyByNetworkTopology.sortTargetForSource(source);
    Collection<DatanodeUsageInfo> potentialTargets = findTargetGreedyByNetworkTopology.getPotentialTargets();
    Object[] sortedPotentialTargetArray = potentialTargets.toArray();
    Assert.assertEquals(sortedPotentialTargetArray.length, 5);
    // although target1 has the highest usage, it has the nearest network
    // topology distance to source, so it should be at the head of the
    // sorted PotentialTargetArray
    Assert.assertEquals(((DatanodeUsageInfo) sortedPotentialTargetArray[0]).getDatanodeDetails(), target1);
    // these targets have same network topology distance to source,
    // so they should be sorted by usage
    Assert.assertEquals(((DatanodeUsageInfo) sortedPotentialTargetArray[1]).getDatanodeDetails(), target4);
    Assert.assertEquals(((DatanodeUsageInfo) sortedPotentialTargetArray[2]).getDatanodeDetails(), target3);
    Assert.assertEquals(((DatanodeUsageInfo) sortedPotentialTargetArray[3]).getDatanodeDetails(), target2);
    // target5 has the lowest usage , but it has the farthest distance to source
    // so it should be at the tail of the sorted PotentialTargetArray
    Assert.assertEquals(((DatanodeUsageInfo) sortedPotentialTargetArray[4]).getDatanodeDetails(), target5);
}