Examples with NetworkTopology org.apache.hadoop.net.NetworkTopology used on opensource projects

Example 6 with NetworkTopology

use of org.apache.hadoop.net.NetworkTopology in project hadoop by apache.

the class TestGetSplitHosts method testGetSplitHosts.

@Test
public void testGetSplitHosts() throws Exception {
    int numBlocks = 3;
    int block1Size = 100, block2Size = 150, block3Size = 75;
    int fileSize = block1Size + block2Size + block3Size;
    int replicationFactor = 3;
    NetworkTopology clusterMap = new NetworkTopology();
    BlockLocation[] bs = new BlockLocation[numBlocks];
    String[] block1Hosts = { "host1", "host2", "host3" };
    String[] block1Names = { "host1:100", "host2:100", "host3:100" };
    String[] block1Racks = { "/rack1/", "/rack1/", "/rack2/" };
    String[] block1Paths = new String[replicationFactor];
    for (int i = 0; i < replicationFactor; i++) {
        block1Paths[i] = block1Racks[i] + block1Names[i];
    }
    bs[0] = new BlockLocation(block1Names, block1Hosts, block1Paths, 0, block1Size);
    String[] block2Hosts = { "host4", "host5", "host6" };
    String[] block2Names = { "host4:100", "host5:100", "host6:100" };
    String[] block2Racks = { "/rack2/", "/rack3/", "/rack3/" };
    String[] block2Paths = new String[replicationFactor];
    for (int i = 0; i < replicationFactor; i++) {
        block2Paths[i] = block2Racks[i] + block2Names[i];
    }
    bs[1] = new BlockLocation(block2Names, block2Hosts, block2Paths, block1Size, block2Size);
    String[] block3Hosts = { "host1", "host7", "host8" };
    String[] block3Names = { "host1:100", "host7:100", "host8:100" };
    String[] block3Racks = { "/rack1/", "/rack4/", "/rack4/" };
    String[] block3Paths = new String[replicationFactor];
    for (int i = 0; i < replicationFactor; i++) {
        block3Paths[i] = block3Racks[i] + block3Names[i];
    }
    bs[2] = new BlockLocation(block3Names, block3Hosts, block3Paths, block1Size + block2Size, block3Size);
    SequenceFileInputFormat<String, String> sif = new SequenceFileInputFormat<String, String>();
    String[] hosts = sif.getSplitHosts(bs, 0, fileSize, clusterMap);
    // Contributions By Racks are
    // Rack1   175       
    // Rack2   275       
    // Rack3   150       
    // So, Rack2 hosts, host4 and host 3 should be returned
    // even if their individual contribution is not the highest
    assertTrue(hosts.length == replicationFactor);
    assertTrue(hosts[0].equalsIgnoreCase("host4"));
    assertTrue(hosts[1].equalsIgnoreCase("host3"));
    assertTrue(hosts[2].equalsIgnoreCase("host1"));
    // Now Create the blocks without topology information
    bs[0] = new BlockLocation(block1Names, block1Hosts, 0, block1Size);
    bs[1] = new BlockLocation(block2Names, block2Hosts, block1Size, block2Size);
    bs[2] = new BlockLocation(block3Names, block3Hosts, block1Size + block2Size, block3Size);
    hosts = sif.getSplitHosts(bs, 0, fileSize, clusterMap);
    // host1 makes the highest contribution among all hosts
    // So, that should be returned before others
    assertTrue(hosts.length == replicationFactor);
    assertTrue(hosts[0].equalsIgnoreCase("host1"));
}

Example 7 with NetworkTopology

use of org.apache.hadoop.net.NetworkTopology in project SSM by Intel-bigdata.

the class MoverProcessor method chooseTarget.

boolean chooseTarget(Dispatcher.DBlock db, Dispatcher.Source source, List<StorageType> targetTypes, Matcher matcher) {
    final NetworkTopology cluster = dispatcher.getCluster();
    for (StorageType t : targetTypes) {
        final List<Dispatcher.DDatanode.StorageGroup> targets = storages.getTargetStorages(t);
        Collections.shuffle(targets);
        for (Dispatcher.DDatanode.StorageGroup target : targets) {
            if (matcher.match(cluster, source.getDatanodeInfo(), target.getDatanodeInfo())) {
                final Dispatcher.PendingMove pm = source.addPendingMove(db, target);
                if (pm != null) {
                    dispatcher.executePendingMove(pm);
                    return true;
                }
            }
        }
    }
    return false;
}

Example 8 with NetworkTopology

use of org.apache.hadoop.net.NetworkTopology in project hadoop by apache.

the class TestBlockStoragePolicy method testChooseSsdOverDisk.

@Test
public void testChooseSsdOverDisk() throws Exception {
    BlockStoragePolicy policy = new BlockStoragePolicy((byte) 9, "TEST1", new StorageType[] { StorageType.SSD, StorageType.DISK, StorageType.ARCHIVE }, new StorageType[] {}, new StorageType[] {});
    final String[] racks = { "/d1/r1", "/d1/r1", "/d1/r1" };
    final String[] hosts = { "host1", "host2", "host3" };
    final StorageType[] disks = { StorageType.DISK, StorageType.DISK, StorageType.DISK };
    final DatanodeStorageInfo[] diskStorages = DFSTestUtil.createDatanodeStorageInfos(3, racks, hosts, disks);
    final DatanodeDescriptor[] dataNodes = DFSTestUtil.toDatanodeDescriptor(diskStorages);
    for (int i = 0; i < dataNodes.length; i++) {
        BlockManagerTestUtil.updateStorage(dataNodes[i], new DatanodeStorage("ssd" + i, DatanodeStorage.State.NORMAL, StorageType.SSD));
    }
    FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
    conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
    File baseDir = PathUtils.getTestDir(TestReplicationPolicy.class);
    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, new File(baseDir, "name").getPath());
    DFSTestUtil.formatNameNode(conf);
    NameNode namenode = new NameNode(conf);
    final BlockManager bm = namenode.getNamesystem().getBlockManager();
    BlockPlacementPolicy replicator = bm.getBlockPlacementPolicy();
    NetworkTopology cluster = bm.getDatanodeManager().getNetworkTopology();
    for (DatanodeDescriptor datanode : dataNodes) {
        cluster.add(datanode);
    }
    DatanodeStorageInfo[] targets = replicator.chooseTarget("/foo", 3, dataNodes[0], Collections.<DatanodeStorageInfo>emptyList(), false, new HashSet<Node>(), 0, policy, null);
    System.out.println(policy.getName() + ": " + Arrays.asList(targets));
    Assert.assertEquals(2, targets.length);
    Assert.assertEquals(StorageType.SSD, targets[0].getStorageType());
    Assert.assertEquals(StorageType.DISK, targets[1].getStorageType());
}

Example 9 with NetworkTopology

use of org.apache.hadoop.net.NetworkTopology in project hadoop by apache.

the class TestBlockManager method addNodes.

private void addNodes(Iterable<DatanodeDescriptor> nodesToAdd) {
    NetworkTopology cluster = bm.getDatanodeManager().getNetworkTopology();
    // construct network topology
    for (DatanodeDescriptor dn : nodesToAdd) {
        cluster.add(dn);
        dn.getStorageInfos()[0].setUtilizationForTesting(2 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, 2 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L);
        dn.updateHeartbeat(BlockManagerTestUtil.getStorageReportsForDatanode(dn), 0L, 0L, 0, 0, null);
        bm.getDatanodeManager().checkIfClusterIsNowMultiRack(dn);
    }
}

Example 10 with NetworkTopology

use of org.apache.hadoop.net.NetworkTopology in project hadoop by apache.

the class TestReconstructStripedBlocksWithRackAwareness method testReconstructForNotEnoughRacks.

/**
   * When there are all the internal blocks available but they are not placed on
   * enough racks, NameNode should avoid normal decoding reconstruction but copy
   * an internal block to a new rack.
   *
   * In this test, we first need to create a scenario that a striped block has
   * all the internal blocks but distributed in <6 racks. Then we check if the
   * redundancy monitor can correctly schedule the reconstruction work for it.
   */
@Test
public void testReconstructForNotEnoughRacks() throws Exception {
    LOG.info("cluster hosts: {}, racks: {}", Arrays.asList(hosts), Arrays.asList(racks));
    conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, StripedFileTestUtil.getDefaultECPolicy().getName());
    cluster = new MiniDFSCluster.Builder(conf).racks(racks).hosts(hosts).numDataNodes(hosts.length).build();
    cluster.waitActive();
    fs = cluster.getFileSystem();
    fs.setErasureCodingPolicy(new Path("/"), StripedFileTestUtil.getDefaultECPolicy().getName());
    FSNamesystem fsn = cluster.getNamesystem();
    BlockManager bm = fsn.getBlockManager();
    MiniDFSCluster.DataNodeProperties lastHost = stopDataNode(hosts[hosts.length - 1]);
    final Path file = new Path("/foo");
    // the file's block is in 9 dn but 5 racks
    DFSTestUtil.createFile(fs, file, cellSize * dataBlocks * 2, (short) 1, 0L);
    Assert.assertEquals(0, bm.numOfUnderReplicatedBlocks());
    final INodeFile fileNode = fsn.getFSDirectory().getINode4Write(file.toString()).asFile();
    BlockInfoStriped blockInfo = (BlockInfoStriped) fileNode.getLastBlock();
    // we now should have 9 internal blocks distributed in 5 racks
    Set<String> rackSet = new HashSet<>();
    for (DatanodeStorageInfo storage : blockInfo.storages) {
        rackSet.add(storage.getDatanodeDescriptor().getNetworkLocation());
    }
    Assert.assertEquals(dataBlocks - 1, rackSet.size());
    // restart the stopped datanode
    cluster.restartDataNode(lastHost);
    cluster.waitActive();
    // make sure we have 6 racks again
    NetworkTopology topology = bm.getDatanodeManager().getNetworkTopology();
    Assert.assertEquals(hosts.length, topology.getNumOfLeaves());
    Assert.assertEquals(dataBlocks, topology.getNumOfRacks());
    // pause all the heartbeats
    for (DataNode dn : cluster.getDataNodes()) {
        DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
    }
    fsn.writeLock();
    try {
        bm.processMisReplicatedBlocks();
    } finally {
        fsn.writeUnlock();
    }
    // check if redundancy monitor correctly schedule the reconstruction work.
    boolean scheduled = false;
    for (int i = 0; i < 5; i++) {
        // retry 5 times
        for (DatanodeStorageInfo storage : blockInfo.storages) {
            if (storage != null) {
                DatanodeDescriptor dn = storage.getDatanodeDescriptor();
                Assert.assertEquals(0, dn.getNumberOfBlocksToBeErasureCoded());
                if (dn.getNumberOfBlocksToBeReplicated() == 1) {
                    scheduled = true;
                }
            }
        }
        if (scheduled) {
            break;
        }
        Thread.sleep(1000);
    }
    Assert.assertTrue(scheduled);
}