Examples with MiniDFSCluster org.apache.hadoop.hdfs.MiniDFSCluster used on opensource projects

Example 96 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestBlocksWithNotEnoughRacks method testReplDueToNodeFailRespectsRackPolicy.

/*
   * Test that when a block is replicated because a replica is lost due
   * to host failure the the rack policy is preserved.
   */
@Test
public void testReplDueToNodeFailRespectsRackPolicy() throws Exception {
    Configuration conf = getConf();
    short REPLICATION_FACTOR = 3;
    final Path filePath = new Path("/testFile");
    // Last datanode is on a different rack
    String[] racks = { "/rack1", "/rack1", "/rack1", "/rack2", "/rack2" };
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(racks.length).racks(racks).build();
    final FSNamesystem ns = cluster.getNameNode().getNamesystem();
    final DatanodeManager dm = ns.getBlockManager().getDatanodeManager();
    try {
        // Create a file with one block with a replication factor of 3
        final FileSystem fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L);
        ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
        DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
        // Make the last datanode look like it failed to heartbeat by 
        // calling removeDatanode and stopping it.
        ArrayList<DataNode> datanodes = cluster.getDataNodes();
        int idx = datanodes.size() - 1;
        DataNode dataNode = datanodes.get(idx);
        DatanodeID dnId = dataNode.getDatanodeId();
        cluster.stopDataNode(idx);
        dm.removeDatanode(dnId);
        // The block should still have sufficient # replicas, across racks.
        // The last node may not have contained a replica, but if it did
        // it should have been replicated within the same rack.
        DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
        // Fail the last datanode again, it's also on rack2 so there is
        // only 1 rack for all the replicas
        datanodes = cluster.getDataNodes();
        idx = datanodes.size() - 1;
        dataNode = datanodes.get(idx);
        dnId = dataNode.getDatanodeId();
        cluster.stopDataNode(idx);
        dm.removeDatanode(dnId);
        // Make sure we have enough live replicas even though we are
        // short one rack. The cluster now has only 1 rack thus we just make sure
        // we still have 3 replicas.
        DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 0);
    } finally {
        cluster.shutdown();
    }
}

Example 97 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestBlocksWithNotEnoughRacks method testNodeDecomissionRespectsRackPolicy.

/*
   * Test that rack policy is still respected when blocks are replicated
   * due to node decommissioning.
   */
@Test
public void testNodeDecomissionRespectsRackPolicy() throws Exception {
    Configuration conf = getConf();
    short REPLICATION_FACTOR = 2;
    final Path filePath = new Path("/testFile");
    HostsFileWriter hostsFileWriter = new HostsFileWriter();
    hostsFileWriter.initialize(conf, "temp/decommission");
    // Two blocks and four racks
    String[] racks = { "/rack1", "/rack1", "/rack2", "/rack2" };
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(racks.length).racks(racks).build();
    final FSNamesystem ns = cluster.getNameNode().getNamesystem();
    try {
        // Create a file with one block
        final FileSystem fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L);
        ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
        DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
        // Decommission one of the hosts with the block, this should cause 
        // the block to get replicated to another host on the same rack,
        // otherwise the rack policy is violated.
        BlockLocation[] locs = fs.getFileBlockLocations(fs.getFileStatus(filePath), 0, Long.MAX_VALUE);
        String name = locs[0].getNames()[0];
        hostsFileWriter.initExcludeHost(name);
        ns.getBlockManager().getDatanodeManager().refreshNodes(conf);
        DFSTestUtil.waitForDecommission(fs, name);
        // Check the block still has sufficient # replicas across racks
        DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
    } finally {
        cluster.shutdown();
        hostsFileWriter.cleanup();
    }
}

Example 98 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestBlocksWithNotEnoughRacks method testReduceReplFactorRespectsRackPolicy.

/*
   * Reduce the replication factor of a file, making sure that the only
   * cross rack replica is not removed when deleting replicas.
   */
@Test
public void testReduceReplFactorRespectsRackPolicy() throws Exception {
    Configuration conf = getConf();
    short REPLICATION_FACTOR = 3;
    final Path filePath = new Path("/testFile");
    String[] racks = { "/rack1", "/rack1", "/rack2", "/rack2" };
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(racks.length).racks(racks).build();
    final FSNamesystem ns = cluster.getNameNode().getNamesystem();
    try {
        // Create a file with one block
        final FileSystem fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L);
        ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
        DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
        // Decrease the replication factor, make sure the deleted replica
        // was not the one that lived on the rack with only one replica,
        // ie we should still have 2 racks after reducing the repl factor.
        REPLICATION_FACTOR = 2;
        NameNodeAdapter.setReplication(ns, "/testFile", REPLICATION_FACTOR);
        DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
    } finally {
        cluster.shutdown();
    }
}

Example 99 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestBlocksWithNotEnoughRacks method testNodeDecomissionWithOverreplicationRespectsRackPolicy.

/*
   * Test that rack policy is still respected when blocks are replicated
   * due to node decommissioning, when the blocks are over-replicated.
   */
@Test
public void testNodeDecomissionWithOverreplicationRespectsRackPolicy() throws Exception {
    Configuration conf = getConf();
    short REPLICATION_FACTOR = 5;
    final Path filePath = new Path("/testFile");
    HostsFileWriter hostsFileWriter = new HostsFileWriter();
    hostsFileWriter.initialize(conf, "temp/decommission");
    // All hosts are on two racks, only one host on /rack2
    String[] racks = { "/rack1", "/rack2", "/rack1", "/rack1", "/rack1" };
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(racks.length).racks(racks).build();
    final FSNamesystem ns = cluster.getNameNode().getNamesystem();
    try {
        final FileSystem fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L);
        ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
        DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
        // Lower the replication factor so the blocks are over replicated
        REPLICATION_FACTOR = 2;
        fs.setReplication(filePath, REPLICATION_FACTOR);
        // Decommission one of the hosts with the block that is not on
        // the lone host on rack2 (if we decomission that host it would
        // be impossible to respect the rack policy).
        BlockLocation[] locs = fs.getFileBlockLocations(fs.getFileStatus(filePath), 0, Long.MAX_VALUE);
        for (String top : locs[0].getTopologyPaths()) {
            if (!top.startsWith("/rack2")) {
                String name = top.substring("/rack1".length() + 1);
                hostsFileWriter.initExcludeHost(name);
                ns.getBlockManager().getDatanodeManager().refreshNodes(conf);
                DFSTestUtil.waitForDecommission(fs, name);
                break;
            }
        }
        // Check the block still has sufficient # replicas across racks,
        // ie we didn't remove the replica on the host on /rack1.
        DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
    } finally {
        cluster.shutdown();
        hostsFileWriter.cleanup();
    }
}

Example 100 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestBlocksWithNotEnoughRacks method testUnderReplicatedUsesNewRacks.

/*
   * Creates a block with all datanodes on the same rack. Add additional
   * datanodes on a different rack and increase the replication factor, 
   * making sure there are enough replicas across racks. If the previous
   * test passes this one should too, however this test may pass when
   * the previous one fails because the replication code is explicitly
   * triggered by setting the replication factor.
   */
@Test
public void testUnderReplicatedUsesNewRacks() throws Exception {
    Configuration conf = getConf();
    short REPLICATION_FACTOR = 3;
    final Path filePath = new Path("/testFile");
    // All datanodes are on the same rack
    String[] racks = { "/rack1", "/rack1", "/rack1", "/rack1", "/rack1" };
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(racks.length).racks(racks).build();
    final FSNamesystem ns = cluster.getNameNode().getNamesystem();
    try {
        // Create a file with one block
        final FileSystem fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L);
        ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
        DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 0);
        // Add new datanodes on a different rack and increase the
        // replication factor so the block is underreplicated and make
        // sure at least one of the hosts on the new rack is used. 
        String[] newRacks = { "/rack2", "/rack2" };
        cluster.startDataNodes(conf, 2, true, null, newRacks);
        REPLICATION_FACTOR = 5;
        NameNodeAdapter.setReplication(ns, "/testFile", REPLICATION_FACTOR);
        DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
    } finally {
        cluster.shutdown();
    }
}