Examples with NameNode org.apache.hadoop.hdfs.server.namenode.NameNode used on opensource projects

Example 21 with NameNode

use of org.apache.hadoop.hdfs.server.namenode.NameNode in project hadoop by apache.

the class TestSnapshotDeletion method testHANNRestartAfterSnapshotDeletion.

/*
   * OP_DELETE_SNAPSHOT edits op was not decrementing the safemode threshold on
   * restart in HA mode. HDFS-5504
   */
@Test(timeout = 60000)
public void testHANNRestartAfterSnapshotDeletion() throws Exception {
    hdfs.close();
    cluster.shutdown();
    conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(1).build();
    cluster.transitionToActive(0);
    // stop the standby namenode
    NameNode snn = cluster.getNameNode(1);
    snn.stop();
    hdfs = (DistributedFileSystem) HATestUtil.configureFailoverFs(cluster, conf);
    Path dir = new Path("/dir");
    Path subDir = new Path(dir, "sub");
    hdfs.mkdirs(dir);
    hdfs.allowSnapshot(dir);
    for (int i = 0; i < 5; i++) {
        DFSTestUtil.createFile(hdfs, new Path(subDir, "" + i), 100, (short) 1, 1024L);
    }
    // take snapshot
    hdfs.createSnapshot(dir, "s0");
    // delete the subdir
    hdfs.delete(subDir, true);
    // roll the edit log
    NameNode ann = cluster.getNameNode(0);
    ann.getRpcServer().rollEditLog();
    hdfs.deleteSnapshot(dir, "s0");
    // wait for the blocks deletion at namenode
    Thread.sleep(2000);
    NameNodeAdapter.abortEditLogs(ann);
    cluster.restartNameNode(0, false);
    cluster.transitionToActive(0);
    // wait till the cluster becomes active
    cluster.waitClusterUp();
}

Example 22 with NameNode

use of org.apache.hadoop.hdfs.server.namenode.NameNode in project hadoop by apache.

the class TestWebHdfsDataLocality method testExcludeDataNodes.

@Test
public void testExcludeDataNodes() throws Exception {
    final Configuration conf = WebHdfsTestUtil.createConf();
    final String[] racks = { RACK0, RACK0, RACK1, RACK1, RACK2, RACK2 };
    final String[] hosts = { "DataNode1", "DataNode2", "DataNode3", "DataNode4", "DataNode5", "DataNode6" };
    final int nDataNodes = hosts.length;
    LOG.info("nDataNodes=" + nDataNodes + ", racks=" + Arrays.asList(racks) + ", hosts=" + Arrays.asList(hosts));
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).hosts(hosts).numDataNodes(nDataNodes).racks(racks).build();
    try {
        cluster.waitActive();
        final DistributedFileSystem dfs = cluster.getFileSystem();
        final NameNode namenode = cluster.getNameNode();
        final DatanodeManager dm = namenode.getNamesystem().getBlockManager().getDatanodeManager();
        LOG.info("dm=" + dm);
        final long blocksize = DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT;
        final String f = "/foo";
        //create a file with three replica.
        final Path p = new Path(f);
        final FSDataOutputStream out = dfs.create(p, (short) 3);
        out.write(1);
        out.close();
        //get replica location.
        final LocatedBlocks locatedblocks = NameNodeAdapter.getBlockLocations(namenode, f, 0, 1);
        final List<LocatedBlock> lb = locatedblocks.getLocatedBlocks();
        Assert.assertEquals(1, lb.size());
        final DatanodeInfo[] locations = lb.get(0).getLocations();
        Assert.assertEquals(3, locations.length);
        //For GETFILECHECKSUM, OPEN and APPEND,
        //the chosen datanode must be different with exclude nodes.
        StringBuffer sb = new StringBuffer();
        for (int i = 0; i < 2; i++) {
            sb.append(locations[i].getXferAddr());
            {
                // test GETFILECHECKSUM
                final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode(namenode, f, GetOpParam.Op.GETFILECHECKSUM, -1L, blocksize, sb.toString(), LOCALHOST);
                for (int j = 0; j <= i; j++) {
                    Assert.assertNotEquals(locations[j].getHostName(), chosen.getHostName());
                }
            }
            {
                // test OPEN
                final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode(namenode, f, GetOpParam.Op.OPEN, 0, blocksize, sb.toString(), LOCALHOST);
                for (int j = 0; j <= i; j++) {
                    Assert.assertNotEquals(locations[j].getHostName(), chosen.getHostName());
                }
            }
            {
                // test APPEND
                final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode(namenode, f, PostOpParam.Op.APPEND, -1L, blocksize, sb.toString(), LOCALHOST);
                for (int j = 0; j <= i; j++) {
                    Assert.assertNotEquals(locations[j].getHostName(), chosen.getHostName());
                }
            }
            sb.append(",");
        }
    } finally {
        cluster.shutdown();
    }
}

Example 23 with NameNode

use of org.apache.hadoop.hdfs.server.namenode.NameNode in project hadoop by apache.

the class TestDecommission method testClusterStats.

public void testClusterStats(int numNameNodes) throws IOException, InterruptedException {
    LOG.info("Starting test testClusterStats");
    int numDatanodes = 1;
    startCluster(numNameNodes, numDatanodes);
    for (int i = 0; i < numNameNodes; i++) {
        FileSystem fileSys = getCluster().getFileSystem(i);
        Path file = new Path("testClusterStats.dat");
        writeFile(fileSys, file, 1);
        FSNamesystem fsn = getCluster().getNamesystem(i);
        NameNode namenode = getCluster().getNameNode(i);
        DatanodeInfo decomInfo = takeNodeOutofService(i, null, 0, null, AdminStates.DECOMMISSION_INPROGRESS);
        DataNode decomNode = getDataNode(decomInfo);
        // Check namenode stats for multiple datanode heartbeats
        verifyStats(namenode, fsn, decomInfo, decomNode, true);
        // Stop decommissioning and verify stats
        DatanodeInfo retInfo = NameNodeAdapter.getDatanode(fsn, decomInfo);
        putNodeInService(i, retInfo);
        DataNode retNode = getDataNode(decomInfo);
        verifyStats(namenode, fsn, retInfo, retNode, false);
    }
}

Example 24 with NameNode

use of org.apache.hadoop.hdfs.server.namenode.NameNode in project hadoop by apache.

the class TestOpenFilesWithSnapshot method doTestMultipleSnapshots.

private void doTestMultipleSnapshots(boolean saveNamespace) throws IOException {
    Path path = new Path("/test");
    doWriteAndAbort(fs, path);
    fs.createSnapshot(path, "s2");
    fs.delete(new Path("/test/test"), true);
    fs.deleteSnapshot(path, "s2");
    cluster.triggerBlockReports();
    if (saveNamespace) {
        NameNode nameNode = cluster.getNameNode();
        NameNodeAdapter.enterSafeMode(nameNode, false);
        NameNodeAdapter.saveNamespace(nameNode);
        NameNodeAdapter.leaveSafeMode(nameNode);
    }
    cluster.restartNameNode(true);
}

Example 25 with NameNode

use of org.apache.hadoop.hdfs.server.namenode.NameNode in project hadoop by apache.

the class TestOpenFilesWithSnapshot method restartNameNode.

private void restartNameNode() throws Exception {
    cluster.triggerBlockReports();
    NameNode nameNode = cluster.getNameNode();
    NameNodeAdapter.enterSafeMode(nameNode, false);
    NameNodeAdapter.saveNamespace(nameNode);
    NameNodeAdapter.leaveSafeMode(nameNode);
    cluster.restartNameNode(true);
}