Examples with FSNamesystem org.apache.hadoop.hdfs.server.namenode.FSNamesystem used on opensource projects

Example 71 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestDecommission method testDecommission.

private void testDecommission(int numNamenodes, int numDatanodes) throws IOException {
    LOG.info("Starting test testDecommission");
    startCluster(numNamenodes, numDatanodes);
    ArrayList<ArrayList<DatanodeInfo>> namenodeDecomList = new ArrayList<ArrayList<DatanodeInfo>>(numNamenodes);
    for (int i = 0; i < numNamenodes; i++) {
        namenodeDecomList.add(i, new ArrayList<DatanodeInfo>(numDatanodes));
    }
    Path file1 = new Path("testDecommission.dat");
    for (int iteration = 0; iteration < numDatanodes - 1; iteration++) {
        int replicas = numDatanodes - iteration - 1;
        // Start decommissioning one namenode at a time
        for (int i = 0; i < numNamenodes; i++) {
            ArrayList<DatanodeInfo> decommissionedNodes = namenodeDecomList.get(i);
            FileSystem fileSys = getCluster().getFileSystem(i);
            FSNamesystem ns = getCluster().getNamesystem(i);
            writeFile(fileSys, file1, replicas);
            int deadDecomissioned = ns.getNumDecomDeadDataNodes();
            int liveDecomissioned = ns.getNumDecomLiveDataNodes();
            // Decommission one node. Verify that node is decommissioned.
            DatanodeInfo decomNode = takeNodeOutofService(i, null, 0, decommissionedNodes, AdminStates.DECOMMISSIONED);
            decommissionedNodes.add(decomNode);
            assertEquals(deadDecomissioned, ns.getNumDecomDeadDataNodes());
            assertEquals(liveDecomissioned + 1, ns.getNumDecomLiveDataNodes());
            // Ensure decommissioned datanode is not automatically shutdown
            DFSClient client = getDfsClient(i);
            assertEquals("All datanodes must be alive", numDatanodes, client.datanodeReport(DatanodeReportType.LIVE).length);
            // wait for the block to be replicated
            int tries = 0;
            while (tries++ < 20) {
                try {
                    Thread.sleep(1000);
                    if (checkFile(fileSys, file1, replicas, decomNode.getXferAddr(), numDatanodes) == null) {
                        break;
                    }
                } catch (InterruptedException ie) {
                }
            }
            assertTrue("Checked if block was replicated after decommission, tried " + tries + " times.", tries < 20);
            cleanupFile(fileSys, file1);
        }
    }
    // Restart the cluster and ensure decommissioned datanodes
    // are allowed to register with the namenode
    shutdownCluster();
    startCluster(numNamenodes, numDatanodes);
}

Example 72 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestDatanodeRegistration method testForcedRegistration.

// IBRs are async operations to free up IPC handlers.  This means the IBR
// response will not contain non-IPC level exceptions - which in practice
// should not occur other than dead/unregistered node which will trigger a
// re-registration.  If a non-IPC exception does occur, the safety net is
// a forced re-registration on the next heartbeat.
@Test
public void testForcedRegistration() throws Exception {
    final Configuration conf = new HdfsConfiguration();
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_HANDLER_COUNT_KEY, 4);
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, Integer.MAX_VALUE);
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    cluster.getHttpUri(0);
    FSNamesystem fsn = cluster.getNamesystem();
    String bpId = fsn.getBlockPoolId();
    DataNode dn = cluster.getDataNodes().get(0);
    DatanodeDescriptor dnd = NameNodeAdapter.getDatanode(fsn, dn.getDatanodeId());
    DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
    DatanodeStorageInfo storage = dnd.getStorageInfos()[0];
    // registration should not change after heartbeat.
    assertTrue(dnd.isRegistered());
    DatanodeRegistration lastReg = dn.getDNRegistrationForBP(bpId);
    waitForHeartbeat(dn, dnd);
    assertSame(lastReg, dn.getDNRegistrationForBP(bpId));
    // force a re-registration on next heartbeat.
    dnd.setForceRegistration(true);
    assertFalse(dnd.isRegistered());
    waitForHeartbeat(dn, dnd);
    assertTrue(dnd.isRegistered());
    DatanodeRegistration newReg = dn.getDNRegistrationForBP(bpId);
    assertNotSame(lastReg, newReg);
    lastReg = newReg;
    // registration should not change on subsequent heartbeats.
    waitForHeartbeat(dn, dnd);
    assertTrue(dnd.isRegistered());
    assertSame(lastReg, dn.getDNRegistrationForBP(bpId));
    assertTrue(waitForBlockReport(dn, dnd));
    assertTrue(dnd.isRegistered());
    assertSame(lastReg, dn.getDNRegistrationForBP(bpId));
    // check that block report is not processed and registration didn't change.
    dnd.setForceRegistration(true);
    assertFalse(waitForBlockReport(dn, dnd));
    assertFalse(dnd.isRegistered());
    assertSame(lastReg, dn.getDNRegistrationForBP(bpId));
    // heartbeat should trigger re-registration, and next block report should
    // not change registration.
    waitForHeartbeat(dn, dnd);
    assertTrue(dnd.isRegistered());
    newReg = dn.getDNRegistrationForBP(bpId);
    assertNotSame(lastReg, newReg);
    lastReg = newReg;
    assertTrue(waitForBlockReport(dn, dnd));
    assertTrue(dnd.isRegistered());
    assertSame(lastReg, dn.getDNRegistrationForBP(bpId));
    // registration doesn't change.
    ExtendedBlock eb = new ExtendedBlock(bpId, 1234);
    dn.notifyNamenodeDeletedBlock(eb, storage.getStorageID());
    DataNodeTestUtils.triggerDeletionReport(dn);
    assertTrue(dnd.isRegistered());
    assertSame(lastReg, dn.getDNRegistrationForBP(bpId));
    // a failed IBR will effectively unregister the node.
    boolean failed = false;
    try {
        // pass null to cause a failure since there aren't any easy failure
        // modes since it shouldn't happen.
        fsn.processIncrementalBlockReport(lastReg, null);
    } catch (NullPointerException npe) {
        failed = true;
    }
    assertTrue("didn't fail", failed);
    assertFalse(dnd.isRegistered());
    // should remain unregistered until next heartbeat.
    dn.notifyNamenodeDeletedBlock(eb, storage.getStorageID());
    DataNodeTestUtils.triggerDeletionReport(dn);
    assertFalse(dnd.isRegistered());
    assertSame(lastReg, dn.getDNRegistrationForBP(bpId));
    waitForHeartbeat(dn, dnd);
    assertTrue(dnd.isRegistered());
    assertNotSame(lastReg, dn.getDNRegistrationForBP(bpId));
}

Example 73 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestDecommission method nodeUsageVerification.

@SuppressWarnings({ "unchecked" })
public void nodeUsageVerification(int numDatanodes, long[] nodesCapacity, AdminStates decommissionState) throws IOException, InterruptedException {
    Map<String, Map<String, String>> usage = null;
    DatanodeInfo decommissionedNodeInfo = null;
    String zeroNodeUsage = "0.00%";
    getConf().setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
    getConf().setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
    getConf().setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1);
    FileSystem fileSys = null;
    Path file1 = new Path("testNodeUsage.dat");
    try {
        SimulatedFSDataset.setFactory(getConf());
        startCluster(1, numDatanodes, false, nodesCapacity, false);
        ArrayList<ArrayList<DatanodeInfo>> namenodeDecomList = new ArrayList<ArrayList<DatanodeInfo>>(1);
        namenodeDecomList.add(0, new ArrayList<DatanodeInfo>(numDatanodes));
        if (decommissionState == AdminStates.DECOMMISSIONED) {
            // Move datanode1 to Decommissioned state
            ArrayList<DatanodeInfo> decommissionedNode = namenodeDecomList.get(0);
            decommissionedNodeInfo = takeNodeOutofService(0, null, 0, decommissionedNode, decommissionState);
        }
        // Write a file(replica 1).Hence will be written to only one live node.
        fileSys = getCluster().getFileSystem(0);
        FSNamesystem ns = getCluster().getNamesystem(0);
        writeFile(fileSys, file1, 1);
        Thread.sleep(2000);
        // min NodeUsage should not be 0.00%
        usage = (Map<String, Map<String, String>>) JSON.parse(ns.getNodeUsage());
        String minUsageBeforeDecom = usage.get("nodeUsage").get("min");
        assertTrue(!minUsageBeforeDecom.equalsIgnoreCase(zeroNodeUsage));
        if (decommissionState == AdminStates.DECOMMISSION_INPROGRESS) {
            // Start decommissioning datanode
            ArrayList<DatanodeInfo> decommissioningNodes = namenodeDecomList.get(0);
            decommissionedNodeInfo = takeNodeOutofService(0, null, 0, decommissioningNodes, decommissionState);
            // NodeUsage should not include DECOMMISSION_INPROGRESS node
            // (minUsage should be 0.00%)
            usage = (Map<String, Map<String, String>>) JSON.parse(ns.getNodeUsage());
            assertTrue(usage.get("nodeUsage").get("min").equalsIgnoreCase(zeroNodeUsage));
        }
        // Recommission node
        putNodeInService(0, decommissionedNodeInfo);
        usage = (Map<String, Map<String, String>>) JSON.parse(ns.getNodeUsage());
        String nodeusageAfterRecommi = decommissionState == AdminStates.DECOMMISSION_INPROGRESS ? minUsageBeforeDecom : zeroNodeUsage;
        assertTrue(usage.get("nodeUsage").get("min").equalsIgnoreCase(nodeusageAfterRecommi));
    } finally {
        cleanupFile(fileSys, file1);
    }
}

Example 74 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestDecommission method testDecommission2.

/**
   * Tests decommission with replicas on the target datanode cannot be migrated
   * to other datanodes and satisfy the replication factor. Make sure the
   * datanode won't get stuck in decommissioning state.
   */
@Test(timeout = 360000)
public void testDecommission2() throws IOException {
    LOG.info("Starting test testDecommission");
    int numNamenodes = 1;
    int numDatanodes = 4;
    getConf().setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3);
    startCluster(numNamenodes, numDatanodes);
    ArrayList<ArrayList<DatanodeInfo>> namenodeDecomList = new ArrayList<ArrayList<DatanodeInfo>>(numNamenodes);
    namenodeDecomList.add(0, new ArrayList<DatanodeInfo>(numDatanodes));
    Path file1 = new Path("testDecommission2.dat");
    int replicas = 4;
    // Start decommissioning one namenode at a time
    ArrayList<DatanodeInfo> decommissionedNodes = namenodeDecomList.get(0);
    FileSystem fileSys = getCluster().getFileSystem(0);
    FSNamesystem ns = getCluster().getNamesystem(0);
    writeFile(fileSys, file1, replicas);
    int deadDecomissioned = ns.getNumDecomDeadDataNodes();
    int liveDecomissioned = ns.getNumDecomLiveDataNodes();
    // Decommission one node. Verify that node is decommissioned.
    DatanodeInfo decomNode = takeNodeOutofService(0, null, 0, decommissionedNodes, AdminStates.DECOMMISSIONED);
    decommissionedNodes.add(decomNode);
    assertEquals(deadDecomissioned, ns.getNumDecomDeadDataNodes());
    assertEquals(liveDecomissioned + 1, ns.getNumDecomLiveDataNodes());
    // Ensure decommissioned datanode is not automatically shutdown
    DFSClient client = getDfsClient(0);
    assertEquals("All datanodes must be alive", numDatanodes, client.datanodeReport(DatanodeReportType.LIVE).length);
    assertNull(checkFile(fileSys, file1, replicas, decomNode.getXferAddr(), numDatanodes));
    cleanupFile(fileSys, file1);
    // Restart the cluster and ensure recommissioned datanodes
    // are allowed to register with the namenode
    shutdownCluster();
    startCluster(1, 4);
}

Example 75 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestFileCreation method testFileCreationWithOverwrite.

/**
   * 1. Check the blocks of old file are cleaned after creating with overwrite
   * 2. Restart NN, check the file
   * 3. Save new checkpoint and restart NN, check the file
   */
@Test(timeout = 120000)
public void testFileCreationWithOverwrite() throws Exception {
    Configuration conf = new Configuration();
    conf.setInt("dfs.blocksize", blockSize);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    DistributedFileSystem dfs = cluster.getFileSystem();
    try {
        dfs.mkdirs(new Path("/foo/dir"));
        String file = "/foo/dir/file";
        Path filePath = new Path(file);
        // Case 1: Create file with overwrite, check the blocks of old file
        // are cleaned after creating with overwrite
        NameNode nn = cluster.getNameNode();
        FSNamesystem fsn = NameNodeAdapter.getNamesystem(nn);
        BlockManager bm = fsn.getBlockManager();
        FSDataOutputStream out = dfs.create(filePath);
        byte[] oldData = AppendTestUtil.randomBytes(seed, fileSize);
        try {
            out.write(oldData);
        } finally {
            out.close();
        }
        LocatedBlocks oldBlocks = NameNodeAdapter.getBlockLocations(nn, file, 0, fileSize);
        assertBlocks(bm, oldBlocks, true);
        out = dfs.create(filePath, true);
        byte[] newData = AppendTestUtil.randomBytes(seed, fileSize);
        try {
            out.write(newData);
        } finally {
            out.close();
        }
        dfs.deleteOnExit(filePath);
        LocatedBlocks newBlocks = NameNodeAdapter.getBlockLocations(nn, file, 0, fileSize);
        assertBlocks(bm, newBlocks, true);
        assertBlocks(bm, oldBlocks, false);
        FSDataInputStream in = dfs.open(filePath);
        byte[] result = null;
        try {
            result = readAll(in);
        } finally {
            in.close();
        }
        Assert.assertArrayEquals(newData, result);
        // Case 2: Restart NN, check the file
        cluster.restartNameNode();
        nn = cluster.getNameNode();
        in = dfs.open(filePath);
        try {
            result = readAll(in);
        } finally {
            in.close();
        }
        Assert.assertArrayEquals(newData, result);
        // Case 3: Save new checkpoint and restart NN, check the file
        NameNodeAdapter.enterSafeMode(nn, false);
        NameNodeAdapter.saveNamespace(nn);
        cluster.restartNameNode();
        nn = cluster.getNameNode();
        in = dfs.open(filePath);
        try {
            result = readAll(in);
        } finally {
            in.close();
        }
        Assert.assertArrayEquals(newData, result);
    } finally {
        if (dfs != null) {
            dfs.close();
        }
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}