Examples with FSNamesystem org.apache.hadoop.hdfs.server.namenode.FSNamesystem used on opensource projects

Example 51 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestPendingReconstruction method testPendingAndInvalidate.

/**
   * Test if BlockManager can correctly remove corresponding pending records
   * when a file is deleted
   *
   * @throws Exception
   */
@Test
public void testPendingAndInvalidate() throws Exception {
    final Configuration CONF = new HdfsConfiguration();
    CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
    CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, DFS_REPLICATION_INTERVAL);
    CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, DFS_REPLICATION_INTERVAL);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(DATANODE_COUNT).build();
    cluster.waitActive();
    FSNamesystem namesystem = cluster.getNamesystem();
    BlockManager bm = namesystem.getBlockManager();
    DistributedFileSystem fs = cluster.getFileSystem();
    try {
        // 1. create a file
        Path filePath = new Path("/tmp.txt");
        DFSTestUtil.createFile(fs, filePath, 1024, (short) 3, 0L);
        // 2. disable the heartbeats
        for (DataNode dn : cluster.getDataNodes()) {
            DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
        }
        // 3. mark a couple of blocks as corrupt
        LocatedBlock block = NameNodeAdapter.getBlockLocations(cluster.getNameNode(), filePath.toString(), 0, 1).get(0);
        cluster.getNamesystem().writeLock();
        try {
            bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0], "STORAGE_ID", "TEST");
            bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[1], "STORAGE_ID", "TEST");
        } finally {
            cluster.getNamesystem().writeUnlock();
        }
        BlockManagerTestUtil.computeAllPendingWork(bm);
        BlockManagerTestUtil.updateState(bm);
        assertEquals(bm.getPendingReconstructionBlocksCount(), 1L);
        BlockInfo storedBlock = bm.getStoredBlock(block.getBlock().getLocalBlock());
        assertEquals(bm.pendingReconstruction.getNumReplicas(storedBlock), 2);
        // 4. delete the file
        fs.delete(filePath, true);
        // retry at most 10 times, each time sleep for 1s. Note that 10s is much
        // less than the default pending record timeout (5~10min)
        int retries = 10;
        long pendingNum = bm.getPendingReconstructionBlocksCount();
        while (pendingNum != 0 && retries-- > 0) {
            // let NN do the deletion
            Thread.sleep(1000);
            BlockManagerTestUtil.updateState(bm);
            pendingNum = bm.getPendingReconstructionBlocksCount();
        }
        assertEquals(pendingNum, 0L);
    } finally {
        cluster.shutdown();
    }
}

Example 52 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestRBWBlockInvalidation method testBlockInvalidationWhenRBWReplicaMissedInDN.

/**
   * Test when a block's replica is removed from RBW folder in one of the
   * datanode, namenode should ask to invalidate that corrupted block and
   * schedule replication for one more replica for that under replicated block.
   */
@Test(timeout = 600000)
public void testBlockInvalidationWhenRBWReplicaMissedInDN() throws IOException, InterruptedException {
    // This test cannot pass on Windows due to file locking enforcement.  It will
    // reject the attempt to delete the block file from the RBW folder.
    assumeNotWindows();
    Configuration conf = new HdfsConfiguration();
    conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2);
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 300);
    conf.setLong(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    FSDataOutputStream out = null;
    try {
        final FSNamesystem namesystem = cluster.getNamesystem();
        FileSystem fs = cluster.getFileSystem();
        Path testPath = new Path("/tmp/TestRBWBlockInvalidation", "foo1");
        out = fs.create(testPath, (short) 2);
        out.writeBytes("HDFS-3157: " + testPath);
        out.hsync();
        cluster.startDataNodes(conf, 1, true, null, null, null);
        ExtendedBlock blk = DFSTestUtil.getFirstBlock(fs, testPath);
        // Delete partial block and its meta information from the RBW folder
        // of first datanode.
        MaterializedReplica replica = cluster.getMaterializedReplica(0, blk);
        replica.deleteData();
        replica.deleteMeta();
        out.close();
        int liveReplicas = 0;
        while (true) {
            if ((liveReplicas = countReplicas(namesystem, blk).liveReplicas()) < 2) {
                // This confirms we have a corrupt replica
                LOG.info("Live Replicas after corruption: " + liveReplicas);
                break;
            }
            Thread.sleep(100);
        }
        assertEquals("There should be less than 2 replicas in the " + "liveReplicasMap", 1, liveReplicas);
        while (true) {
            if ((liveReplicas = countReplicas(namesystem, blk).liveReplicas()) > 1) {
                //Wait till the live replica count becomes equal to Replication Factor
                LOG.info("Live Replicas after Rereplication: " + liveReplicas);
                break;
            }
            Thread.sleep(100);
        }
        assertEquals("There should be two live replicas", 2, liveReplicas);
        while (true) {
            Thread.sleep(100);
            if (countReplicas(namesystem, blk).corruptReplicas() == 0) {
                LOG.info("Corrupt Replicas becomes 0");
                break;
            }
        }
    } finally {
        if (out != null) {
            out.close();
        }
        cluster.shutdown();
    }
}

Example 53 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestReconstructStripedBlocksWithRackAwareness method testReconstructForNotEnoughRacks.

/**
   * When there are all the internal blocks available but they are not placed on
   * enough racks, NameNode should avoid normal decoding reconstruction but copy
   * an internal block to a new rack.
   *
   * In this test, we first need to create a scenario that a striped block has
   * all the internal blocks but distributed in <6 racks. Then we check if the
   * redundancy monitor can correctly schedule the reconstruction work for it.
   */
@Test
public void testReconstructForNotEnoughRacks() throws Exception {
    LOG.info("cluster hosts: {}, racks: {}", Arrays.asList(hosts), Arrays.asList(racks));
    conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, StripedFileTestUtil.getDefaultECPolicy().getName());
    cluster = new MiniDFSCluster.Builder(conf).racks(racks).hosts(hosts).numDataNodes(hosts.length).build();
    cluster.waitActive();
    fs = cluster.getFileSystem();
    fs.setErasureCodingPolicy(new Path("/"), StripedFileTestUtil.getDefaultECPolicy().getName());
    FSNamesystem fsn = cluster.getNamesystem();
    BlockManager bm = fsn.getBlockManager();
    MiniDFSCluster.DataNodeProperties lastHost = stopDataNode(hosts[hosts.length - 1]);
    final Path file = new Path("/foo");
    // the file's block is in 9 dn but 5 racks
    DFSTestUtil.createFile(fs, file, cellSize * dataBlocks * 2, (short) 1, 0L);
    Assert.assertEquals(0, bm.numOfUnderReplicatedBlocks());
    final INodeFile fileNode = fsn.getFSDirectory().getINode4Write(file.toString()).asFile();
    BlockInfoStriped blockInfo = (BlockInfoStriped) fileNode.getLastBlock();
    // we now should have 9 internal blocks distributed in 5 racks
    Set<String> rackSet = new HashSet<>();
    for (DatanodeStorageInfo storage : blockInfo.storages) {
        rackSet.add(storage.getDatanodeDescriptor().getNetworkLocation());
    }
    Assert.assertEquals(dataBlocks - 1, rackSet.size());
    // restart the stopped datanode
    cluster.restartDataNode(lastHost);
    cluster.waitActive();
    // make sure we have 6 racks again
    NetworkTopology topology = bm.getDatanodeManager().getNetworkTopology();
    Assert.assertEquals(hosts.length, topology.getNumOfLeaves());
    Assert.assertEquals(dataBlocks, topology.getNumOfRacks());
    // pause all the heartbeats
    for (DataNode dn : cluster.getDataNodes()) {
        DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
    }
    fsn.writeLock();
    try {
        bm.processMisReplicatedBlocks();
    } finally {
        fsn.writeUnlock();
    }
    // check if redundancy monitor correctly schedule the reconstruction work.
    boolean scheduled = false;
    for (int i = 0; i < 5; i++) {
        // retry 5 times
        for (DatanodeStorageInfo storage : blockInfo.storages) {
            if (storage != null) {
                DatanodeDescriptor dn = storage.getDatanodeDescriptor();
                Assert.assertEquals(0, dn.getNumberOfBlocksToBeErasureCoded());
                if (dn.getNumberOfBlocksToBeReplicated() == 1) {
                    scheduled = true;
                }
            }
        }
        if (scheduled) {
            break;
        }
        Thread.sleep(1000);
    }
    Assert.assertTrue(scheduled);
}

Example 54 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestReplicationPolicy method testAddStoredBlockDoesNotCauseSkippedReplication.

@Test(timeout = 60000)
public void testAddStoredBlockDoesNotCauseSkippedReplication() throws IOException {
    FSNamesystem mockNS = mock(FSNamesystem.class);
    when(mockNS.hasWriteLock()).thenReturn(true);
    when(mockNS.hasReadLock()).thenReturn(true);
    BlockManager bm = new BlockManager(mockNS, false, new HdfsConfiguration());
    LowRedundancyBlocks lowRedundancyBlocks = bm.neededReconstruction;
    BlockInfo block1 = genBlockInfo(ThreadLocalRandom.current().nextLong());
    BlockInfo block2 = genBlockInfo(ThreadLocalRandom.current().nextLong());
    // Adding QUEUE_LOW_REDUNDANCY block
    lowRedundancyBlocks.add(block1, 0, 0, 1, 1);
    // Adding QUEUE_LOW_REDUNDANCY block
    lowRedundancyBlocks.add(block2, 0, 0, 1, 1);
    List<List<BlockInfo>> chosenBlocks;
    // Choose 1 block from lowRedundancyBlocks. Then it should pick 1 block
    // from QUEUE_VERY_LOW_REDUNDANCY.
    chosenBlocks = lowRedundancyBlocks.chooseLowRedundancyBlocks(1);
    assertTheChosenBlocks(chosenBlocks, 1, 0, 0, 0, 0);
    // Adding this block collection to the BlockManager, so that when we add
    // block under construction, the BlockManager will realize the expected
    // replication has been achieved and remove it from the low redundancy
    // queue.
    BlockInfoContiguous info = new BlockInfoContiguous(block1, (short) 1);
    info.convertToBlockUnderConstruction(BlockUCState.UNDER_CONSTRUCTION, null);
    info.setBlockCollectionId(1000L);
    final INodeFile file = TestINodeFile.createINodeFile(1000L);
    when(mockNS.getBlockCollection(1000L)).thenReturn(file);
    bm.addBlockCollection(info, file);
    // Adding this block will increase its current replication, and that will
    // remove it from the queue.
    bm.addStoredBlockUnderConstruction(new StatefulBlockInfo(info, info, ReplicaState.FINALIZED), storages[0]);
    // Choose 1 block from UnderReplicatedBlocks. Then it should pick 1 block
    // from QUEUE_VERY_LOW_REDUNDANCY.
    // This block remains and should not be skipped over.
    chosenBlocks = lowRedundancyBlocks.chooseLowRedundancyBlocks(1);
    assertTheChosenBlocks(chosenBlocks, 1, 0, 0, 0, 0);
}

Example 55 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestNodeCount method testNodeCount.

@Test(timeout = 60000)
public void testNodeCount() throws Exception {
    final Configuration conf = new HdfsConfiguration();
    // avoid invalidation by startup delay in order to make test non-transient
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY, 60);
    // reduce intervals to make test execution time shorter
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
    conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
    // start a mini dfs cluster of 2 nodes
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION_FACTOR).build();
    try {
        final FSNamesystem namesystem = cluster.getNamesystem();
        final BlockManager bm = namesystem.getBlockManager();
        final HeartbeatManager hm = bm.getDatanodeManager().getHeartbeatManager();
        final FileSystem fs = cluster.getFileSystem();
        // populate the cluster with a one block file
        final Path FILE_PATH = new Path("/testfile");
        DFSTestUtil.createFile(fs, FILE_PATH, 1L, REPLICATION_FACTOR, 1L);
        DFSTestUtil.waitReplication(fs, FILE_PATH, REPLICATION_FACTOR);
        ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, FILE_PATH);
        // keep a copy of all datanode descriptor
        final DatanodeDescriptor[] datanodes = hm.getDatanodes();
        // start two new nodes
        cluster.startDataNodes(conf, 2, true, null, null);
        cluster.waitActive();
        // bring down first datanode
        DatanodeDescriptor datanode = datanodes[0];
        DataNodeProperties dnprop = cluster.stopDataNode(datanode.getXferAddr());
        // make sure that NN detects that the datanode is down
        BlockManagerTestUtil.noticeDeadDatanode(cluster.getNameNode(), datanode.getXferAddr());
        // the block will be replicated
        DFSTestUtil.waitReplication(fs, FILE_PATH, REPLICATION_FACTOR);
        // restart the first datanode
        cluster.restartDataNode(dnprop);
        cluster.waitActive();
        // check if excessive replica is detected
        initializeTimeout(TIMEOUT);
        while (countNodes(block.getLocalBlock(), namesystem).excessReplicas() == 0) {
            checkTimeout("excess replicas not detected");
        }
        // find out a non-excess node
        DatanodeDescriptor nonExcessDN = null;
        for (DatanodeStorageInfo storage : bm.blocksMap.getStorages(block.getLocalBlock())) {
            final DatanodeDescriptor dn = storage.getDatanodeDescriptor();
            final BlockInfo info = new BlockInfoContiguous(block.getLocalBlock(), (short) 0);
            if (!bm.isExcess(dn, info)) {
                nonExcessDN = dn;
                break;
            }
        }
        assertTrue(nonExcessDN != null);
        // bring down non excessive datanode
        dnprop = cluster.stopDataNode(nonExcessDN.getXferAddr());
        // make sure that NN detects that the datanode is down
        BlockManagerTestUtil.noticeDeadDatanode(cluster.getNameNode(), nonExcessDN.getXferAddr());
        // The block should be replicated
        initializeTimeout(TIMEOUT);
        while (countNodes(block.getLocalBlock(), namesystem).liveReplicas() != REPLICATION_FACTOR) {
            checkTimeout("live replica count not correct", 1000);
        }
        // restart the first datanode
        cluster.restartDataNode(dnprop);
        cluster.waitActive();
        // check if excessive replica is detected
        initializeTimeout(TIMEOUT);
        while (countNodes(block.getLocalBlock(), namesystem).excessReplicas() != 2) {
            checkTimeout("excess replica count not equal to 2");
        }
    } finally {
        cluster.shutdown();
    }
}