Examples with DataNode org.apache.hadoop.hdfs.server.datanode.DataNode used on opensource projects

Example 51 with DataNode

use of org.apache.hadoop.hdfs.server.datanode.DataNode in project hadoop by apache.

the class TestBlocksWithNotEnoughRacks method testReplDueToNodeFailRespectsRackPolicy.

/*
   * Test that when a block is replicated because a replica is lost due
   * to host failure the the rack policy is preserved.
   */
@Test
public void testReplDueToNodeFailRespectsRackPolicy() throws Exception {
    Configuration conf = getConf();
    short REPLICATION_FACTOR = 3;
    final Path filePath = new Path("/testFile");
    // Last datanode is on a different rack
    String[] racks = { "/rack1", "/rack1", "/rack1", "/rack2", "/rack2" };
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(racks.length).racks(racks).build();
    final FSNamesystem ns = cluster.getNameNode().getNamesystem();
    final DatanodeManager dm = ns.getBlockManager().getDatanodeManager();
    try {
        // Create a file with one block with a replication factor of 3
        final FileSystem fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L);
        ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
        DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
        // Make the last datanode look like it failed to heartbeat by 
        // calling removeDatanode and stopping it.
        ArrayList<DataNode> datanodes = cluster.getDataNodes();
        int idx = datanodes.size() - 1;
        DataNode dataNode = datanodes.get(idx);
        DatanodeID dnId = dataNode.getDatanodeId();
        cluster.stopDataNode(idx);
        dm.removeDatanode(dnId);
        // The block should still have sufficient # replicas, across racks.
        // The last node may not have contained a replica, but if it did
        // it should have been replicated within the same rack.
        DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
        // Fail the last datanode again, it's also on rack2 so there is
        // only 1 rack for all the replicas
        datanodes = cluster.getDataNodes();
        idx = datanodes.size() - 1;
        dataNode = datanodes.get(idx);
        dnId = dataNode.getDatanodeId();
        cluster.stopDataNode(idx);
        dm.removeDatanode(dnId);
        // Make sure we have enough live replicas even though we are
        // short one rack. The cluster now has only 1 rack thus we just make sure
        // we still have 3 replicas.
        DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 0);
    } finally {
        cluster.shutdown();
    }
}

Example 52 with DataNode

use of org.apache.hadoop.hdfs.server.datanode.DataNode in project hadoop by apache.

the class TestNameNodePrunesMissingStorages method testNameNodePrunesUnreportedStorages.

@Test(timeout = 300000)
public void testNameNodePrunesUnreportedStorages() throws Exception {
    Configuration conf = new HdfsConfiguration();
    // Create a cluster with one datanode with two storages
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).storagesPerDatanode(2).build();
    // Create two files to ensure each storage has a block
    DFSTestUtil.createFile(cluster.getFileSystem(), new Path("file1"), 102400, 102400, 102400, (short) 1, 0x1BAD5EE);
    DFSTestUtil.createFile(cluster.getFileSystem(), new Path("file2"), 102400, 102400, 102400, (short) 1, 0x1BAD5EED);
    // Get the datanode storages and data directories
    DataNode dn = cluster.getDataNodes().get(0);
    BlockManager bm = cluster.getNameNode().getNamesystem().getBlockManager();
    DatanodeDescriptor dnDescriptor = bm.getDatanodeManager().getDatanode(cluster.getDataNodes().get(0).getDatanodeUuid());
    DatanodeStorageInfo[] dnStoragesInfosBeforeRestart = dnDescriptor.getStorageInfos();
    Collection<String> oldDirs = new ArrayList<String>(dn.getConf().getTrimmedStringCollection(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY));
    // Keep the first data directory and remove the second.
    String newDirs = oldDirs.iterator().next();
    conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, newDirs);
    // Restart the datanode with the new conf
    cluster.stopDataNode(0);
    cluster.startDataNodes(conf, 1, false, null, null);
    dn = cluster.getDataNodes().get(0);
    cluster.waitActive();
    // Assert that the dnDescriptor has both the storages after restart
    assertArrayEquals(dnStoragesInfosBeforeRestart, dnDescriptor.getStorageInfos());
    // Assert that the removed storage is marked as FAILED
    // when DN heartbeats to the NN
    int numFailedStoragesWithBlocks = 0;
    DatanodeStorageInfo failedStorageInfo = null;
    for (DatanodeStorageInfo dnStorageInfo : dnDescriptor.getStorageInfos()) {
        if (dnStorageInfo.areBlocksOnFailedStorage()) {
            numFailedStoragesWithBlocks++;
            failedStorageInfo = dnStorageInfo;
        }
    }
    assertEquals(1, numFailedStoragesWithBlocks);
    // Heartbeat manager removes the blocks associated with this failed storage
    bm.getDatanodeManager().getHeartbeatManager().heartbeatCheck();
    assertTrue(!failedStorageInfo.areBlocksOnFailedStorage());
    // pruneStorageMap removes the unreported storage
    cluster.triggerHeartbeats();
    // Assert that the unreported storage is pruned
    assertEquals(DataNode.getStorageLocations(dn.getConf()).size(), dnDescriptor.getStorageInfos().length);
}

Example 53 with DataNode

use of org.apache.hadoop.hdfs.server.datanode.DataNode in project hadoop by apache.

the class TestDeadDatanode method testDeadDatanode.

/**
   * Test to ensure namenode rejects request from dead datanode
   * - Start a cluster
   * - Shutdown the datanode and wait for it to be marked dead at the namenode
   * - Send datanode requests to Namenode and make sure it is rejected 
   *   appropriately.
   */
@Test
public void testDeadDatanode() throws Exception {
    Configuration conf = new HdfsConfiguration();
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500);
    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
    cluster = new MiniDFSCluster.Builder(conf).build();
    cluster.waitActive();
    String poolId = cluster.getNamesystem().getBlockPoolId();
    // wait for datanode to be marked live
    DataNode dn = cluster.getDataNodes().get(0);
    DatanodeRegistration reg = InternalDataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId);
    DFSTestUtil.waitForDatanodeState(cluster, reg.getDatanodeUuid(), true, 20000);
    // Shutdown and wait for datanode to be marked dead
    dn.shutdown();
    DFSTestUtil.waitForDatanodeState(cluster, reg.getDatanodeUuid(), false, 20000);
    DatanodeProtocol dnp = cluster.getNameNodeRpc();
    ReceivedDeletedBlockInfo[] blocks = { new ReceivedDeletedBlockInfo(new Block(0), ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) };
    StorageReceivedDeletedBlocks[] storageBlocks = { new StorageReceivedDeletedBlocks(new DatanodeStorage(reg.getDatanodeUuid()), blocks) };
    // Ensure blockReceived call from dead datanode is not rejected with
    // IOException, since it's async, but the node remains unregistered.
    dnp.blockReceivedAndDeleted(reg, poolId, storageBlocks);
    BlockManager bm = cluster.getNamesystem().getBlockManager();
    // IBRs are async, make sure the NN processes all of them.
    bm.flushBlockOps();
    assertFalse(bm.getDatanodeManager().getDatanode(reg).isRegistered());
    // Ensure blockReport from dead datanode is rejected with IOException
    StorageBlockReport[] report = { new StorageBlockReport(new DatanodeStorage(reg.getDatanodeUuid()), BlockListAsLongs.EMPTY) };
    try {
        dnp.blockReport(reg, poolId, report, new BlockReportContext(1, 0, System.nanoTime(), 0L, true));
        fail("Expected IOException is not thrown");
    } catch (IOException ex) {
    // Expected
    }
    // Ensure heartbeat from dead datanode is rejected with a command
    // that asks datanode to register again
    StorageReport[] rep = { new StorageReport(new DatanodeStorage(reg.getDatanodeUuid()), false, 0, 0, 0, 0, 0) };
    DatanodeCommand[] cmd = dnp.sendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null, true, SlowPeerReports.EMPTY_REPORT).getCommands();
    assertEquals(1, cmd.length);
    assertEquals(cmd[0].getAction(), RegisterCommand.REGISTER.getAction());
}

Example 54 with DataNode

use of org.apache.hadoop.hdfs.server.datanode.DataNode in project hadoop by apache.

the class TestDeadDatanode method testDeadNodeAsBlockTarget.

@Test
public void testDeadNodeAsBlockTarget() throws Exception {
    Configuration conf = new HdfsConfiguration();
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500);
    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    cluster.waitActive();
    String poolId = cluster.getNamesystem().getBlockPoolId();
    // wait for datanode to be marked live
    DataNode dn = cluster.getDataNodes().get(0);
    DatanodeRegistration reg = InternalDataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId);
    // Get the updated datanode descriptor
    BlockManager bm = cluster.getNamesystem().getBlockManager();
    DatanodeManager dm = bm.getDatanodeManager();
    Node clientNode = dm.getDatanode(reg);
    DFSTestUtil.waitForDatanodeState(cluster, reg.getDatanodeUuid(), true, 20000);
    // Shutdown and wait for datanode to be marked dead
    dn.shutdown();
    DFSTestUtil.waitForDatanodeState(cluster, reg.getDatanodeUuid(), false, 20000);
    // Get the updated datanode descriptor available in DNM
    // choose the targets, but local node should not get selected as this is not
    // part of the cluster anymore
    DatanodeStorageInfo[] results = bm.chooseTarget4NewBlock("/hello", 3, clientNode, new HashSet<Node>(), 256 * 1024 * 1024L, null, (byte) 7, BlockType.CONTIGUOUS, null);
    for (DatanodeStorageInfo datanodeStorageInfo : results) {
        assertFalse("Dead node should not be choosen", datanodeStorageInfo.getDatanodeDescriptor().equals(clientNode));
    }
}

Example 55 with DataNode

use of org.apache.hadoop.hdfs.server.datanode.DataNode in project hadoop by apache.

the class TestCacheDirectives method checkPendingCachedEmpty.

/**
   * Check that the NameNode is not attempting to cache anything.
   */
private void checkPendingCachedEmpty(MiniDFSCluster cluster) throws Exception {
    Thread.sleep(1000);
    cluster.getNamesystem().readLock();
    try {
        final DatanodeManager datanodeManager = cluster.getNamesystem().getBlockManager().getDatanodeManager();
        for (DataNode dn : cluster.getDataNodes()) {
            DatanodeDescriptor descriptor = datanodeManager.getDatanode(dn.getDatanodeId());
            Assert.assertTrue("Pending cached list of " + descriptor + " is not empty, " + Arrays.toString(descriptor.getPendingCached().toArray()), descriptor.getPendingCached().isEmpty());
        }
    } finally {
        cluster.getNamesystem().readUnlock();
    }
}