Examples with DatanodeDescriptor org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor used on opensource projects

Example 6 with DatanodeDescriptor

use of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor in project hadoop by apache.

the class TestDFSOutputStream method testNoLocalWriteFlag.

@Test
public void testNoLocalWriteFlag() throws IOException {
    DistributedFileSystem fs = cluster.getFileSystem();
    EnumSet<CreateFlag> flags = EnumSet.of(CreateFlag.NO_LOCAL_WRITE, CreateFlag.CREATE);
    BlockManager bm = cluster.getNameNode().getNamesystem().getBlockManager();
    DatanodeManager dm = bm.getDatanodeManager();
    try (FSDataOutputStream os = fs.create(new Path("/test-no-local"), FsPermission.getDefault(), flags, 512, (short) 2, 512, null)) {
        // Inject a DatanodeManager that returns one DataNode as local node for
        // the client.
        DatanodeManager spyDm = spy(dm);
        DatanodeDescriptor dn1 = dm.getDatanodeListForReport(HdfsConstants.DatanodeReportType.LIVE).get(0);
        doReturn(dn1).when(spyDm).getDatanodeByHost("127.0.0.1");
        Whitebox.setInternalState(bm, "datanodeManager", spyDm);
        byte[] buf = new byte[512 * 16];
        new Random().nextBytes(buf);
        os.write(buf);
    } finally {
        Whitebox.setInternalState(bm, "datanodeManager", dm);
    }
    cluster.triggerBlockReports();
    final String bpid = cluster.getNamesystem().getBlockPoolId();
    // Total number of DataNodes is 3.
    assertEquals(3, cluster.getAllBlockReports(bpid).size());
    int numDataNodesWithData = 0;
    for (Map<DatanodeStorage, BlockListAsLongs> dnBlocks : cluster.getAllBlockReports(bpid)) {
        for (BlockListAsLongs blocks : dnBlocks.values()) {
            if (blocks.getNumberOfBlocks() > 0) {
                numDataNodesWithData++;
                break;
            }
        }
    }
    // Verify that only one DN has no data.
    assertEquals(1, 3 - numDataNodesWithData);
}

Example 7 with DatanodeDescriptor

use of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor in project hadoop by apache.

the class TestAddStripedBlocks method testBlockScheduledUpdate.

/**
   * Check if the scheduled block size on each DN storage is correctly updated
   */
@Test
public void testBlockScheduledUpdate() throws Exception {
    final FSNamesystem fsn = cluster.getNamesystem();
    final Path foo = new Path("/foo");
    try (FSDataOutputStream out = dfs.create(foo, true)) {
        DFSStripedOutputStream sout = (DFSStripedOutputStream) out.getWrappedStream();
        writeAndFlushStripedOutputStream(sout, DFS_BYTES_PER_CHECKSUM_DEFAULT);
        // make sure the scheduled block size has been updated for each DN storage
        // in NN
        final List<DatanodeDescriptor> dnList = new ArrayList<>();
        fsn.getBlockManager().getDatanodeManager().fetchDatanodes(dnList, null, false);
        for (DatanodeDescriptor dn : dnList) {
            Assert.assertEquals(1, dn.getBlocksScheduled());
        }
    }
    // we have completed the file, force the DN to flush IBR
    for (DataNode dn : cluster.getDataNodes()) {
        DataNodeTestUtils.triggerBlockReport(dn);
    }
    // check the scheduled block size again
    final List<DatanodeDescriptor> dnList = new ArrayList<>();
    fsn.getBlockManager().getDatanodeManager().fetchDatanodes(dnList, null, false);
    for (DatanodeDescriptor dn : dnList) {
        Assert.assertEquals(0, dn.getBlocksScheduled());
    }
}

Example 8 with DatanodeDescriptor

use of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor in project hadoop by apache.

the class TestReconstructStripedBlocks method getNumberOfBlocksToBeErasureCoded.

private static int getNumberOfBlocksToBeErasureCoded(MiniDFSCluster cluster) throws Exception {
    DatanodeManager dm = cluster.getNamesystem().getBlockManager().getDatanodeManager();
    int count = 0;
    for (DataNode dn : cluster.getDataNodes()) {
        DatanodeDescriptor dd = dm.getDatanode(dn.getDatanodeId());
        count += dd.getNumberOfBlocksToBeErasureCoded();
    }
    return count;
}

Example 9 with DatanodeDescriptor

use of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor in project hadoop by apache.

the class TestNamenodeCapacityReport method testXceiverCountInternal.

public void testXceiverCountInternal(int minMaintenanceR) throws Exception {
    Configuration conf = new HdfsConfiguration();
    // retry one time, if close fails
    conf.setInt(HdfsClientConfigKeys.BlockWrite.LOCATEFOLLOWINGBLOCK_RETRIES_KEY, 1);
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAINTENANCE_REPLICATION_MIN_KEY, minMaintenanceR);
    MiniDFSCluster cluster = null;
    final int nodes = 8;
    final int fileCount = 5;
    final short fileRepl = 3;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(nodes).build();
        cluster.waitActive();
        final FSNamesystem namesystem = cluster.getNamesystem();
        final DatanodeManager dnm = namesystem.getBlockManager().getDatanodeManager();
        List<DataNode> datanodes = cluster.getDataNodes();
        final DistributedFileSystem fs = cluster.getFileSystem();
        // trigger heartbeats in case not already sent
        triggerHeartbeats(datanodes);
        // check that all nodes are live and in service
        // xceiver server adds 1 to load
        int expectedTotalLoad = nodes;
        int expectedInServiceNodes = nodes;
        int expectedInServiceLoad = nodes;
        checkClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes, expectedInServiceLoad);
        // Ensure counts are accurate.
        for (int i = 0; i < nodes / 2; i++) {
            DataNode dn = datanodes.get(i);
            DatanodeDescriptor dnd = dnm.getDatanode(dn.getDatanodeId());
            dn.shutdown();
            DFSTestUtil.setDatanodeDead(dnd);
            BlockManagerTestUtil.checkHeartbeat(namesystem.getBlockManager());
            //Admin operations on dead nodes won't impact nodesInService metrics.
            startDecommissionOrMaintenance(dnm, dnd, (i % 2 == 0));
            expectedInServiceNodes--;
            assertEquals(expectedInServiceNodes, namesystem.getNumLiveDataNodes());
            assertEquals(expectedInServiceNodes, getNumDNInService(namesystem));
            stopDecommissionOrMaintenance(dnm, dnd, (i % 2 == 0));
            assertEquals(expectedInServiceNodes, getNumDNInService(namesystem));
        }
        // restart the nodes to verify that counts are correct after
        // node re-registration 
        cluster.restartDataNodes();
        cluster.waitActive();
        datanodes = cluster.getDataNodes();
        expectedInServiceNodes = nodes;
        assertEquals(nodes, datanodes.size());
        checkClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes, expectedInServiceLoad);
        // create streams and hsync to force datastreamers to start
        DFSOutputStream[] streams = new DFSOutputStream[fileCount];
        for (int i = 0; i < fileCount; i++) {
            streams[i] = (DFSOutputStream) fs.create(new Path("/f" + i), fileRepl).getWrappedStream();
            streams[i].write("1".getBytes());
            streams[i].hsync();
            // the load for writers is 2 because both the write xceiver & packet
            // responder threads are counted in the load
            expectedTotalLoad += 2 * fileRepl;
            expectedInServiceLoad += 2 * fileRepl;
        }
        // force nodes to send load update
        triggerHeartbeats(datanodes);
        checkClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes, expectedInServiceLoad);
        // expected load, trigger heartbeat to force load update.
        for (int i = 0; i < fileRepl; i++) {
            expectedInServiceNodes--;
            DatanodeDescriptor dnd = dnm.getDatanode(datanodes.get(i).getDatanodeId());
            expectedInServiceLoad -= dnd.getXceiverCount();
            startDecommissionOrMaintenance(dnm, dnd, (i % 2 == 0));
            DataNodeTestUtils.triggerHeartbeat(datanodes.get(i));
            Thread.sleep(100);
            checkClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes, expectedInServiceLoad);
        }
        // load based on whether the nodes in the pipeline are decomm
        for (int i = 0; i < fileCount; i++) {
            int adminOps = 0;
            for (DatanodeInfo dni : streams[i].getPipeline()) {
                DatanodeDescriptor dnd = dnm.getDatanode(dni);
                expectedTotalLoad -= 2;
                if (!dnd.isInService()) {
                    adminOps++;
                } else {
                    expectedInServiceLoad -= 2;
                }
            }
            try {
                streams[i].close();
            } catch (IOException ioe) {
                // bug for now
                if (adminOps < fileRepl) {
                    throw ioe;
                }
            }
            triggerHeartbeats(datanodes);
            // verify node count and loads 
            checkClusterHealth(nodes, namesystem, expectedTotalLoad, expectedInServiceNodes, expectedInServiceLoad);
        }
        // shutdown each node, verify node counts based on admin state
        for (int i = 0; i < nodes; i++) {
            DataNode dn = datanodes.get(i);
            dn.shutdown();
            // force it to appear dead so live count decreases
            DatanodeDescriptor dnDesc = dnm.getDatanode(dn.getDatanodeId());
            DFSTestUtil.setDatanodeDead(dnDesc);
            BlockManagerTestUtil.checkHeartbeat(namesystem.getBlockManager());
            assertEquals(nodes - 1 - i, namesystem.getNumLiveDataNodes());
            // first few nodes are already out of service
            if (i >= fileRepl) {
                expectedInServiceNodes--;
            }
            assertEquals(expectedInServiceNodes, getNumDNInService(namesystem));
            // live nodes always report load of 1.  no nodes is load 0
            double expectedXceiverAvg = (i == nodes - 1) ? 0.0 : 1.0;
            assertEquals((double) expectedXceiverAvg, getInServiceXceiverAverage(namesystem), EPSILON);
        }
        // final sanity check
        checkClusterHealth(0, namesystem, 0.0, 0, 0.0);
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 10 with DatanodeDescriptor

use of org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor in project hadoop by apache.

the class TestNamenodeCapacityReport method testVolumeSize.

/**
   * The following test first creates a file.
   * It verifies the block information from a datanode.
   * Then, it updates the block with new information and verifies again. 
   */
@Test
public void testVolumeSize() throws Exception {
    Configuration conf = new HdfsConfiguration();
    MiniDFSCluster cluster = null;
    // Set aside fifth of the total capacity as reserved
    long reserved = 10000;
    conf.setLong(DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY, reserved);
    try {
        cluster = new MiniDFSCluster.Builder(conf).build();
        cluster.waitActive();
        final FSNamesystem namesystem = cluster.getNamesystem();
        final DatanodeManager dm = cluster.getNamesystem().getBlockManager().getDatanodeManager();
        // Ensure the data reported for each data node is right
        final List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
        final List<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
        dm.fetchDatanodes(live, dead, false);
        assertTrue(live.size() == 1);
        long used, remaining, configCapacity, nonDFSUsed, bpUsed;
        float percentUsed, percentRemaining, percentBpUsed;
        for (final DatanodeDescriptor datanode : live) {
            used = datanode.getDfsUsed();
            remaining = datanode.getRemaining();
            nonDFSUsed = datanode.getNonDfsUsed();
            configCapacity = datanode.getCapacity();
            percentUsed = datanode.getDfsUsedPercent();
            percentRemaining = datanode.getRemainingPercent();
            bpUsed = datanode.getBlockPoolUsed();
            percentBpUsed = datanode.getBlockPoolUsedPercent();
            LOG.info("Datanode configCapacity " + configCapacity + " used " + used + " non DFS used " + nonDFSUsed + " remaining " + remaining + " perentUsed " + percentUsed + " percentRemaining " + percentRemaining);
            // There will be 5% space reserved in ext filesystem which is not
            // considered.
            assertTrue(configCapacity >= (used + remaining + nonDFSUsed));
            assertTrue(percentUsed == DFSUtilClient.getPercentUsed(used, configCapacity));
            assertTrue(percentRemaining == DFSUtilClient.getPercentRemaining(remaining, configCapacity));
            assertTrue(percentBpUsed == DFSUtilClient.getPercentUsed(bpUsed, configCapacity));
        }
        //
        // Currently two data directories are created by the data node
        // in the MiniDFSCluster. This results in each data directory having
        // capacity equals to the disk capacity of the data directory.
        // Hence the capacity reported by the data node is twice the disk space
        // the disk capacity
        //
        // So multiply the disk capacity and reserved space by two 
        // for accommodating it
        //
        final FsDatasetTestUtils utils = cluster.getFsDatasetTestUtils(0);
        int numOfDataDirs = utils.getDefaultNumOfDataDirs();
        long diskCapacity = numOfDataDirs * utils.getRawCapacity();
        reserved *= numOfDataDirs;
        configCapacity = namesystem.getCapacityTotal();
        used = namesystem.getCapacityUsed();
        nonDFSUsed = namesystem.getNonDfsUsedSpace();
        remaining = namesystem.getCapacityRemaining();
        percentUsed = namesystem.getPercentUsed();
        percentRemaining = namesystem.getPercentRemaining();
        bpUsed = namesystem.getBlockPoolUsedSpace();
        percentBpUsed = namesystem.getPercentBlockPoolUsed();
        LOG.info("Data node directory " + cluster.getDataDirectory());
        LOG.info("Name node diskCapacity " + diskCapacity + " configCapacity " + configCapacity + " reserved " + reserved + " used " + used + " remaining " + remaining + " nonDFSUsed " + nonDFSUsed + " remaining " + remaining + " percentUsed " + percentUsed + " percentRemaining " + percentRemaining + " bpUsed " + bpUsed + " percentBpUsed " + percentBpUsed);
        // Ensure new total capacity reported excludes the reserved space
        assertTrue(configCapacity == diskCapacity - reserved);
        // Ensure new total capacity reported excludes the reserved space
        // There will be 5% space reserved in ext filesystem which is not
        // considered.
        assertTrue(configCapacity >= (used + remaining + nonDFSUsed));
        // Ensure percent used is calculated based on used and present capacity
        assertTrue(percentUsed == DFSUtilClient.getPercentUsed(used, configCapacity));
        // Ensure percent used is calculated based on used and present capacity
        assertTrue(percentBpUsed == DFSUtilClient.getPercentUsed(bpUsed, configCapacity));
        // Ensure percent used is calculated based on used and present capacity
        assertTrue(percentRemaining == ((float) remaining * 100.0f) / (float) configCapacity);
        //Adding testcase for non-dfs used where we need to consider
        // reserved replica also.
        final int fileCount = 5;
        final DistributedFileSystem fs = cluster.getFileSystem();
        // create streams and hsync to force datastreamers to start
        DFSOutputStream[] streams = new DFSOutputStream[fileCount];
        for (int i = 0; i < fileCount; i++) {
            streams[i] = (DFSOutputStream) fs.create(new Path("/f" + i)).getWrappedStream();
            streams[i].write("1".getBytes());
            streams[i].hsync();
        }
        triggerHeartbeats(cluster.getDataNodes());
        assertTrue(configCapacity > (namesystem.getCapacityUsed() + namesystem.getCapacityRemaining() + namesystem.getNonDfsUsedSpace()));
        // There is a chance that nonDFS usage might have slightly due to
        // testlogs, So assume 1MB other files used within this gap
        assertTrue((namesystem.getCapacityUsed() + namesystem.getCapacityRemaining() + namesystem.getNonDfsUsedSpace() + fileCount * fs.getDefaultBlockSize()) - configCapacity < 1 * 1024);
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}