Examples with DataNode org.apache.hadoop.hdfs.server.datanode.DataNode used on opensource projects

Example 46 with DataNode

use of org.apache.hadoop.hdfs.server.datanode.DataNode in project hadoop by apache.

the class TestPipelines method pipeline_01.

/**
   * Creates and closes a file of certain length.
   * Calls append to allow next write() operation to add to the end of it
   * After write() invocation, calls hflush() to make sure that data sunk through
   * the pipeline and check the state of the last block's replica.
   * It supposes to be in RBW state
   *
   * @throws IOException in case of an error
   */
@Test
public void pipeline_01() throws IOException {
    final String METHOD_NAME = GenericTestUtils.getMethodName();
    if (LOG.isDebugEnabled()) {
        LOG.debug("Running " + METHOD_NAME);
    }
    Path filePath = new Path("/" + METHOD_NAME + ".dat");
    DFSTestUtil.createFile(fs, filePath, FILE_SIZE, REPL_FACTOR, rand.nextLong());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Invoking append but doing nothing otherwise...");
    }
    FSDataOutputStream ofs = fs.append(filePath);
    ofs.writeBytes("Some more stuff to write");
    ((DFSOutputStream) ofs.getWrappedStream()).hflush();
    List<LocatedBlock> lb = cluster.getNameNodeRpc().getBlockLocations(filePath.toString(), FILE_SIZE - 1, FILE_SIZE).getLocatedBlocks();
    for (DataNode dn : cluster.getDataNodes()) {
        Replica r = cluster.getFsDatasetTestUtils(dn).fetchReplica(lb.get(0).getBlock());
        assertTrue("Replica on DN " + dn + " shouldn't be null", r != null);
        assertEquals("Should be RBW replica on " + dn + " after sequence of calls append()/write()/hflush()", HdfsServerConstants.ReplicaState.RBW, r.getState());
    }
    ofs.close();
}

Example 47 with DataNode

use of org.apache.hadoop.hdfs.server.datanode.DataNode in project hadoop by apache.

the class TestMaintenanceState method testFileCloseAfterEnteringMaintenance.

@Test(timeout = 120000)
public void testFileCloseAfterEnteringMaintenance() throws Exception {
    LOG.info("Starting testFileCloseAfterEnteringMaintenance");
    int expirationInMs = 30 * 1000;
    int numDataNodes = 3;
    int numNameNodes = 1;
    getConf().setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, 2);
    startCluster(numNameNodes, numDataNodes);
    getCluster().waitActive();
    FSNamesystem fsn = getCluster().getNameNode().getNamesystem();
    List<String> hosts = new ArrayList<>();
    for (DataNode dn : getCluster().getDataNodes()) {
        hosts.add(dn.getDisplayName());
        putNodeInService(0, dn.getDatanodeUuid());
    }
    assertEquals(numDataNodes, fsn.getNumLiveDataNodes());
    Path openFile = new Path("/testClosingFileInMaintenance.dat");
    // Lets write 2 blocks of data to the openFile
    writeFile(getCluster().getFileSystem(), openFile, (short) 3);
    // Lets write some more data and keep the file open
    FSDataOutputStream fsDataOutputStream = getCluster().getFileSystem().append(openFile);
    byte[] bytes = new byte[1024];
    fsDataOutputStream.write(bytes);
    fsDataOutputStream.hsync();
    LocatedBlocks lbs = NameNodeAdapter.getBlockLocations(getCluster().getNameNode(0), openFile.toString(), 0, 3 * blockSize);
    DatanodeInfo[] dnInfos4LastBlock = lbs.getLastLocatedBlock().getLocations();
    // Request maintenance for DataNodes 1 and 2 which has the last block.
    takeNodeOutofService(0, Lists.newArrayList(dnInfos4LastBlock[0].getDatanodeUuid(), dnInfos4LastBlock[1].getDatanodeUuid()), Time.now() + expirationInMs, null, null, AdminStates.ENTERING_MAINTENANCE);
    // Closing the file should succeed even when the
    // last blocks' nodes are entering maintenance.
    fsDataOutputStream.close();
    cleanupFile(getCluster().getFileSystem(), openFile);
}

Example 48 with DataNode

use of org.apache.hadoop.hdfs.server.datanode.DataNode in project hadoop by apache.

the class TestMiniDFSCluster method testClusterSetDatanodeDifferentStorageType.

@Test
public void testClusterSetDatanodeDifferentStorageType() throws IOException {
    final Configuration conf = new HdfsConfiguration();
    StorageType[][] storageType = new StorageType[][] { { StorageType.DISK, StorageType.ARCHIVE }, { StorageType.DISK }, { StorageType.ARCHIVE } };
    try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).storageTypes(storageType).build()) {
        cluster.waitActive();
        ArrayList<DataNode> dataNodes = cluster.getDataNodes();
        // Check the number of directory in DN's
        for (int i = 0; i < storageType.length; i++) {
            assertEquals(DataNode.getStorageLocations(dataNodes.get(i).getConf()).size(), storageType[i].length);
        }
    }
}

Example 49 with DataNode

use of org.apache.hadoop.hdfs.server.datanode.DataNode in project hadoop by apache.

the class TestReadStripedFileWithDecoding method testReportBadBlock.

/**
   * After reading a corrupted block, make sure the client can correctly report
   * the corruption to the NameNode.
   */
@Test
public void testReportBadBlock() throws IOException {
    // create file
    final Path file = new Path("/corrupted");
    // length of "corruption"
    final int length = 10;
    final byte[] bytes = StripedFileTestUtil.generateBytes(length);
    DFSTestUtil.writeFile(fs, file, bytes);
    // corrupt the first data block
    int dnIndex = findFirstDataNode(file, cellSize * dataBlocks);
    Assert.assertNotEquals(-1, dnIndex);
    LocatedStripedBlock slb = (LocatedStripedBlock) fs.getClient().getLocatedBlocks(file.toString(), 0, cellSize * dataBlocks).get(0);
    final LocatedBlock[] blks = StripedBlockUtil.parseStripedBlockGroup(slb, cellSize, dataBlocks, parityBlocks);
    // find the first block file
    File storageDir = cluster.getInstanceStorageDir(dnIndex, 0);
    File blkFile = MiniDFSCluster.getBlockFile(storageDir, blks[0].getBlock());
    Assert.assertTrue("Block file does not exist", blkFile.exists());
    // corrupt the block file
    LOG.info("Deliberately corrupting file " + blkFile.getName());
    try (FileOutputStream out = new FileOutputStream(blkFile)) {
        out.write("corruption".getBytes());
    }
    // in NameNode
    for (DataNode dn : cluster.getDataNodes()) {
        DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
    }
    try {
        // do stateful read
        StripedFileTestUtil.verifyStatefulRead(fs, file, length, bytes, ByteBuffer.allocate(1024));
        // check whether the corruption has been reported to the NameNode
        final FSNamesystem ns = cluster.getNamesystem();
        final BlockManager bm = ns.getBlockManager();
        BlockInfo blockInfo = (ns.getFSDirectory().getINode4Write(file.toString()).asFile().getBlocks())[0];
        Assert.assertEquals(1, bm.getCorruptReplicas(blockInfo).size());
    } finally {
        for (DataNode dn : cluster.getDataNodes()) {
            DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, false);
        }
    }
}

Example 50 with DataNode

use of org.apache.hadoop.hdfs.server.datanode.DataNode in project hadoop by apache.

the class TestBalancer method doTest.

/** This test start a cluster with specified number of nodes,
   * and fills it to be 30% full (with a single file replicated identically
   * to all datanodes);
   * It then adds one new empty node and starts balancing.
   *
   * @param conf - configuration
   * @param capacities - array of capacities of original nodes in cluster
   * @param racks - array of racks for original nodes in cluster
   * @param newCapacity - new node's capacity
   * @param newRack - new node's rack
   * @param nodes - information about new nodes to be started.
   * @param useTool - if true run test via Cli with command-line argument
   *   parsing, etc.   Otherwise invoke balancer API directly.
   * @param useFile - if true, the hosts to included or excluded will be stored in a
   *   file and then later read from the file.
   * @throws Exception
   */
private void doTest(Configuration conf, long[] capacities, String[] racks, long newCapacity, String newRack, NewNodeInfo nodes, boolean useTool, boolean useFile) throws Exception {
    LOG.info("capacities = " + long2String(capacities));
    LOG.info("racks      = " + Arrays.asList(racks));
    LOG.info("newCapacity= " + newCapacity);
    LOG.info("newRack    = " + newRack);
    LOG.info("useTool    = " + useTool);
    assertEquals(capacities.length, racks.length);
    int numOfDatanodes = capacities.length;
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(capacities.length).racks(racks).simulatedCapacities(capacities).build();
    try {
        cluster.waitActive();
        client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy();
        long totalCapacity = sum(capacities);
        // fill up the cluster to be 30% full
        long totalUsedSpace = totalCapacity * 3 / 10;
        createFile(cluster, filePath, totalUsedSpace / numOfDatanodes, (short) numOfDatanodes, 0);
        if (nodes == null) {
            // there is no specification of new nodes.
            // start up an empty node with the same capacity and on the same rack
            cluster.startDataNodes(conf, 1, true, null, new String[] { newRack }, null, new long[] { newCapacity });
            totalCapacity += newCapacity;
        } else {
            //if running a test with "include list", include original nodes as well
            if (nodes.getNumberofIncludeNodes() > 0) {
                for (DataNode dn : cluster.getDataNodes()) nodes.getNodesToBeIncluded().add(dn.getDatanodeId().getHostName());
            }
            String[] newRacks = new String[nodes.getNumberofNewNodes()];
            long[] newCapacities = new long[nodes.getNumberofNewNodes()];
            for (int i = 0; i < nodes.getNumberofNewNodes(); i++) {
                newRacks[i] = newRack;
                newCapacities[i] = newCapacity;
            }
            // if host names are specified for the new nodes to be created.
            if (nodes.getNames() != null) {
                cluster.startDataNodes(conf, nodes.getNumberofNewNodes(), true, null, newRacks, nodes.getNames(), newCapacities);
                totalCapacity += newCapacity * nodes.getNumberofNewNodes();
            } else {
                // host names are not specified
                cluster.startDataNodes(conf, nodes.getNumberofNewNodes(), true, null, newRacks, null, newCapacities);
                totalCapacity += newCapacity * nodes.getNumberofNewNodes();
                //populate the include nodes
                if (nodes.getNumberofIncludeNodes() > 0) {
                    int totalNodes = cluster.getDataNodes().size();
                    for (int i = 0; i < nodes.getNumberofIncludeNodes(); i++) {
                        nodes.getNodesToBeIncluded().add(cluster.getDataNodes().get(totalNodes - 1 - i).getDatanodeId().getXferAddr());
                    }
                }
                //polulate the exclude nodes
                if (nodes.getNumberofExcludeNodes() > 0) {
                    int totalNodes = cluster.getDataNodes().size();
                    for (int i = 0; i < nodes.getNumberofExcludeNodes(); i++) {
                        nodes.getNodesToBeExcluded().add(cluster.getDataNodes().get(totalNodes - 1 - i).getDatanodeId().getXferAddr());
                    }
                }
            }
        }
        // run balancer and validate results
        BalancerParameters.Builder pBuilder = new BalancerParameters.Builder();
        if (nodes != null) {
            pBuilder.setExcludedNodes(nodes.getNodesToBeExcluded());
            pBuilder.setIncludedNodes(nodes.getNodesToBeIncluded());
            pBuilder.setRunDuringUpgrade(false);
        }
        BalancerParameters p = pBuilder.build();
        int expectedExcludedNodes = 0;
        if (nodes != null) {
            if (!nodes.getNodesToBeExcluded().isEmpty()) {
                expectedExcludedNodes = nodes.getNodesToBeExcluded().size();
            } else if (!nodes.getNodesToBeIncluded().isEmpty()) {
                expectedExcludedNodes = cluster.getDataNodes().size() - nodes.getNodesToBeIncluded().size();
            }
        }
        // run balancer and validate results
        if (useTool) {
            runBalancerCli(conf, totalUsedSpace, totalCapacity, p, useFile, expectedExcludedNodes);
        } else {
            runBalancer(conf, totalUsedSpace, totalCapacity, p, expectedExcludedNodes);
        }
    } finally {
        cluster.shutdown();
    }
}