Examples with FSNamesystem org.apache.hadoop.hdfs.server.namenode.FSNamesystem used on opensource projects

Example 11 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestReadStripedFileWithDecoding method testReportBadBlock.

/**
   * After reading a corrupted block, make sure the client can correctly report
   * the corruption to the NameNode.
   */
@Test
public void testReportBadBlock() throws IOException {
    // create file
    final Path file = new Path("/corrupted");
    // length of "corruption"
    final int length = 10;
    final byte[] bytes = StripedFileTestUtil.generateBytes(length);
    DFSTestUtil.writeFile(fs, file, bytes);
    // corrupt the first data block
    int dnIndex = findFirstDataNode(file, cellSize * dataBlocks);
    Assert.assertNotEquals(-1, dnIndex);
    LocatedStripedBlock slb = (LocatedStripedBlock) fs.getClient().getLocatedBlocks(file.toString(), 0, cellSize * dataBlocks).get(0);
    final LocatedBlock[] blks = StripedBlockUtil.parseStripedBlockGroup(slb, cellSize, dataBlocks, parityBlocks);
    // find the first block file
    File storageDir = cluster.getInstanceStorageDir(dnIndex, 0);
    File blkFile = MiniDFSCluster.getBlockFile(storageDir, blks[0].getBlock());
    Assert.assertTrue("Block file does not exist", blkFile.exists());
    // corrupt the block file
    LOG.info("Deliberately corrupting file " + blkFile.getName());
    try (FileOutputStream out = new FileOutputStream(blkFile)) {
        out.write("corruption".getBytes());
    }
    // in NameNode
    for (DataNode dn : cluster.getDataNodes()) {
        DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
    }
    try {
        // do stateful read
        StripedFileTestUtil.verifyStatefulRead(fs, file, length, bytes, ByteBuffer.allocate(1024));
        // check whether the corruption has been reported to the NameNode
        final FSNamesystem ns = cluster.getNamesystem();
        final BlockManager bm = ns.getBlockManager();
        BlockInfo blockInfo = (ns.getFSDirectory().getINode4Write(file.toString()).asFile().getBlocks())[0];
        Assert.assertEquals(1, bm.getCorruptReplicas(blockInfo).size());
    } finally {
        for (DataNode dn : cluster.getDataNodes()) {
            DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, false);
        }
    }
}

Example 12 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestMaintenanceState method testTransitionToDecommission.

/**
   * Transition from IN_MAINTENANCE to DECOMMISSIONED.
   */
@Test(timeout = 360000)
public void testTransitionToDecommission() throws IOException {
    LOG.info("Starting testTransitionToDecommission");
    final int numNamenodes = 1;
    final int numDatanodes = 4;
    startCluster(numNamenodes, numDatanodes);
    final Path file = new Path("testTransitionToDecommission.dat");
    final int replicas = 3;
    FileSystem fileSys = getCluster().getFileSystem(0);
    FSNamesystem ns = getCluster().getNamesystem(0);
    writeFile(fileSys, file, replicas, 1);
    DatanodeInfo nodeOutofService = takeNodeOutofService(0, getFirstBlockFirstReplicaUuid(fileSys, file), Long.MAX_VALUE, null, AdminStates.IN_MAINTENANCE);
    DFSClient client = getDfsClient(0);
    assertEquals("All datanodes must be alive", numDatanodes, client.datanodeReport(DatanodeReportType.LIVE).length);
    // test 1, verify the replica in IN_MAINTENANCE state isn't in LocatedBlock
    assertNull(checkWithRetry(ns, fileSys, file, replicas - 1, nodeOutofService));
    takeNodeOutofService(0, nodeOutofService.getDatanodeUuid(), 0, null, AdminStates.DECOMMISSIONED);
    // test 2 after decommission has completed, the replication count is
    // replicas + 1 which includes the decommissioned node.
    assertNull(checkWithRetry(ns, fileSys, file, replicas + 1, null));
    // test 3, put the node in service, replication count should restore.
    putNodeInService(0, nodeOutofService.getDatanodeUuid());
    assertNull(checkWithRetry(ns, fileSys, file, replicas, null));
    cleanupFile(fileSys, file);
}

Example 13 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestMaintenanceState method testMultipleNodesMaintenance.

/**
   * Verify if multiple DataNodes can transition to maintenance state
   * at the same time.
   */
@Test(timeout = 360000)
public void testMultipleNodesMaintenance() throws Exception {
    startCluster(1, 5);
    final Path file = new Path("/testMultipleNodesMaintenance.dat");
    final FileSystem fileSys = getCluster().getFileSystem(0);
    final FSNamesystem ns = getCluster().getNamesystem(0);
    int repl = 3;
    writeFile(fileSys, file, repl, 1);
    final DatanodeInfo[] nodes = getFirstBlockReplicasDatanodeInfos(fileSys, file);
    // Request maintenance for DataNodes 1 and 2 which has the file blocks.
    List<DatanodeInfo> maintenanceDN = takeNodeOutofService(0, Lists.newArrayList(nodes[0].getDatanodeUuid(), nodes[1].getDatanodeUuid()), Long.MAX_VALUE, null, null, AdminStates.IN_MAINTENANCE);
    // Verify file replication matches maintenance state min replication
    assertNull(checkWithRetry(ns, fileSys, file, 1, null, nodes[0]));
    // Put the maintenance nodes back in service
    for (DatanodeInfo datanodeInfo : maintenanceDN) {
        putNodeInService(0, datanodeInfo);
    }
    // Verify file replication catching up to the old state
    assertNull(checkWithRetry(ns, fileSys, file, repl, null));
    cleanupFile(fileSys, file);
}

Example 14 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestMaintenanceState method testDecommissionDifferentNodeAfterMaintenance.

private void testDecommissionDifferentNodeAfterMaintenance(int repl) throws Exception {
    setup();
    startCluster(1, 5);
    final Path file = new Path("/testDecommissionDifferentNodeAfterMaintenance.dat");
    final FileSystem fileSys = getCluster().getFileSystem(0);
    final FSNamesystem ns = getCluster().getNamesystem(0);
    writeFile(fileSys, file, repl, 1);
    final DatanodeInfo[] nodes = getFirstBlockReplicasDatanodeInfos(fileSys, file);
    String maintenanceDNUuid = nodes[0].getDatanodeUuid();
    String decommissionDNUuid = nodes[1].getDatanodeUuid();
    DatanodeInfo maintenanceDN = takeNodeOutofService(0, maintenanceDNUuid, Long.MAX_VALUE, null, null, AdminStates.IN_MAINTENANCE);
    Map<DatanodeInfo, Long> maintenanceNodes = new HashMap<>();
    maintenanceNodes.put(nodes[0], Long.MAX_VALUE);
    takeNodeOutofService(0, decommissionDNUuid, 0, null, maintenanceNodes, AdminStates.DECOMMISSIONED);
    // Out of the replicas returned, one is the decommissioned node.
    assertNull(checkWithRetry(ns, fileSys, file, repl, maintenanceDN));
    putNodeInService(0, maintenanceDN);
    assertNull(checkWithRetry(ns, fileSys, file, repl + 1, null));
    cleanupFile(fileSys, file);
    teardown();
}

Example 15 with FSNamesystem

use of org.apache.hadoop.hdfs.server.namenode.FSNamesystem in project hadoop by apache.

the class TestBlocksWithNotEnoughRacks method testSufficientlySingleReplBlockUsesNewRack.

/*
   * Like the previous test but the block starts with a single replica,
   * and therefore unlike the previous test the block does not start
   * off needing replicas.
   */
@Test
public void testSufficientlySingleReplBlockUsesNewRack() throws Exception {
    Configuration conf = getConf();
    short REPLICATION_FACTOR = 1;
    final Path filePath = new Path("/testFile");
    String[] racks = { "/rack1", "/rack1", "/rack1", "/rack2" };
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(racks.length).racks(racks).build();
    final FSNamesystem ns = cluster.getNameNode().getNamesystem();
    try {
        // Create a file with one block with a replication factor of 1
        final FileSystem fs = cluster.getFileSystem();
        DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L);
        ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath);
        DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 0);
        REPLICATION_FACTOR = 2;
        NameNodeAdapter.setReplication(ns, "/testFile", REPLICATION_FACTOR);
        DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0);
    } finally {
        cluster.shutdown();
    }
}