Examples with DFSTestUtil org.apache.hadoop.hdfs.DFSTestUtil used on opensource projects

Example 21 with DFSTestUtil

use of org.apache.hadoop.hdfs.DFSTestUtil in project hadoop by apache.

the class TestListCorruptFileBlocks method testListCorruptFileBlocksOnRelativePath.

@Test(timeout = 60000)
public void testListCorruptFileBlocksOnRelativePath() throws Exception {
    Configuration conf = new Configuration();
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000);
    conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
    MiniDFSCluster cluster = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).build();
        cluster.waitActive();
        FileSystem fs = cluster.getFileSystem();
        DistributedFileSystem dfs = (DistributedFileSystem) fs;
        final Path baseDir = new Path("/somewhere/base");
        fs.mkdirs(baseDir);
        // set working dir
        fs.setWorkingDirectory(baseDir);
        DFSTestUtil util = new DFSTestUtil.Builder().setName("testGetCorruptFilesOnRelativePath").setNumFiles(3).setMaxLevels(1).setMaxSize(1024).build();
        util.createFiles(fs, "corruptData");
        RemoteIterator<Path> corruptFileBlocks = dfs.listCorruptFileBlocks(new Path("corruptData"));
        int numCorrupt = countPaths(corruptFileBlocks);
        assertTrue(numCorrupt == 0);
        // delete the blocks
        String bpid = cluster.getNamesystem().getBlockPoolId();
        // For loop through number of data directories per datanode (2)
        for (int i = 0; i < 2; i++) {
            File storageDir = cluster.getInstanceStorageDir(0, i);
            File data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
            List<File> metadataFiles = MiniDFSCluster.getAllBlockMetadataFiles(data_dir);
            if (metadataFiles == null)
                continue;
            for (File metadataFile : metadataFiles) {
                File blockFile = Block.metaToBlockFile(metadataFile);
                LOG.info("Deliberately removing file " + blockFile.getName());
                assertTrue("Cannot remove file.", blockFile.delete());
                LOG.info("Deliberately removing file " + metadataFile.getName());
                assertTrue("Cannot remove file.", metadataFile.delete());
            }
        }
        int count = 0;
        corruptFileBlocks = dfs.listCorruptFileBlocks(new Path("corruptData"));
        numCorrupt = countPaths(corruptFileBlocks);
        while (numCorrupt < 3) {
            Thread.sleep(1000);
            corruptFileBlocks = dfs.listCorruptFileBlocks(new Path("corruptData"));
            numCorrupt = countPaths(corruptFileBlocks);
            count++;
            if (count > 30)
                break;
        }
        // Validate we get all the corrupt files
        LOG.info("Namenode has bad files. " + numCorrupt);
        assertTrue("Failed to get corrupt files!", numCorrupt == 3);
        util.cleanup(fs, "corruptData");
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 22 with DFSTestUtil

use of org.apache.hadoop.hdfs.DFSTestUtil in project hadoop by apache.

the class TestFsck method testBlockIdCKCorruption.

/**
   * Test for blockIdCK with block corruption.
   */
@Test
public void testBlockIdCKCorruption() throws Exception {
    short numDn = 1;
    final long blockSize = 512;
    Random random = new Random();
    ExtendedBlock block;
    short repFactor = 1;
    String[] racks = { "/rack1" };
    String[] hosts = { "host1" };
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000);
    // Set short retry timeouts so this test runs faster
    conf.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10);
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
    conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
    DistributedFileSystem dfs = null;
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts).racks(racks).build();
    assertNotNull("Failed Cluster Creation", cluster);
    cluster.waitClusterUp();
    dfs = cluster.getFileSystem();
    assertNotNull("Failed to get FileSystem", dfs);
    DFSTestUtil util = new DFSTestUtil.Builder().setName(getClass().getSimpleName()).setNumFiles(1).build();
    //create files
    final String pathString = new String("/testfile");
    final Path path = new Path(pathString);
    util.createFile(dfs, path, 1024, repFactor, 1000L);
    util.waitReplication(dfs, path, repFactor);
    StringBuilder sb = new StringBuilder();
    for (LocatedBlock lb : util.getAllBlocks(dfs, path)) {
        sb.append(lb.getBlock().getLocalBlock().getBlockName() + " ");
    }
    String[] bIds = sb.toString().split(" ");
    //make sure block is healthy before we corrupt it
    String outStr = runFsck(conf, 0, true, "/", "-blockId", bIds[0]);
    System.out.println(outStr);
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    // corrupt replicas
    block = DFSTestUtil.getFirstBlock(dfs, path);
    File blockFile = cluster.getBlockFile(0, block);
    if (blockFile != null && blockFile.exists()) {
        RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw");
        FileChannel channel = raFile.getChannel();
        String badString = "BADBAD";
        int rand = random.nextInt((int) channel.size() / 2);
        raFile.seek(rand);
        raFile.write(badString.getBytes());
        raFile.close();
    }
    util.waitCorruptReplicas(dfs, cluster.getNamesystem(), path, block, 1);
    outStr = runFsck(conf, 1, false, "/", "-blockId", block.getBlockName());
    System.out.println(outStr);
    assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
}

Example 23 with DFSTestUtil

use of org.apache.hadoop.hdfs.DFSTestUtil in project hadoop by apache.

the class TestFsck method testFsckMoveAndDelete.

@Test
public void testFsckMoveAndDelete() throws Exception {
    final int maxMoveTries = 5;
    DFSTestUtil util = new DFSTestUtil.Builder().setName("TestFsckMoveAndDelete").setNumFiles(5).build();
    FileSystem fs = null;
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
    conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
    String topDir = "/srcdat";
    fs = cluster.getFileSystem();
    cluster.waitActive();
    util.createFiles(fs, topDir);
    util.waitReplication(fs, topDir, (short) 3);
    String outStr = runFsck(conf, 0, true, "/");
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    // Corrupt a block by deleting it
    String[] fileNames = util.getFileNames(topDir);
    DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost", cluster.getNameNodePort()), conf);
    String corruptFileName = fileNames[0];
    ExtendedBlock block = dfsClient.getNamenode().getBlockLocations(corruptFileName, 0, Long.MAX_VALUE).get(0).getBlock();
    for (int i = 0; i < 4; i++) {
        File blockFile = cluster.getBlockFile(i, block);
        if (blockFile != null && blockFile.exists()) {
            assertTrue(blockFile.delete());
        }
    }
    // We excpect the filesystem to be corrupted
    outStr = runFsck(conf, 1, false, "/");
    while (!outStr.contains(NamenodeFsck.CORRUPT_STATUS)) {
        try {
            Thread.sleep(100);
        } catch (InterruptedException ignore) {
        }
        outStr = runFsck(conf, 1, false, "/");
    }
    // After a fsck -move, the corrupted file should still exist.
    for (int i = 0; i < maxMoveTries; i++) {
        outStr = runFsck(conf, 1, true, "/", "-move");
        assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
        String[] newFileNames = util.getFileNames(topDir);
        boolean found = false;
        for (String f : newFileNames) {
            if (f.equals(corruptFileName)) {
                found = true;
                break;
            }
        }
        assertTrue(found);
    }
    // Fix the filesystem by moving corrupted files to lost+found
    outStr = runFsck(conf, 1, true, "/", "-move", "-delete");
    assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
    // Check to make sure we have healthy filesystem
    outStr = runFsck(conf, 0, true, "/");
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    util.cleanup(fs, topDir);
}

Example 24 with DFSTestUtil

use of org.apache.hadoop.hdfs.DFSTestUtil in project hadoop by apache.

the class TestFsck method testFsckMoveAfterCorruption.

@Test(timeout = 300000)
public void testFsckMoveAfterCorruption() throws Exception {
    final int dfsBlockSize = 512 * 1024;
    final int numDatanodes = 1;
    final int replication = 1;
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, dfsBlockSize);
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L);
    conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
    conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, replication);
    cluster = new MiniDFSCluster.Builder(conf).build();
    DistributedFileSystem dfs = cluster.getFileSystem();
    cluster.waitActive();
    final String srcDir = "/srcdat";
    final DFSTestUtil util = new DFSTestUtil.Builder().setName("TestFsck").setMinSize(dfsBlockSize * 2).setMaxSize(dfsBlockSize * 3).setNumFiles(1).build();
    util.createFiles(dfs, srcDir, (short) replication);
    final String[] fileNames = util.getFileNames(srcDir);
    LOG.info("Created files: " + Arrays.toString(fileNames));
    // Run fsck here. The output is automatically logged for easier debugging
    String outStr = runFsck(conf, 0, true, "/", "-files", "-blocks");
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    // Corrupt the first block
    final DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost", cluster.getNameNodePort()), conf);
    final String blockFileToCorrupt = fileNames[0];
    final CorruptedTestFile ctf = new CorruptedTestFile(blockFileToCorrupt, Sets.newHashSet(0), dfsClient, numDatanodes, dfsBlockSize);
    ctf.corruptBlocks(cluster);
    // Wait for fsck to discover all the missing blocks
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            try {
                final String str = runFsck(conf, 1, false, "/");
                String numCorrupt = null;
                for (String line : str.split(LINE_SEPARATOR)) {
                    Matcher m = NUM_CORRUPT_BLOCKS_PATTERN.matcher(line);
                    if (m.matches()) {
                        numCorrupt = m.group(1);
                        break;
                    }
                }
                if (numCorrupt == null) {
                    Assert.fail("Cannot find corrupt blocks count in fsck output.");
                }
                if (Integer.parseInt(numCorrupt) == ctf.getTotalMissingBlocks()) {
                    assertTrue(str.contains(NamenodeFsck.CORRUPT_STATUS));
                    return true;
                }
            } catch (Exception e) {
                LOG.error("Exception caught", e);
                Assert.fail("Caught unexpected exception.");
            }
            return false;
        }
    }, 1000, 60000);
    runFsck(conf, 1, true, "/", "-files", "-blocks", "-racks");
    LOG.info("Moving blocks to lost+found");
    // Fsck will return error since we corrupted a block
    runFsck(conf, 1, false, "/", "-move");
    final List<LocatedFileStatus> retVal = new ArrayList<>();
    final RemoteIterator<LocatedFileStatus> iter = dfs.listFiles(new Path("/lost+found"), true);
    while (iter.hasNext()) {
        retVal.add(iter.next());
    }
    LOG.info("Items in lost+found: " + retVal);
    // Expect all good blocks moved, only corrupted block skipped.
    long totalLength = 0;
    for (LocatedFileStatus lfs : retVal) {
        totalLength += lfs.getLen();
    }
    Assert.assertTrue("Nothing is moved to lost+found!", totalLength > 0);
    util.cleanup(dfs, srcDir);
}

Example 25 with DFSTestUtil

use of org.apache.hadoop.hdfs.DFSTestUtil in project hadoop by apache.

the class TestParallelImageWrite method testRestartDFS.

/** check if DFS remains in proper condition after a restart */
@Test
public void testRestartDFS() throws Exception {
    final Configuration conf = new HdfsConfiguration();
    MiniDFSCluster cluster = null;
    FSNamesystem fsn = null;
    int numNamenodeDirs;
    DFSTestUtil files = new DFSTestUtil.Builder().setName("TestRestartDFS").setNumFiles(200).build();
    final String dir = "/srcdat";
    final Path rootpath = new Path("/");
    final Path dirpath = new Path(dir);
    long rootmtime;
    FileStatus rootstatus;
    FileStatus dirstatus;
    try {
        cluster = new MiniDFSCluster.Builder(conf).format(true).numDataNodes(NUM_DATANODES).build();
        String[] nameNodeDirs = conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, new String[] {});
        numNamenodeDirs = nameNodeDirs.length;
        assertTrue("failed to get number of Namenode StorageDirs", numNamenodeDirs != 0);
        FileSystem fs = cluster.getFileSystem();
        files.createFiles(fs, dir);
        rootmtime = fs.getFileStatus(rootpath).getModificationTime();
        rootstatus = fs.getFileStatus(dirpath);
        dirstatus = fs.getFileStatus(dirpath);
        fs.setOwner(rootpath, rootstatus.getOwner() + "_XXX", null);
        fs.setOwner(dirpath, null, dirstatus.getGroup() + "_XXX");
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
    try {
        // Force the NN to save its images on startup so long as
        // there are any uncheckpointed txns
        conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1);
        // Here we restart the MiniDFScluster without formatting namenode
        cluster = new MiniDFSCluster.Builder(conf).format(false).numDataNodes(NUM_DATANODES).build();
        fsn = cluster.getNamesystem();
        FileSystem fs = cluster.getFileSystem();
        assertTrue("Filesystem corrupted after restart.", files.checkFiles(fs, dir));
        final FileStatus newrootstatus = fs.getFileStatus(rootpath);
        assertEquals(rootmtime, newrootstatus.getModificationTime());
        assertEquals(rootstatus.getOwner() + "_XXX", newrootstatus.getOwner());
        assertEquals(rootstatus.getGroup(), newrootstatus.getGroup());
        final FileStatus newdirstatus = fs.getFileStatus(dirpath);
        assertEquals(dirstatus.getOwner(), newdirstatus.getOwner());
        assertEquals(dirstatus.getGroup() + "_XXX", newdirstatus.getGroup());
        rootmtime = fs.getFileStatus(rootpath).getModificationTime();
        final String checkAfterRestart = checkImages(fsn, numNamenodeDirs);
        // Modify the system and then perform saveNamespace
        files.cleanup(fs, dir);
        files.createFiles(fs, dir);
        fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
        cluster.getNameNodeRpc().saveNamespace(0, 0);
        final String checkAfterModify = checkImages(fsn, numNamenodeDirs);
        assertFalse("Modified namespace should change fsimage contents. " + "was: " + checkAfterRestart + " now: " + checkAfterModify, checkAfterRestart.equals(checkAfterModify));
        fsn.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
        files.cleanup(fs, dir);
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}