Examples with DFSClient org.apache.hadoop.hdfs.DFSClient used on opensource projects

Example 31 with DFSClient

use of org.apache.hadoop.hdfs.DFSClient in project hadoop by apache.

the class TestFsck method testFsckMove.

@Test
public void testFsckMove() throws Exception {
    final int dfsBlockSize = 1024;
    final int numDatanodes = 4;
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, dfsBlockSize);
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
    conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
    DFSTestUtil util = new DFSTestUtil("TestFsck", 5, 3, (5 * dfsBlockSize) + (dfsBlockSize - 1), 5 * dfsBlockSize);
    FileSystem fs = null;
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).build();
    String topDir = "/srcdat";
    fs = cluster.getFileSystem();
    cluster.waitActive();
    util.createFiles(fs, topDir);
    util.waitReplication(fs, topDir, (short) 3);
    String outStr = runFsck(conf, 0, true, "/");
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost", cluster.getNameNodePort()), conf);
    String[] fileNames = util.getFileNames(topDir);
    CorruptedTestFile[] ctFiles = new CorruptedTestFile[] { new CorruptedTestFile(fileNames[0], Sets.newHashSet(0), dfsClient, numDatanodes, dfsBlockSize), new CorruptedTestFile(fileNames[1], Sets.newHashSet(2, 3), dfsClient, numDatanodes, dfsBlockSize), new CorruptedTestFile(fileNames[2], Sets.newHashSet(4), dfsClient, numDatanodes, dfsBlockSize), new CorruptedTestFile(fileNames[3], Sets.newHashSet(0, 1, 2, 3), dfsClient, numDatanodes, dfsBlockSize), new CorruptedTestFile(fileNames[4], Sets.newHashSet(1, 2, 3, 4), dfsClient, numDatanodes, dfsBlockSize) };
    int totalMissingBlocks = 0;
    for (CorruptedTestFile ctFile : ctFiles) {
        totalMissingBlocks += ctFile.getTotalMissingBlocks();
    }
    for (CorruptedTestFile ctFile : ctFiles) {
        ctFile.removeBlocks(cluster);
    }
    // Wait for fsck to discover all the missing blocks
    while (true) {
        outStr = runFsck(conf, 1, false, "/");
        String numMissing = null;
        String numCorrupt = null;
        for (String line : outStr.split(LINE_SEPARATOR)) {
            Matcher m = NUM_MISSING_BLOCKS_PATTERN.matcher(line);
            if (m.matches()) {
                numMissing = m.group(1);
            }
            m = NUM_CORRUPT_BLOCKS_PATTERN.matcher(line);
            if (m.matches()) {
                numCorrupt = m.group(1);
            }
            if (numMissing != null && numCorrupt != null) {
                break;
            }
        }
        if (numMissing == null || numCorrupt == null) {
            throw new IOException("failed to find number of missing or corrupt" + " blocks in fsck output.");
        }
        if (numMissing.equals(Integer.toString(totalMissingBlocks))) {
            assertTrue(numCorrupt.equals(Integer.toString(0)));
            assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
            break;
        }
        try {
            Thread.sleep(100);
        } catch (InterruptedException ignore) {
        }
    }
    // Copy the non-corrupt blocks of corruptFileName to lost+found.
    outStr = runFsck(conf, 1, false, "/", "-move");
    LOG.info("WATERMELON: outStr = " + outStr);
    assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
    // to lost+found
    for (CorruptedTestFile ctFile : ctFiles) {
        ctFile.checkSalvagedRemains();
    }
    // Fix the filesystem by removing corruptFileName
    outStr = runFsck(conf, 1, true, "/", "-delete");
    assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
    // Check to make sure we have a healthy filesystem
    outStr = runFsck(conf, 0, true, "/");
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    util.cleanup(fs, topDir);
}

Example 32 with DFSClient

use of org.apache.hadoop.hdfs.DFSClient in project hadoop by apache.

the class TestFsck method testCorruptBlock.

@Test
public void testCorruptBlock() throws Exception {
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000);
    // Set short retry timeouts so this test runs faster
    conf.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10);
    FileSystem fs = null;
    DFSClient dfsClient = null;
    LocatedBlocks blocks = null;
    int replicaCount = 0;
    Random random = new Random();
    String outStr = null;
    short factor = 1;
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    fs = cluster.getFileSystem();
    Path file1 = new Path("/testCorruptBlock");
    DFSTestUtil.createFile(fs, file1, 1024, factor, 0);
    // Wait until file replication has completed
    DFSTestUtil.waitReplication(fs, file1, factor);
    ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1);
    // Make sure filesystem is in healthy state
    outStr = runFsck(conf, 0, true, "/");
    System.out.println(outStr);
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    // corrupt replicas
    File blockFile = cluster.getBlockFile(0, block);
    if (blockFile != null && blockFile.exists()) {
        RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw");
        FileChannel channel = raFile.getChannel();
        String badString = "BADBAD";
        int rand = random.nextInt((int) channel.size() / 2);
        raFile.seek(rand);
        raFile.write(badString.getBytes());
        raFile.close();
    }
    // Read the file to trigger reportBadBlocks
    try {
        IOUtils.copyBytes(fs.open(file1), new IOUtils.NullOutputStream(), conf, true);
    } catch (IOException ie) {
        assertTrue(ie instanceof ChecksumException);
    }
    dfsClient = new DFSClient(new InetSocketAddress("localhost", cluster.getNameNodePort()), conf);
    blocks = dfsClient.getNamenode().getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
    replicaCount = blocks.get(0).getLocations().length;
    while (replicaCount != factor) {
        try {
            Thread.sleep(100);
        } catch (InterruptedException ignore) {
        }
        blocks = dfsClient.getNamenode().getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
        replicaCount = blocks.get(0).getLocations().length;
    }
    assertTrue(blocks.get(0).isCorrupt());
    // Check if fsck reports the same
    outStr = runFsck(conf, 1, true, "/");
    System.out.println(outStr);
    assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
    assertTrue(outStr.contains("testCorruptBlock"));
}

Example 33 with DFSClient

use of org.apache.hadoop.hdfs.DFSClient in project hadoop by apache.

the class DFSClientCache method inputStreamLoader.

private CacheLoader<DFSInputStreamCaheKey, FSDataInputStream> inputStreamLoader() {
    return new CacheLoader<DFSInputStreamCaheKey, FSDataInputStream>() {

        @Override
        public FSDataInputStream load(DFSInputStreamCaheKey key) throws Exception {
            DFSClient client = getDfsClient(key.userId);
            DFSInputStream dis = client.open(key.inodePath);
            return client.createWrappedInputStream(dis);
        }
    };
}

Example 34 with DFSClient

use of org.apache.hadoop.hdfs.DFSClient in project hadoop by apache.

the class TestHASafeMode method testOpenFileWhenNNAndClientCrashAfterAddBlock.

/** Test NN crash and client crash/stuck immediately after block allocation */
@Test(timeout = 100000)
public void testOpenFileWhenNNAndClientCrashAfterAddBlock() throws Exception {
    cluster.getConfiguration(0).set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, "1.0f");
    String testData = "testData";
    // to make sure we write the full block before creating dummy block at NN.
    cluster.getConfiguration(0).setInt("io.bytes.per.checksum", testData.length());
    cluster.restartNameNode(0);
    try {
        cluster.waitActive();
        cluster.transitionToActive(0);
        cluster.transitionToStandby(1);
        DistributedFileSystem dfs = cluster.getFileSystem(0);
        String pathString = "/tmp1.txt";
        Path filePath = new Path(pathString);
        FSDataOutputStream create = dfs.create(filePath, FsPermission.getDefault(), true, 1024, (short) 3, testData.length(), null);
        create.write(testData.getBytes());
        create.hflush();
        long fileId = ((DFSOutputStream) create.getWrappedStream()).getFileId();
        FileStatus fileStatus = dfs.getFileStatus(filePath);
        DFSClient client = DFSClientAdapter.getClient(dfs);
        // add one dummy block at NN, but not write to DataNode
        ExtendedBlock previousBlock = DFSClientAdapter.getPreviousBlock(client, fileId);
        DFSClientAdapter.getNamenode(client).addBlock(pathString, client.getClientName(), new ExtendedBlock(previousBlock), new DatanodeInfo[0], DFSClientAdapter.getFileId((DFSOutputStream) create.getWrappedStream()), null, null);
        cluster.restartNameNode(0, true);
        cluster.restartDataNode(0);
        cluster.transitionToActive(0);
        // let the block reports be processed.
        Thread.sleep(2000);
        FSDataInputStream is = dfs.open(filePath);
        is.close();
        // initiate recovery
        dfs.recoverLease(filePath);
        assertTrue("Recovery also should be success", dfs.recoverLease(filePath));
    } finally {
        cluster.shutdown();
    }
}

Example 35 with DFSClient

use of org.apache.hadoop.hdfs.DFSClient in project hadoop by apache.

the class TestRetryCacheWithHA method testRenameSnapshot.

@Test(timeout = 60000)
public void testRenameSnapshot() throws Exception {
    final DFSClient client = genClientWithDummyHandler();
    AtMostOnceOp op = new RenameSnapshotOp(client, "/test", "s1", "s2");
    testClientRetryWithFailover(op);
}