Examples with DFSOutputStream org.apache.hadoop.hdfs.DFSOutputStream used on opensource projects

Example 1 with DFSOutputStream

use of org.apache.hadoop.hdfs.DFSOutputStream in project hadoop by apache.

the class TestHASafeMode method testOpenFileWhenNNAndClientCrashAfterAddBlock.

/** Test NN crash and client crash/stuck immediately after block allocation */
@Test(timeout = 100000)
public void testOpenFileWhenNNAndClientCrashAfterAddBlock() throws Exception {
    cluster.getConfiguration(0).set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, "1.0f");
    String testData = "testData";
    // to make sure we write the full block before creating dummy block at NN.
    cluster.getConfiguration(0).setInt("io.bytes.per.checksum", testData.length());
    cluster.restartNameNode(0);
    try {
        cluster.waitActive();
        cluster.transitionToActive(0);
        cluster.transitionToStandby(1);
        DistributedFileSystem dfs = cluster.getFileSystem(0);
        String pathString = "/tmp1.txt";
        Path filePath = new Path(pathString);
        FSDataOutputStream create = dfs.create(filePath, FsPermission.getDefault(), true, 1024, (short) 3, testData.length(), null);
        create.write(testData.getBytes());
        create.hflush();
        long fileId = ((DFSOutputStream) create.getWrappedStream()).getFileId();
        FileStatus fileStatus = dfs.getFileStatus(filePath);
        DFSClient client = DFSClientAdapter.getClient(dfs);
        // add one dummy block at NN, but not write to DataNode
        ExtendedBlock previousBlock = DFSClientAdapter.getPreviousBlock(client, fileId);
        DFSClientAdapter.getNamenode(client).addBlock(pathString, client.getClientName(), new ExtendedBlock(previousBlock), new DatanodeInfo[0], DFSClientAdapter.getFileId((DFSOutputStream) create.getWrappedStream()), null, null);
        cluster.restartNameNode(0, true);
        cluster.restartDataNode(0);
        cluster.transitionToActive(0);
        // let the block reports be processed.
        Thread.sleep(2000);
        FSDataInputStream is = dfs.open(filePath);
        is.close();
        // initiate recovery
        dfs.recoverLease(filePath);
        assertTrue("Recovery also should be success", dfs.recoverLease(filePath));
    } finally {
        cluster.shutdown();
    }
}

Example 2 with DFSOutputStream

use of org.apache.hadoop.hdfs.DFSOutputStream in project hadoop by apache.

the class TestFsck method testFsckOpenFiles.

@Test
public void testFsckOpenFiles() throws Exception {
    DFSTestUtil util = new DFSTestUtil.Builder().setName("TestFsck").setNumFiles(4).build();
    FileSystem fs = null;
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
    String topDir = "/srcdat";
    String randomString = "HADOOP  ";
    fs = cluster.getFileSystem();
    cluster.waitActive();
    util.createFiles(fs, topDir);
    util.waitReplication(fs, topDir, (short) 3);
    String outStr = runFsck(conf, 0, true, "/");
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    // Open a file for writing and do not close for now
    Path openFile = new Path(topDir + "/openFile");
    FSDataOutputStream out = fs.create(openFile);
    int writeCount = 0;
    while (writeCount != 100) {
        out.write(randomString.getBytes());
        writeCount++;
    }
    ((DFSOutputStream) out.getWrappedStream()).hflush();
    // We expect the filesystem to be HEALTHY and show one open file
    outStr = runFsck(conf, 0, true, topDir);
    System.out.println(outStr);
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    assertFalse(outStr.contains("OPENFORWRITE"));
    // Use -openforwrite option to list open files
    outStr = runFsck(conf, 0, true, topDir, "-files", "-blocks", "-locations", "-openforwrite");
    System.out.println(outStr);
    assertTrue(outStr.contains("OPENFORWRITE"));
    assertTrue(outStr.contains("Under Construction Block:"));
    assertTrue(outStr.contains("openFile"));
    // Close the file
    out.close();
    // Now, fsck should show HEALTHY fs and should not show any open files
    outStr = runFsck(conf, 0, true, topDir);
    System.out.println(outStr);
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    assertFalse(outStr.contains("OPENFORWRITE"));
    assertFalse(outStr.contains("Under Construction Block:"));
    util.cleanup(fs, topDir);
}

Example 3 with DFSOutputStream

use of org.apache.hadoop.hdfs.DFSOutputStream in project hadoop by apache.

the class TestLeaseRenewer method testRenewal.

@Test
public void testRenewal() throws Exception {
    // Keep track of how many times the lease gets renewed
    final AtomicInteger leaseRenewalCount = new AtomicInteger();
    Mockito.doAnswer(new Answer<Boolean>() {

        @Override
        public Boolean answer(InvocationOnMock invocation) throws Throwable {
            leaseRenewalCount.incrementAndGet();
            return true;
        }
    }).when(MOCK_DFSCLIENT).renewLease();
    // Set up a file so that we start renewing our lease.
    DFSOutputStream mockStream = Mockito.mock(DFSOutputStream.class);
    long fileId = 123L;
    renewer.put(fileId, mockStream, MOCK_DFSCLIENT);
    // Wait for lease to get renewed
    long failTime = Time.monotonicNow() + 5000;
    while (Time.monotonicNow() < failTime && leaseRenewalCount.get() == 0) {
        Thread.sleep(50);
    }
    if (leaseRenewalCount.get() == 0) {
        Assert.fail("Did not renew lease at all!");
    }
    renewer.closeFile(fileId, MOCK_DFSCLIENT);
}

Example 4 with DFSOutputStream

use of org.apache.hadoop.hdfs.DFSOutputStream in project hadoop by apache.

the class TestUpdatePipelineWithSnapshots method testUpdatePipelineAfterDelete.

// Regression test for HDFS-6647.
@Test
public void testUpdatePipelineAfterDelete() throws Exception {
    Configuration conf = new HdfsConfiguration();
    Path file = new Path("/test-file");
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
    try {
        FileSystem fs = cluster.getFileSystem();
        NamenodeProtocols namenode = cluster.getNameNodeRpc();
        DFSOutputStream out = null;
        try {
            // Create a file and make sure a block is allocated for it.
            out = (DFSOutputStream) (fs.create(file).getWrappedStream());
            out.write(1);
            out.hflush();
            // Create a snapshot that includes the file.
            SnapshotTestHelper.createSnapshot((DistributedFileSystem) fs, new Path("/"), "s1");
            // Grab the block info of this file for later use.
            FSDataInputStream in = null;
            ExtendedBlock oldBlock = null;
            try {
                in = fs.open(file);
                oldBlock = DFSTestUtil.getAllBlocks(in).get(0).getBlock();
            } finally {
                IOUtils.closeStream(in);
            }
            // Allocate a new block ID/gen stamp so we can simulate pipeline
            // recovery.
            String clientName = ((DistributedFileSystem) fs).getClient().getClientName();
            LocatedBlock newLocatedBlock = namenode.updateBlockForPipeline(oldBlock, clientName);
            ExtendedBlock newBlock = new ExtendedBlock(oldBlock.getBlockPoolId(), oldBlock.getBlockId(), oldBlock.getNumBytes(), newLocatedBlock.getBlock().getGenerationStamp());
            // Delete the file from the present FS. It will still exist the
            // previously-created snapshot. This will log an OP_DELETE for the
            // file in question.
            fs.delete(file, true);
            // logged for the file in question.
            try {
                namenode.updatePipeline(clientName, oldBlock, newBlock, newLocatedBlock.getLocations(), newLocatedBlock.getStorageIDs());
            } catch (IOException ioe) {
                // normal
                assertExceptionContains("does not exist or it is not under construction", ioe);
            }
            // Make sure the NN can restart with the edit logs as we have them now.
            cluster.restartNameNode(true);
        } finally {
            IOUtils.closeStream(out);
        }
    } finally {
        cluster.shutdown();
    }
}

Example 5 with DFSOutputStream

use of org.apache.hadoop.hdfs.DFSOutputStream in project hadoop by apache.

the class TestFSImage method testPersistHelper.

private void testPersistHelper(Configuration conf) throws IOException {
    MiniDFSCluster cluster = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).build();
        cluster.waitActive();
        FSNamesystem fsn = cluster.getNamesystem();
        DistributedFileSystem fs = cluster.getFileSystem();
        final Path dir = new Path("/abc/def");
        final Path file1 = new Path(dir, "f1");
        final Path file2 = new Path(dir, "f2");
        // create an empty file f1
        fs.create(file1).close();
        // create an under-construction file f2
        FSDataOutputStream out = fs.create(file2);
        out.writeBytes("hello");
        ((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet.of(SyncFlag.UPDATE_LENGTH));
        // checkpoint
        fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
        fs.saveNamespace();
        fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
        cluster.restartNameNode();
        cluster.waitActive();
        fs = cluster.getFileSystem();
        assertTrue(fs.isDirectory(dir));
        assertTrue(fs.exists(file1));
        assertTrue(fs.exists(file2));
        // check internals of file2
        INodeFile file2Node = fsn.dir.getINode4Write(file2.toString()).asFile();
        assertEquals("hello".length(), file2Node.computeFileSize());
        assertTrue(file2Node.isUnderConstruction());
        BlockInfo[] blks = file2Node.getBlocks();
        assertEquals(1, blks.length);
        assertEquals(BlockUCState.UNDER_CONSTRUCTION, blks[0].getBlockUCState());
        // check lease manager
        Lease lease = fsn.leaseManager.getLease(file2Node);
        Assert.assertNotNull(lease);
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}