Examples with FSDataOutputStream org.apache.hadoop.fs.FSDataOutputStream used on opensource projects

Example 66 with FSDataOutputStream

use of org.apache.hadoop.fs.FSDataOutputStream in project hadoop by apache.

the class TestFileConcurrentReader method runTestUnfinishedBlockCRCError.

private void runTestUnfinishedBlockCRCError(final boolean transferToAllowed, final SyncType syncType, final int writeSize, Configuration conf) throws IOException {
    conf.setBoolean(DFSConfigKeys.DFS_DATANODE_TRANSFERTO_ALLOWED_KEY, transferToAllowed);
    init(conf);
    final Path file = new Path("/block-being-written-to");
    final int numWrites = 2000;
    final AtomicBoolean writerDone = new AtomicBoolean(false);
    final AtomicBoolean writerStarted = new AtomicBoolean(false);
    final AtomicBoolean error = new AtomicBoolean(false);
    final Thread writer = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                FSDataOutputStream outputStream = fileSystem.create(file);
                if (syncType == SyncType.APPEND) {
                    outputStream.close();
                    outputStream = fileSystem.append(file);
                }
                try {
                    for (int i = 0; !error.get() && i < numWrites; i++) {
                        final byte[] writeBuf = DFSTestUtil.generateSequentialBytes(i * writeSize, writeSize);
                        outputStream.write(writeBuf);
                        if (syncType == SyncType.SYNC) {
                            outputStream.hflush();
                        }
                        writerStarted.set(true);
                    }
                } catch (IOException e) {
                    error.set(true);
                    LOG.error("error writing to file", e);
                } finally {
                    outputStream.close();
                }
                writerDone.set(true);
            } catch (Exception e) {
                LOG.error("error in writer", e);
                throw new RuntimeException(e);
            }
        }
    });
    Thread tailer = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                long startPos = 0;
                while (!writerDone.get() && !error.get()) {
                    if (writerStarted.get()) {
                        try {
                            startPos = tailFile(file, startPos);
                        } catch (IOException e) {
                            LOG.error(String.format("error tailing file %s", file), e);
                            throw new RuntimeException(e);
                        }
                    }
                }
            } catch (RuntimeException e) {
                if (e.getCause() instanceof ChecksumException) {
                    error.set(true);
                }
                writer.interrupt();
                LOG.error("error in tailer", e);
                throw e;
            }
        }
    });
    writer.start();
    tailer.start();
    try {
        writer.join();
        tailer.join();
        assertFalse("error occurred, see log above", error.get());
    } catch (InterruptedException e) {
        LOG.info("interrupted waiting for writer or tailer to complete");
        Thread.currentThread().interrupt();
    }
}

Example 67 with FSDataOutputStream

use of org.apache.hadoop.fs.FSDataOutputStream in project hadoop by apache.

the class TestFileCorruption method testSetReplicationWhenBatchIBR.

@Test
public void testSetReplicationWhenBatchIBR() throws Exception {
    Configuration conf = new HdfsConfiguration();
    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 100);
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INCREMENTAL_INTERVAL_MSEC_KEY, 30000);
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_FILE_CLOSE_NUM_COMMITTED_ALLOWED_KEY, 1);
    DistributedFileSystem dfs;
    try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build()) {
        // 1024 Bytes each time
        final int bufferSize = 1024;
        byte[] outBuffer = new byte[bufferSize];
        dfs = cluster.getFileSystem();
        String fileName = "/testSetRep1";
        Path filePath = new Path(fileName);
        FSDataOutputStream out = dfs.create(filePath);
        out.write(outBuffer, 0, bufferSize);
        out.close();
        //sending the FBR to Delay next IBR
        cluster.triggerBlockReports();
        GenericTestUtils.waitFor(new Supplier<Boolean>() {

            @Override
            public Boolean get() {
                try {
                    cluster.triggerBlockReports();
                    if (cluster.getNamesystem().getBlocksTotal() == 1) {
                        return true;
                    }
                } catch (Exception e) {
                // Ignore the exception
                }
                return false;
            }
        }, 10, 3000);
        fileName = "/testSetRep2";
        filePath = new Path(fileName);
        out = dfs.create(filePath);
        out.write(outBuffer, 0, bufferSize);
        out.close();
        dfs.setReplication(filePath, (short) 10);
        // underreplicated Blocks should be one after setrep
        GenericTestUtils.waitFor(new Supplier<Boolean>() {

            @Override
            public Boolean get() {
                try {
                    return cluster.getNamesystem().getBlockManager().getUnderReplicatedBlocksCount() == 1;
                } catch (Exception e) {
                    e.printStackTrace();
                    return false;
                }
            }
        }, 10, 3000);
        assertEquals(0, cluster.getNamesystem().getBlockManager().getMissingBlocksCount());
    }
}

Example 68 with FSDataOutputStream

use of org.apache.hadoop.fs.FSDataOutputStream in project hadoop by apache.

the class TestFileCreation method testFsClose.

// test closing file system before all file handles are closed.
@Test
public void testFsClose() throws Exception {
    System.out.println("test file system close start");
    final int DATANODE_NUM = 3;
    Configuration conf = new HdfsConfiguration();
    // create cluster
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(DATANODE_NUM).build();
    DistributedFileSystem dfs = null;
    try {
        cluster.waitActive();
        dfs = cluster.getFileSystem();
        // create a new file.
        final String f = DIR + "foofs";
        final Path fpath = new Path(f);
        FSDataOutputStream out = TestFileCreation.createFile(dfs, fpath, DATANODE_NUM);
        out.write("something".getBytes());
        // close file system without closing file
        dfs.close();
    } finally {
        System.out.println("testFsClose successful");
        cluster.shutdown();
    }
}

Example 69 with FSDataOutputStream

use of org.apache.hadoop.fs.FSDataOutputStream in project hadoop by apache.

the class TestFileCreation method testFsCloseAfterClusterShutdown.

// test closing file after cluster is shutdown
@Test
public void testFsCloseAfterClusterShutdown() throws IOException {
    System.out.println("test testFsCloseAfterClusterShutdown start");
    final int DATANODE_NUM = 3;
    Configuration conf = new HdfsConfiguration();
    conf.setInt(DFS_NAMENODE_REPLICATION_MIN_KEY, 3);
    // hdfs timeout is default 60 seconds
    conf.setBoolean("ipc.client.ping", false);
    // hdfs timeout is now 10 second
    conf.setInt("ipc.ping.interval", 10000);
    // create cluster
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(DATANODE_NUM).build();
    DistributedFileSystem dfs = null;
    try {
        cluster.waitActive();
        dfs = cluster.getFileSystem();
        // create a new file.
        final String f = DIR + "testFsCloseAfterClusterShutdown";
        final Path fpath = new Path(f);
        FSDataOutputStream out = TestFileCreation.createFile(dfs, fpath, DATANODE_NUM);
        out.write("something_test".getBytes());
        // ensure that block is allocated
        out.hflush();
        // shutdown last datanode in pipeline.
        cluster.stopDataNode(2);
        // close file. Since we have set the minReplcatio to 3 but have killed one
        // of the three datanodes, the close call will loop until the hdfsTimeout is
        // encountered.
        boolean hasException = false;
        try {
            out.close();
            System.out.println("testFsCloseAfterClusterShutdown: Error here");
        } catch (IOException e) {
            hasException = true;
        }
        assertTrue("Failed to close file after cluster shutdown", hasException);
    } finally {
        System.out.println("testFsCloseAfterClusterShutdown successful");
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 70 with FSDataOutputStream

use of org.apache.hadoop.fs.FSDataOutputStream in project hadoop by apache.

the class TestFileCreation method testDFSClientDeath.

/**
   * Test that all open files are closed when client dies abnormally.
   */
@Test
public void testDFSClientDeath() throws IOException, InterruptedException {
    Configuration conf = new HdfsConfiguration();
    System.out.println("Testing adbornal client death.");
    if (simulatedStorage) {
        SimulatedFSDataset.setFactory(conf);
    }
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
    FileSystem fs = cluster.getFileSystem();
    DistributedFileSystem dfs = (DistributedFileSystem) fs;
    DFSClient dfsclient = dfs.dfs;
    try {
        // create a new file in home directory. Do not close it.
        //
        Path file1 = new Path("/clienttest.dat");
        FSDataOutputStream stm = createFile(fs, file1, 1);
        System.out.println("Created file clienttest.dat");
        // write to file
        writeFile(stm);
        // close the dfsclient before closing the output stream.
        // This should close all existing file.
        dfsclient.close();
        // reopen file system and verify that file exists.
        assertTrue(file1 + " does not exist.", AppendTestUtil.createHdfsWithDifferentUsername(conf).exists(file1));
    } finally {
        cluster.shutdown();
    }
}