Examples with FSDataInputStream org.apache.hadoop.fs.FSDataInputStream used on opensource projects

Example 51 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class TestPersistBlocks method testRestartWithAppend.

@Test
public void testRestartWithAppend() throws IOException {
    final Configuration conf = new HdfsConfiguration();
    // Turn off persistent IPC, so that the DFSClient can survive NN restart
    conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);
    MiniDFSCluster cluster = null;
    FSDataOutputStream stream;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
        FileSystem fs = cluster.getFileSystem();
        DFSUtilClient.getNNAddress(conf).getPort();
        // Creating a file with 4096 blockSize to write multiple blocks
        stream = fs.create(FILE_PATH, true, BLOCK_SIZE, (short) 1, BLOCK_SIZE);
        stream.write(DATA_BEFORE_RESTART, 0, DATA_BEFORE_RESTART.length / 2);
        stream.close();
        stream = fs.append(FILE_PATH, BLOCK_SIZE);
        stream.write(DATA_BEFORE_RESTART, DATA_BEFORE_RESTART.length / 2, DATA_BEFORE_RESTART.length / 2);
        stream.close();
        assertEquals(DATA_BEFORE_RESTART.length, fs.getFileStatus(FILE_PATH).getLen());
        cluster.restartNameNode();
        assertEquals(DATA_BEFORE_RESTART.length, fs.getFileStatus(FILE_PATH).getLen());
        FSDataInputStream readStream = fs.open(FILE_PATH);
        try {
            byte[] verifyBuf = new byte[DATA_BEFORE_RESTART.length];
            IOUtils.readFully(readStream, verifyBuf, 0, verifyBuf.length);
            assertArrayEquals(DATA_BEFORE_RESTART, verifyBuf);
        } finally {
            IOUtils.closeStream(readStream);
        }
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 52 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class TestPersistBlocks method testRestartWithPartialBlockHflushed.

@Test
public void testRestartWithPartialBlockHflushed() throws IOException {
    final Configuration conf = new HdfsConfiguration();
    // Turn off persistent IPC, so that the DFSClient can survive NN restart
    conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);
    MiniDFSCluster cluster = null;
    FSDataOutputStream stream;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
        FileSystem fs = cluster.getFileSystem();
        DFSUtilClient.getNNAddress(conf).getPort();
        // Creating a file with 4096 blockSize to write multiple blocks
        stream = fs.create(FILE_PATH, true, BLOCK_SIZE, (short) 1, BLOCK_SIZE);
        stream.write(DATA_BEFORE_RESTART);
        stream.write((byte) 1);
        stream.hflush();
        // explicitly do NOT close the file before restarting the NN.
        cluster.restartNameNode();
        // this will fail if the final block of the file is prematurely COMPLETEd
        stream.write((byte) 2);
        stream.hflush();
        stream.close();
        assertEquals(DATA_BEFORE_RESTART.length + 2, fs.getFileStatus(FILE_PATH).getLen());
        FSDataInputStream readStream = fs.open(FILE_PATH);
        try {
            byte[] verifyBuf = new byte[DATA_BEFORE_RESTART.length + 2];
            IOUtils.readFully(readStream, verifyBuf, 0, verifyBuf.length);
            byte[] expectedBuf = new byte[DATA_BEFORE_RESTART.length + 2];
            System.arraycopy(DATA_BEFORE_RESTART, 0, expectedBuf, 0, DATA_BEFORE_RESTART.length);
            System.arraycopy(new byte[] { 1, 2 }, 0, expectedBuf, DATA_BEFORE_RESTART.length, 2);
            assertArrayEquals(expectedBuf, verifyBuf);
        } finally {
            IOUtils.closeStream(readStream);
        }
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 53 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class TestPersistBlocks method testRestartDfs.

/** check if DFS remains in proper condition after a restart
   * @param useFlush - if true then flush is used instead of sync (ie hflush)
   */
void testRestartDfs(boolean useFlush) throws Exception {
    final Configuration conf = new HdfsConfiguration();
    // Turn off persistent IPC, so that the DFSClient can survive NN restart
    conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);
    MiniDFSCluster cluster = null;
    long len = 0;
    FSDataOutputStream stream;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
        FileSystem fs = cluster.getFileSystem();
        // Creating a file with 4096 blockSize to write multiple blocks
        stream = fs.create(FILE_PATH, true, BLOCK_SIZE, (short) 1, BLOCK_SIZE);
        stream.write(DATA_BEFORE_RESTART);
        if (useFlush)
            stream.flush();
        else
            stream.hflush();
        // Wait for at least a few blocks to get through
        while (len <= BLOCK_SIZE) {
            FileStatus status = fs.getFileStatus(FILE_PATH);
            len = status.getLen();
            Thread.sleep(100);
        }
        // explicitly do NOT close the file.
        cluster.restartNameNode();
        // Check that the file has no less bytes than before the restart
        // This would mean that blocks were successfully persisted to the log
        FileStatus status = fs.getFileStatus(FILE_PATH);
        assertTrue("Length too short: " + status.getLen(), status.getLen() >= len);
        // And keep writing (ensures that leases are also persisted correctly)
        stream.write(DATA_AFTER_RESTART);
        stream.close();
        // Verify that the data showed up, both from before and after the restart.
        FSDataInputStream readStream = fs.open(FILE_PATH);
        try {
            byte[] verifyBuf = new byte[DATA_BEFORE_RESTART.length];
            IOUtils.readFully(readStream, verifyBuf, 0, verifyBuf.length);
            assertArrayEquals(DATA_BEFORE_RESTART, verifyBuf);
            IOUtils.readFully(readStream, verifyBuf, 0, verifyBuf.length);
            assertArrayEquals(DATA_AFTER_RESTART, verifyBuf);
        } finally {
            IOUtils.closeStream(readStream);
        }
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 54 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class TestPread method datanodeRestartTest.

// test pread can survive datanode restarts
private void datanodeRestartTest(MiniDFSCluster cluster, FileSystem fileSys, Path name) throws IOException {
    // don't survive datanode restarts.
    if (simulatedStorage) {
        return;
    }
    int numBlocks = 1;
    assertTrue(numBlocks <= HdfsClientConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT);
    byte[] expected = new byte[numBlocks * blockSize];
    Random rand = new Random(seed);
    rand.nextBytes(expected);
    byte[] actual = new byte[numBlocks * blockSize];
    FSDataInputStream stm = fileSys.open(name);
    // read a block and get block locations cached as a result
    stm.readFully(0, actual);
    checkAndEraseData(actual, 0, expected, "Pread Datanode Restart Setup");
    // restart all datanodes. it is expected that they will
    // restart on different ports, hence, cached block locations
    // will no longer work.
    assertTrue(cluster.restartDataNodes());
    cluster.waitActive();
    // verify the block can be read again using the same InputStream 
    // (via re-fetching of block locations from namenode). there is a 
    // 3 sec sleep in chooseDataNode(), which can be shortened for 
    // this test if configurable.
    stm.readFully(0, actual);
    checkAndEraseData(actual, 0, expected, "Pread Datanode Restart Test");
}

Example 55 with FSDataInputStream

use of org.apache.hadoop.fs.FSDataInputStream in project hadoop by apache.

the class TestRead method testEOF.

private void testEOF(MiniDFSCluster cluster, int fileLength) throws IOException {
    FileSystem fs = cluster.getFileSystem();
    Path path = new Path("testEOF." + fileLength);
    DFSTestUtil.createFile(fs, path, fileLength, (short) 1, 0xBEEFBEEF);
    FSDataInputStream fis = fs.open(path);
    ByteBuffer empty = ByteBuffer.allocate(0);
    // A read into an empty bytebuffer at the beginning of the file gives 0.
    Assert.assertEquals(0, fis.read(empty));
    fis.seek(fileLength);
    // A read into an empty bytebuffer at the end of the file gives -1.
    Assert.assertEquals(-1, fis.read(empty));
    if (fileLength > BLOCK_SIZE) {
        fis.seek(fileLength - BLOCK_SIZE + 1);
        ByteBuffer dbb = ByteBuffer.allocateDirect(BLOCK_SIZE);
        Assert.assertEquals(BLOCK_SIZE - 1, fis.read(dbb));
    }
    fis.close();
}