Examples with FSInputStream org.apache.hadoop.fs.FSInputStream used on opensource projects

Example 1 with FSInputStream

use of org.apache.hadoop.fs.FSInputStream in project hadoop by apache.

the class TestDFSUpgradeFromImage method verifyDir.

private void verifyDir(DistributedFileSystem dfs, Path dir, CRC32 overallChecksum) throws IOException {
    FileStatus[] fileArr = dfs.listStatus(dir);
    TreeMap<Path, Boolean> fileMap = new TreeMap<Path, Boolean>();
    for (FileStatus file : fileArr) {
        fileMap.put(file.getPath(), Boolean.valueOf(file.isDirectory()));
    }
    for (Iterator<Path> it = fileMap.keySet().iterator(); it.hasNext(); ) {
        Path path = it.next();
        boolean isDir = fileMap.get(path);
        String pathName = path.toUri().getPath();
        overallChecksum.update(pathName.getBytes());
        if (isDir) {
            verifyDir(dfs, path, overallChecksum);
        } else {
            // this is not a directory. Checksum the file data.
            CRC32 fileCRC = new CRC32();
            FSInputStream in = dfsOpenFileWithRetries(dfs, pathName);
            byte[] buf = new byte[4096];
            int nRead = 0;
            while ((nRead = in.read(buf, 0, buf.length)) > 0) {
                fileCRC.update(buf, 0, nRead);
            }
            verifyChecksum(pathName, fileCRC.getValue());
        }
    }
}

Example 2 with FSInputStream

use of org.apache.hadoop.fs.FSInputStream in project hadoop by apache.

the class TestCopy method testCopyStreamTarget.

@Test
public void testCopyStreamTarget() throws Exception {
    FSDataOutputStream out = mock(FSDataOutputStream.class);
    whenFsCreate().thenReturn(out);
    when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat);
    when(mockFs.rename(eq(tmpPath), eq(path))).thenReturn(true);
    FSInputStream in = mock(FSInputStream.class);
    when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1);
    tryCopyStream(in, true);
    verify(mockFs, never()).delete(eq(path), anyBoolean());
    verify(mockFs).rename(eq(tmpPath), eq(path));
    verify(mockFs, never()).delete(eq(tmpPath), anyBoolean());
    verify(mockFs, never()).close();
}

Example 3 with FSInputStream

use of org.apache.hadoop.fs.FSInputStream in project hadoop by apache.

the class TestCopy method testInterruptedRename.

@Test
public void testInterruptedRename() throws Exception {
    FSDataOutputStream out = mock(FSDataOutputStream.class);
    whenFsCreate().thenReturn(out);
    when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat);
    when(mockFs.rename(eq(tmpPath), eq(path))).thenThrow(new InterruptedIOException());
    FSInputStream in = mock(FSInputStream.class);
    when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1);
    tryCopyStream(in, false);
    verify(mockFs).delete(eq(tmpPath), anyBoolean());
    verify(mockFs).rename(eq(tmpPath), eq(path));
    verify(mockFs, never()).delete(eq(path), anyBoolean());
    verify(mockFs, never()).close();
}

Example 4 with FSInputStream

use of org.apache.hadoop.fs.FSInputStream in project hadoop by apache.

the class TestCopy method testCopyStreamTargetExists.

@Test
public void testCopyStreamTargetExists() throws Exception {
    FSDataOutputStream out = mock(FSDataOutputStream.class);
    whenFsCreate().thenReturn(out);
    when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat);
    // so it's updated as existing
    target.refreshStatus();
    cmd.setOverwrite(true);
    when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat);
    when(mockFs.delete(eq(path), eq(false))).thenReturn(true);
    when(mockFs.rename(eq(tmpPath), eq(path))).thenReturn(true);
    FSInputStream in = mock(FSInputStream.class);
    when(in.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1);
    tryCopyStream(in, true);
    verify(mockFs).delete(eq(path), anyBoolean());
    verify(mockFs).rename(eq(tmpPath), eq(path));
    verify(mockFs, never()).delete(eq(tmpPath), anyBoolean());
    verify(mockFs, never()).close();
}

Example 5 with FSInputStream

use of org.apache.hadoop.fs.FSInputStream in project hadoop by apache.

the class TestCopy method testInterruptedCopyBytes.

@Test
public void testInterruptedCopyBytes() throws Exception {
    FSDataOutputStream out = mock(FSDataOutputStream.class);
    whenFsCreate().thenReturn(out);
    when(mockFs.getFileStatus(eq(tmpPath))).thenReturn(fileStat);
    FSInputStream in = mock(FSInputStream.class);
    // make IOUtils.copyBytes fail
    when(in.read(any(byte[].class), anyInt(), anyInt())).thenThrow(new InterruptedIOException());
    tryCopyStream(in, false);
    verify(mockFs).delete(eq(tmpPath), anyBoolean());
    verify(mockFs, never()).rename(any(Path.class), any(Path.class));
    verify(mockFs, never()).delete(eq(path), anyBoolean());
    verify(mockFs, never()).close();
}