Examples with FileSystem org.apache.flink.core.fs.FileSystem used on opensource projects

Example 76 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class FileUtils method compressDirectory.

public static Path compressDirectory(Path directory, Path target) throws IOException {
    FileSystem sourceFs = directory.getFileSystem();
    FileSystem targetFs = target.getFileSystem();
    Path absolutePath = absolutizePath(directory);
    Path absoluteTargetPath = absolutizePath(target);
    try (ZipOutputStream out = new ZipOutputStream(targetFs.create(absoluteTargetPath, FileSystem.WriteMode.NO_OVERWRITE))) {
        addToZip(absolutePath, sourceFs, absolutePath.getParent(), out);
    }
    return target;
}

Example 77 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class FileUtils method expandDirectory.

public static Path expandDirectory(Path file, Path targetDirectory) throws IOException {
    FileSystem sourceFs = file.getFileSystem();
    FileSystem targetFs = targetDirectory.getFileSystem();
    Path rootDir = null;
    try (ZipInputStream zis = new ZipInputStream(sourceFs.open(file))) {
        ZipEntry entry;
        while ((entry = zis.getNextEntry()) != null) {
            Path relativePath = new Path(entry.getName());
            if (rootDir == null) {
                // the first entry contains the name of the original directory that was zipped
                rootDir = relativePath;
            }
            Path newFile = new Path(targetDirectory, relativePath);
            if (entry.isDirectory()) {
                targetFs.mkdirs(newFile);
            } else {
                try (FSDataOutputStream fileStream = targetFs.create(newFile, FileSystem.WriteMode.NO_OVERWRITE)) {
                    // do not close the streams here as it prevents access to further zip
                    // entries
                    IOUtils.copyBytes(zis, fileStream, false);
                }
            }
            zis.closeEntry();
        }
    }
    return new Path(targetDirectory, rootDir);
}

Example 78 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class LocalFileSystemTest method testRenameNonExistingFile.

@Test
public void testRenameNonExistingFile() throws IOException {
    final FileSystem fs = FileSystem.getLocalFileSystem();
    final File srcFile = new File(temporaryFolder.newFolder(), "someFile.txt");
    final File destFile = new File(temporaryFolder.newFolder(), "target");
    final Path srcFilePath = new Path(srcFile.toURI());
    final Path destFilePath = new Path(destFile.toURI());
    // this cannot succeed because the source file does not exist
    assertFalse(fs.rename(srcFilePath, destFilePath));
}

Example 79 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class LocalFileSystemTest method testRenamePath.

@Test
public void testRenamePath() throws IOException {
    final File rootDirectory = temporaryFolder.newFolder();
    // create a file /root/src/B/test.csv
    final File srcDirectory = new File(new File(rootDirectory, "src"), "B");
    assertTrue(srcDirectory.mkdirs());
    final File srcFile = new File(srcDirectory, "test.csv");
    assertTrue(srcFile.createNewFile());
    // Move/rename B and its content to /root/dst/A
    final File destDirectory = new File(new File(rootDirectory, "dst"), "B");
    final File destFile = new File(destDirectory, "test.csv");
    final Path srcDirPath = new Path(srcDirectory.toURI());
    final Path srcFilePath = new Path(srcFile.toURI());
    final Path destDirPath = new Path(destDirectory.toURI());
    final Path destFilePath = new Path(destFile.toURI());
    FileSystem fs = FileSystem.getLocalFileSystem();
    // pre-conditions: /root/src/B exists but /root/dst/B does not
    assertTrue(fs.exists(srcDirPath));
    assertFalse(fs.exists(destDirPath));
    // do the move/rename: /root/src/B -> /root/dst/
    assertTrue(fs.rename(srcDirPath, destDirPath));
    // post-conditions: /root/src/B doesn't exists, /root/dst/B/test.csv has been created
    assertTrue(fs.exists(destFilePath));
    assertFalse(fs.exists(srcDirPath));
    // re-create source file and test overwrite
    assertTrue(srcDirectory.mkdirs());
    assertTrue(srcFile.createNewFile());
    // overwrite the destination file
    assertTrue(fs.rename(srcFilePath, destFilePath));
    // post-conditions: now only the src file has been moved
    assertFalse(fs.exists(srcFilePath));
    assertTrue(fs.exists(srcDirPath));
    assertTrue(fs.exists(destFilePath));
}

Example 80 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class LocalFileSystemTest method testRenameToNonEmptyTargetDir.

@Test
public void testRenameToNonEmptyTargetDir() throws IOException {
    final FileSystem fs = FileSystem.getLocalFileSystem();
    // a source folder with a file
    final File srcFolder = temporaryFolder.newFolder();
    final File srcFile = new File(srcFolder, "someFile.txt");
    assertTrue(srcFile.createNewFile());
    // a non-empty destination folder
    final File dstFolder = temporaryFolder.newFolder();
    final File dstFile = new File(dstFolder, "target");
    assertTrue(dstFile.createNewFile());
    // this cannot succeed because the destination folder is not empty
    assertFalse(fs.rename(new Path(srcFolder.toURI()), new Path(dstFolder.toURI())));
    // retry after deleting the occupying target file
    assertTrue(dstFile.delete());
    assertTrue(fs.rename(new Path(srcFolder.toURI()), new Path(dstFolder.toURI())));
    assertTrue(new File(dstFolder, srcFile.getName()).exists());
}