Examples with FileSystem org.apache.flink.core.fs.FileSystem used on opensource projects

Example 21 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class FileOutputFormat method initializeGlobal.

/**
	 * Initialization of the distributed file system if it is used.
	 *
	 * @param parallelism The task parallelism.
	 */
@Override
public void initializeGlobal(int parallelism) throws IOException {
    final Path path = getOutputFilePath();
    final FileSystem fs = path.getFileSystem();
    // only distributed file systems can be initialized at start-up time.
    if (fs.isDistributedFS()) {
        final WriteMode writeMode = getWriteMode();
        final OutputDirectoryMode outDirMode = getOutputDirectoryMode();
        if (parallelism == 1 && outDirMode == OutputDirectoryMode.PARONLY) {
            // prepare distributed output path
            if (!fs.initOutPathDistFS(path, writeMode, false)) {
                // output preparation failed! Cancel task.
                throw new IOException("Output path could not be initialized.");
            }
        } else {
            // only distributed file systems can be initialized at start-up time.
            if (!fs.initOutPathDistFS(path, writeMode, true)) {
                throw new IOException("Output directory could not be created.");
            }
        }
    }
}

Example 22 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class FilesystemSchemeConfigTest method testSettingFilesystemScheme.

private void testSettingFilesystemScheme(boolean useDefaultScheme, String configFileScheme, boolean useExplicitScheme) {
    final File tmpDir = getTmpDir();
    final File confFile = new File(tmpDir, GlobalConfiguration.FLINK_CONF_FILENAME);
    try {
        confFile.createNewFile();
    } catch (IOException e) {
        throw new RuntimeException("Couldn't create file", e);
    }
    final File testFile = new File(tmpDir.getAbsolutePath() + File.separator + "testing.txt");
    try {
        try {
            final PrintWriter pw1 = new PrintWriter(confFile);
            if (!useDefaultScheme) {
                pw1.println(configFileScheme);
            }
            pw1.close();
            final PrintWriter pwTest = new PrintWriter(testFile);
            pwTest.close();
        } catch (FileNotFoundException e) {
            fail(e.getMessage());
        }
        Configuration conf = GlobalConfiguration.loadConfiguration(tmpDir.getAbsolutePath());
        try {
            FileSystem.setDefaultScheme(conf);
            // remove the scheme.
            String noSchemePath = testFile.toURI().getPath();
            URI uri = new URI(noSchemePath);
            // check if the scheme == null (so that we get the configuration one.
            assertTrue(uri.getScheme() == null);
            // get the filesystem with the default scheme as set in the confFile1
            FileSystem fs = useExplicitScheme ? FileSystem.get(testFile.toURI()) : FileSystem.get(uri);
            assertTrue(fs.exists(new Path(noSchemePath)));
        } catch (IOException e) {
            fail(e.getMessage());
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    } finally {
        try {
            // clear the default scheme set in the FileSystem class.
            // we do it through reflection to avoid creating a publicly
            // accessible method, which could also be wrongly used by users.
            Field f = FileSystem.class.getDeclaredField("defaultScheme");
            f.setAccessible(true);
            f.set(null, null);
        } catch (IllegalAccessException | NoSuchFieldException e) {
            e.printStackTrace();
            fail("Cannot reset default scheme: " + e.getMessage());
        }
        confFile.delete();
        testFile.delete();
        tmpDir.delete();
    }
}

Example 23 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class LocalFileSystemTest method testDeletePathIfEmpty.

/**
	 * Test that {@link FileUtils#deletePathIfEmpty(FileSystem, Path)} deletes the path if it is
	 * empty. A path can only be empty if it is a directory which does not contain any
	 * files/directories.
	 */
@Test
public void testDeletePathIfEmpty() throws IOException {
    File file = temporaryFolder.newFile();
    File directory = temporaryFolder.newFolder();
    File directoryFile = new File(directory, UUID.randomUUID().toString());
    assertTrue(directoryFile.createNewFile());
    Path filePath = new Path(file.toURI());
    Path directoryPath = new Path(directory.toURI());
    Path directoryFilePath = new Path(directoryFile.toURI());
    FileSystem fs = FileSystem.getLocalFileSystem();
    // verify that the files have been created
    assertTrue(fs.exists(filePath));
    assertTrue(fs.exists(directoryFilePath));
    // delete the single file
    assertFalse(FileUtils.deletePathIfEmpty(fs, filePath));
    assertTrue(fs.exists(filePath));
    // try to delete the non-empty directory
    assertFalse(FileUtils.deletePathIfEmpty(fs, directoryPath));
    assertTrue(fs.exists(directoryPath));
    // delete the file contained in the directory
    assertTrue(fs.delete(directoryFilePath, false));
    // now the deletion should work
    assertTrue(FileUtils.deletePathIfEmpty(fs, directoryPath));
    assertFalse(fs.exists(directoryPath));
}

Example 24 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class TestFileSystem method registerTestFileSysten.

public static void registerTestFileSysten() throws Exception {
    Class<FileSystem> fsClass = FileSystem.class;
    Field dirField = fsClass.getDeclaredField("FSDIRECTORY");
    dirField.setAccessible(true);
    @SuppressWarnings("unchecked") Map<String, String> map = (Map<String, String>) dirField.get(null);
    dirField.setAccessible(false);
    map.put("test", TestFileSystem.class.getName());
}

Example 25 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class HDFSTest method testDeletePathIfEmpty.

/**
	 * Test that {@link FileUtils#deletePathIfEmpty(FileSystem, Path)} deletes the path if it is
	 * empty. A path can only be empty if it is a directory which does not contain any
	 * files/directories.
	 */
@Test
public void testDeletePathIfEmpty() throws IOException {
    final Path basePath = new Path(hdfsURI);
    final Path directory = new Path(basePath, UUID.randomUUID().toString());
    final Path directoryFile = new Path(directory, UUID.randomUUID().toString());
    final Path singleFile = new Path(basePath, UUID.randomUUID().toString());
    FileSystem fs = basePath.getFileSystem();
    fs.mkdirs(directory);
    byte[] data = "HDFSTest#testDeletePathIfEmpty".getBytes(ConfigConstants.DEFAULT_CHARSET);
    for (Path file : Arrays.asList(singleFile, directoryFile)) {
        org.apache.flink.core.fs.FSDataOutputStream outputStream = fs.create(file, true);
        outputStream.write(data);
        outputStream.close();
    }
    // verify that the files have been created
    assertTrue(fs.exists(singleFile));
    assertTrue(fs.exists(directoryFile));
    // delete the single file
    assertFalse(FileUtils.deletePathIfEmpty(fs, singleFile));
    assertTrue(fs.exists(singleFile));
    // try to delete the non-empty directory
    assertFalse(FileUtils.deletePathIfEmpty(fs, directory));
    assertTrue(fs.exists(directory));
    // delete the file contained in the directory
    assertTrue(fs.delete(directoryFile, false));
    // now the deletion should work
    assertTrue(FileUtils.deletePathIfEmpty(fs, directory));
    assertFalse(fs.exists(directory));
}