Examples with FileSystem org.apache.flink.core.fs.FileSystem used on opensource projects

Example 46 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class AzureFileSystemBehaviorITCase method testSimpleFileWriteAndRead.

@Test
public void testSimpleFileWriteAndRead() throws Exception {
    // 30 secs
    final long deadline = System.nanoTime() + 30_000_000_000L;
    final String testLine = "Hello Upload!";
    final Path path = new Path(getBasePath() + "/test.txt");
    final FileSystem fs = path.getFileSystem();
    try {
        try (FSDataOutputStream out = fs.create(path, FileSystem.WriteMode.OVERWRITE);
            OutputStreamWriter writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) {
            writer.write(testLine);
        }
        // just in case, wait for the path to exist
        checkPathEventualExistence(fs, path, true, deadline);
        try (FSDataInputStream in = fs.open(path);
            InputStreamReader ir = new InputStreamReader(in, StandardCharsets.UTF_8);
            BufferedReader reader = new BufferedReader(ir)) {
            String line = reader.readLine();
            assertEquals(testLine, line);
        }
    } finally {
        fs.delete(path, false);
    }
    // now file must be gone
    checkPathEventualExistence(fs, path, false, deadline);
}

Example 47 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class FileCacheDirectoriesTest method testDirectoryDownloaded.

private void testDirectoryDownloaded(DistributedCache.DistributedCacheEntry entry) throws Exception {
    JobID jobID = new JobID();
    ExecutionAttemptID attemptID = new ExecutionAttemptID();
    // copy / create the file
    final String fileName = "test_file";
    Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);
    final Path dstPath = copyResult.get();
    final FileSystem fs = dstPath.getFileSystem();
    final FileStatus fileStatus = fs.getFileStatus(dstPath);
    assertTrue(fileStatus.isDir());
    final Path cacheFile = new Path(dstPath, "cacheFile");
    assertTrue(fs.exists(cacheFile));
    final String actualContent = FileUtils.readFileUtf8(new File(cacheFile.getPath()));
    assertEquals(testFileContent, actualContent);
}

Example 48 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class CheckpointStateOutputStreamTest method testCleanupWhenFailingCloseAndGetHandle.

/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
    final Path folder = new Path(tmp.newFolder().toURI());
    final String fileName = "test_name";
    final Path filePath = new Path(folder, fileName);
    final FileSystem fs = spy(new FsWithoutRecoverableWriter((path) -> new FailingCloseStream(new File(path.getPath()))));
    FSDataOutputStream stream = createTestStream(fs, folder, fileName);
    stream.write(new byte[] { 1, 2, 3, 4, 5 });
    try {
        closeAndGetResult(stream);
        fail("Expected IOException");
    } catch (IOException ignored) {
    // expected exception
    }
    verify(fs).delete(filePath, false);
}

Example 49 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class CheckpointStateOutputStreamTest method testWriteAndRead.

/**
 * Simple write and read test.
 */
@Test
public void testWriteAndRead() throws Exception {
    final FileSystem fs = FileSystem.getLocalFileSystem();
    final Path folder = baseFolder();
    final String fileName = "fooBarName";
    final Random rnd = new Random();
    final byte[] data = new byte[1694523];
    // write the data (mixed single byte writes and array writes)
    final FileStateHandle handle;
    try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
        for (int i = 0; i < data.length; ) {
            if (rnd.nextBoolean()) {
                stream.write(data[i++]);
            } else {
                int len = rnd.nextInt(Math.min(data.length - i, 32));
                stream.write(data, i, len);
                i += len;
            }
        }
        handle = closeAndGetResult(stream);
    }
    // (1) stream from handle must hold the contents
    try (FSDataInputStream in = handle.openInputStream()) {
        byte[] buffer = new byte[data.length];
        readFully(in, buffer);
        assertArrayEquals(data, buffer);
    }
    // (2) the pointer must point to a file with that contents
    try (FSDataInputStream in = fs.open(handle.getFilePath())) {
        byte[] buffer = new byte[data.length];
        readFully(in, buffer);
        assertArrayEquals(data, buffer);
    }
}

Example 50 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class CheckpointStateOutputStreamTest method testCloseDoesNotLock.

/**
 * This test validates that a close operation can happen even while a 'closeAndGetHandle()' call
 * is in progress.
 *
 * <p>That behavior is essential for fast cancellation (concurrent cleanup).
 */
@Test
public void testCloseDoesNotLock() throws Exception {
    final Path folder = new Path(tmp.newFolder().toURI());
    final String fileName = "this-is-ignored-anyways.file";
    final FileSystem fileSystem = spy(new FsWithoutRecoverableWriter((path) -> new BlockerStream()));
    final FSDataOutputStream checkpointStream = createTestStream(fileSystem, folder, fileName);
    final OneShotLatch sync = new OneShotLatch();
    final CheckedThread thread = new CheckedThread() {

        @Override
        public void go() throws Exception {
            sync.trigger();
            // that call should now block, because it accesses the position
            closeAndGetResult(checkpointStream);
        }
    };
    thread.start();
    sync.await();
    checkpointStream.close();
    // it is not important for this test, important is that the thread does not freeze/lock up
    try {
        thread.sync();
    } catch (IOException ignored) {
    }
}