Examples with FileSystem org.apache.flink.core.fs.FileSystem used on opensource projects

Example 91 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class CheckpointStateOutputStreamTest method testEmptyState.

// ------------------------------------------------------------------------
// Tests
// ------------------------------------------------------------------------
/**
 * Validates that even empty streams create a file and a file state handle.
 */
@Test
public void testEmptyState() throws Exception {
    final FileSystem fs = FileSystem.getLocalFileSystem();
    final Path folder = baseFolder();
    final String fileName = "myFileName";
    final Path filePath = new Path(folder, fileName);
    final FileStateHandle handle;
    try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
        handle = closeAndGetResult(stream);
    }
    // must have created a handle
    assertNotNull(handle);
    assertEquals(filePath, handle.getFilePath());
    // the pointer path should exist as a directory
    assertTrue(fs.exists(handle.getFilePath()));
    assertFalse(fs.getFileStatus(filePath).isDir());
    // the contents should be empty
    try (FSDataInputStream in = handle.openInputStream()) {
        assertEquals(-1, in.read());
    }
}

Example 92 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class CheckpointStateOutputStreamTest method testCleanupWhenClosingStream.

/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {
    final FileSystem fs = FileSystem.getLocalFileSystem();
    final Path folder = new Path(tmp.newFolder().toURI());
    final String fileName = "nonCreativeTestFileName";
    final Path path = new Path(folder, fileName);
    // write some test data and close the stream
    try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
        Random rnd = new Random();
        for (int i = 0; i < rnd.nextInt(1000); i++) {
            stream.write(rnd.nextInt(100));
        }
    }
    assertFalse(fs.exists(path));
}

Example 93 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class FsCheckpointMetadataOutputStreamTest method testCleanupWhenCommitFailed.

@Test
public void testCleanupWhenCommitFailed() throws Exception {
    Path metaDataFilePath = baseFolder();
    if (fileSystem instanceof FsWithoutRecoverableWriter) {
        fileSystem = ((FsWithoutRecoverableWriter) fileSystem).withStreamFactory((path) -> new FailingCloseStream(new File(path.getPath())));
    } else {
        fileSystem = ((FsWithRecoverableWriter) fileSystem).withStreamFactory((path, temp) -> new FailingRecoverableFsStream(new File(path.getPath()), new File(temp.getPath())));
    }
    FsCheckpointMetadataOutputStream stream = createTestStream(metaDataFilePath, fileSystem);
    try {
        stream.closeAndFinalizeCheckpoint();
        fail("Exception expected when committing the meta file.");
    } catch (Exception e) {
    // ignore
    }
    assertFalse(fileSystem.exists(metaDataFilePath));
    if (fileSystem instanceof FsWithoutRecoverableWriter) {
        ((FsWithoutRecoverableWriter) fileSystem).resetStreamFactory();
    } else {
        ((FsWithRecoverableWriter) fileSystem).resetStreamFactory();
    }
}

Example 94 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class FsCheckpointStorageAccessTest method testDirectoriesForExclusiveAndSharedState.

@Test
public void testDirectoriesForExclusiveAndSharedState() throws Exception {
    final FileSystem fs = LocalFileSystem.getSharedInstance();
    final Path checkpointDir = randomTempPath();
    final Path sharedStateDir = randomTempPath();
    FsCheckpointStorageLocation storageLocation = new FsCheckpointStorageLocation(fs, checkpointDir, sharedStateDir, randomTempPath(), CheckpointStorageLocationReference.getDefault(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);
    assertNotEquals(storageLocation.getCheckpointDirectory(), storageLocation.getSharedStateDirectory());
    assertEquals(0, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
    assertEquals(0, fs.listStatus(storageLocation.getSharedStateDirectory()).length);
    // create exclusive state
    FsCheckpointStateOutputStream exclusiveStream = storageLocation.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE);
    exclusiveStream.write(42);
    exclusiveStream.flushToFile();
    StreamStateHandle exclusiveHandle = exclusiveStream.closeAndGetHandle();
    assertEquals(1, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
    assertEquals(0, fs.listStatus(storageLocation.getSharedStateDirectory()).length);
    // create shared state
    FsCheckpointStateOutputStream sharedStream = storageLocation.createCheckpointStateOutputStream(CheckpointedStateScope.SHARED);
    sharedStream.write(42);
    sharedStream.flushToFile();
    StreamStateHandle sharedHandle = sharedStream.closeAndGetHandle();
    assertEquals(1, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
    assertEquals(1, fs.listStatus(storageLocation.getSharedStateDirectory()).length);
    // drop state
    exclusiveHandle.discardState();
    sharedHandle.discardState();
}

Example 95 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class FsCheckpointStateOutputStreamTest method testCleanupWhenClosingStream.

/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {
    final FileSystem fs = mock(FileSystem.class);
    final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);
    final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
    when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);
    CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(Path.fromLocalFile(tempDir.newFolder()), fs, 4, 0, relativePaths);
    // this should create the underlying file stream
    stream.write(new byte[] { 1, 2, 3, 4, 5 });
    verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));
    stream.close();
    verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}