Examples with FileSystem org.apache.flink.core.fs.FileSystem used on opensource projects

Example 51 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class FsCheckpointStorageAccessTest method testResolveCheckpointStorageLocation.

@Test
public void testResolveCheckpointStorageLocation() throws Exception {
    final FileSystem checkpointFileSystem = mock(FileSystem.class);
    final FsCheckpointStorageAccess storage = new FsCheckpointStorageAccess(new TestingPath("hdfs:///checkpoint/", checkpointFileSystem), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);
    final FsCheckpointStorageLocation checkpointStreamFactory = (FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(1L, CheckpointStorageLocationReference.getDefault());
    assertEquals(checkpointFileSystem, checkpointStreamFactory.getFileSystem());
    final CheckpointStorageLocationReference savepointLocationReference = AbstractFsCheckpointStorageAccess.encodePathAsReference(new Path("file:///savepoint/"));
    final FsCheckpointStorageLocation savepointStreamFactory = (FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(2L, savepointLocationReference);
    final FileSystem fileSystem = savepointStreamFactory.getFileSystem();
    assertTrue(fileSystem instanceof LocalFileSystem);
}

Example 52 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class FsCheckpointStateOutputStreamTest method testStreamDoesNotTryToCleanUpParentOnError.

// ------------------------------------------------------------------------
// Not deleting parent directories
// ------------------------------------------------------------------------
/**
 * This test checks that the stream does not check and clean the parent directory when
 * encountering a write error.
 */
@Test
public void testStreamDoesNotTryToCleanUpParentOnError() throws Exception {
    final File directory = tempDir.newFolder();
    // prevent creation of files in that directory
    // this operation does not work reliably on Windows, so we use an "assume" to skip the test
    // is this prerequisite operation is not supported.
    assumeTrue(directory.setWritable(false, true));
    checkDirectoryNotWritable(directory);
    FileSystem fs = spy(FileSystem.getLocalFileSystem());
    FsCheckpointStateOutputStream stream1 = new FsCheckpointStateOutputStream(Path.fromLocalFile(directory), fs, 1024, 1, relativePaths);
    FsCheckpointStateOutputStream stream2 = new FsCheckpointStateOutputStream(Path.fromLocalFile(directory), fs, 1024, 1, relativePaths);
    stream1.write(new byte[61]);
    stream2.write(new byte[61]);
    try {
        stream1.closeAndGetHandle();
        fail("this should fail with an exception");
    } catch (IOException ignored) {
    }
    stream2.close();
    // no delete call must have happened
    verify(fs, times(0)).delete(any(Path.class), anyBoolean());
    // the directory must still exist as a proper directory
    assertTrue(directory.exists());
    assertTrue(directory.isDirectory());
}

Example 53 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class FsCheckpointStateOutputStreamTest method testCleanupWhenFailingCloseAndGetHandle.

/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
    final FileSystem fs = mock(FileSystem.class);
    final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);
    final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
    when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);
    doThrow(new IOException("Test IOException.")).when(outputStream).close();
    CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(Path.fromLocalFile(tempDir.newFolder()), fs, 4, 0, relativePaths);
    // this should create the underlying file stream
    stream.write(new byte[] { 1, 2, 3, 4, 5 });
    verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));
    try {
        stream.closeAndGetHandle();
        fail("Expected IOException");
    } catch (IOException ioE) {
    // expected exception
    }
    verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}

Example 54 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class FsStateBackendEntropyTest method testEntropyInjection.

@Test
public void testEntropyInjection() throws Exception {
    final int fileSizeThreshold = 1024;
    final FileSystem fs = new TestEntropyAwareFs();
    final Path checkpointDir = new Path(Path.fromLocalFile(tmp.newFolder()), ENTROPY_MARKER + "/checkpoints");
    final String checkpointDirStr = checkpointDir.toString();
    final FsCheckpointStorageAccess storage = new FsCheckpointStorageAccess(fs, checkpointDir, null, new JobID(), fileSizeThreshold, 4096);
    storage.initializeBaseLocationsForCheckpoint();
    final FsCheckpointStorageLocation location = (FsCheckpointStorageLocation) storage.initializeLocationForCheckpoint(96562);
    assertThat(location.getCheckpointDirectory().toString(), startsWith(checkpointDirStr));
    assertThat(location.getSharedStateDirectory().toString(), startsWith(checkpointDirStr));
    assertThat(location.getTaskOwnedStateDirectory().toString(), startsWith(checkpointDirStr));
    assertThat(location.getMetadataFilePath().toString(), not(containsString(ENTROPY_MARKER)));
    // check entropy in task-owned state
    try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
        stream.write(new byte[fileSizeThreshold + 1], 0, fileSizeThreshold + 1);
        FileStateHandle handle = (FileStateHandle) stream.closeAndGetHandle();
        assertNotNull(handle);
        assertThat(handle.getFilePath().toString(), not(containsString(ENTROPY_MARKER)));
        assertThat(handle.getFilePath().toString(), containsString(RESOLVED_MARKER));
    }
    // check entropy in the exclusive/shared state
    try (CheckpointStateOutputStream stream = location.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE)) {
        stream.write(new byte[fileSizeThreshold + 1], 0, fileSizeThreshold + 1);
        FileStateHandle handle = (FileStateHandle) stream.closeAndGetHandle();
        assertNotNull(handle);
        assertThat(handle.getFilePath().toString(), not(containsString(ENTROPY_MARKER)));
        assertThat(handle.getFilePath().toString(), containsString(RESOLVED_MARKER));
    }
    // check entropy in the exclusive/shared state
    try (CheckpointMetadataOutputStream stream = location.createMetadataOutputStream()) {
        stream.flush();
        FsCompletedCheckpointStorageLocation handle = (FsCompletedCheckpointStorageLocation) stream.closeAndFinalizeCheckpoint();
        assertNotNull(handle);
        // metadata files have no entropy
        assertThat(handle.getMetadataHandle().getFilePath().toString(), not(containsString(ENTROPY_MARKER)));
        assertThat(handle.getMetadataHandle().getFilePath().toString(), not(containsString(RESOLVED_MARKER)));
        // external location is the same as metadata, without the file name
        assertEquals(handle.getMetadataHandle().getFilePath().getParent().toString(), handle.getExternalPointer());
    }
}

Example 55 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class AbstractFileCheckpointStorageAccessTestBase method testPointerPathResolution.

// ------------------------------------------------------------------------
// pointers
// ------------------------------------------------------------------------
@Test
public void testPointerPathResolution() throws Exception {
    final FileSystem fs = FileSystem.getLocalFileSystem();
    final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorageAccess.METADATA_FILE_NAME);
    final String basePointer = metadataFile.getParent().toString();
    final String pointer1 = metadataFile.toString();
    final String pointer2 = metadataFile.getParent().toString();
    final String pointer3 = metadataFile.getParent().toString() + '/';
    // create the storage for some random checkpoint directory
    final CheckpointStorageAccess storage = createCheckpointStorage(randomTempPath());
    final byte[] data = new byte[23686];
    new Random().nextBytes(data);
    try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
        out.write(data);
    }
    CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
    CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
    CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);
    assertEquals(basePointer, completed1.getExternalPointer());
    assertEquals(basePointer, completed2.getExternalPointer());
    assertEquals(basePointer, completed3.getExternalPointer());
    StreamStateHandle handle1 = completed1.getMetadataHandle();
    StreamStateHandle handle2 = completed2.getMetadataHandle();
    StreamStateHandle handle3 = completed3.getMetadataHandle();
    assertNotNull(handle1);
    assertNotNull(handle2);
    assertNotNull(handle3);
    validateContents(handle1, data);
    validateContents(handle2, data);
    validateContents(handle3, data);
}