Examples with FsCheckpointStateOutputStream org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory.FsCheckpointStateOutputStream used on opensource projects

Example 1 with FsCheckpointStateOutputStream

use of org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory.FsCheckpointStateOutputStream in project flink by apache.

the class FsSavepointStreamFactoryTest method testSavepointStreamDirectoryLayout.

/**
	 * Tests that the factory creates all files in the given directory without
	 * creating any sub directories.
	 */
@Test
public void testSavepointStreamDirectoryLayout() throws Exception {
    File testRoot = folder.newFolder();
    JobID jobId = new JobID();
    FsSavepointStreamFactory savepointStreamFactory = new FsSavepointStreamFactory(new Path(testRoot.getAbsolutePath()), jobId, 0);
    File[] listed = testRoot.listFiles();
    assertNotNull(listed);
    assertEquals(0, listed.length);
    FsCheckpointStateOutputStream stream = savepointStreamFactory.createCheckpointStateOutputStream(1273, 19231);
    stream.write(1);
    FileStateHandle handle = (FileStateHandle) stream.closeAndGetHandle();
    listed = testRoot.listFiles();
    assertNotNull(listed);
    assertEquals(1, listed.length);
    assertEquals(handle.getFilePath().getPath(), listed[0].getPath());
}

Example 2 with FsCheckpointStateOutputStream

use of org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory.FsCheckpointStateOutputStream in project flink by apache.

the class FsCheckpointStateOutputStreamTest method testMixedBelowAndAboveThreshold.

@Test
public void testMixedBelowAndAboveThreshold() throws Exception {
    final byte[] state1 = new byte[1274673];
    final byte[] state2 = new byte[1];
    final byte[] state3 = new byte[0];
    final byte[] state4 = new byte[177];
    final Random rnd = new Random();
    rnd.nextBytes(state1);
    rnd.nextBytes(state2);
    rnd.nextBytes(state3);
    rnd.nextBytes(state4);
    final File directory = tempDir.newFolder();
    final Path basePath = Path.fromLocalFile(directory);
    final Supplier<CheckpointStateOutputStream> factory = () -> new FsCheckpointStateOutputStream(basePath, FileSystem.getLocalFileSystem(), 1024, 15, relativePaths);
    CheckpointStateOutputStream stream1 = factory.get();
    CheckpointStateOutputStream stream2 = factory.get();
    CheckpointStateOutputStream stream3 = factory.get();
    stream1.write(state1);
    stream2.write(state2);
    stream3.write(state3);
    FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
    ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
    ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();
    // use with try-with-resources
    StreamStateHandle handle4;
    try (CheckpointStateOutputStream stream4 = factory.get()) {
        stream4.write(state4);
        handle4 = stream4.closeAndGetHandle();
    }
    // close before accessing handle
    CheckpointStateOutputStream stream5 = factory.get();
    stream5.write(state4);
    stream5.close();
    try {
        stream5.closeAndGetHandle();
        fail();
    } catch (IOException e) {
    // uh-huh
    }
    validateBytesInStream(handle1.openInputStream(), state1);
    handle1.discardState();
    assertFalse(isDirectoryEmpty(directory));
    ensureLocalFileDeleted(handle1.getFilePath());
    validateBytesInStream(handle2.openInputStream(), state2);
    handle2.discardState();
    assertFalse(isDirectoryEmpty(directory));
    // nothing was written to the stream, so it will return nothing
    assertNull(handle3);
    assertFalse(isDirectoryEmpty(directory));
    validateBytesInStream(handle4.openInputStream(), state4);
    handle4.discardState();
    assertTrue(isDirectoryEmpty(directory));
}

Example 3 with FsCheckpointStateOutputStream

use of org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory.FsCheckpointStateOutputStream in project flink by apache.

the class FsCheckpointStateOutputStreamTest method testStreamDoesNotTryToCleanUpParentOnError.

// ------------------------------------------------------------------------
// Not deleting parent directories
// ------------------------------------------------------------------------
/**
 * This test checks that the stream does not check and clean the parent directory when
 * encountering a write error.
 */
@Test
public void testStreamDoesNotTryToCleanUpParentOnError() throws Exception {
    final File directory = tempDir.newFolder();
    // prevent creation of files in that directory
    // this operation does not work reliably on Windows, so we use an "assume" to skip the test
    // is this prerequisite operation is not supported.
    assumeTrue(directory.setWritable(false, true));
    checkDirectoryNotWritable(directory);
    FileSystem fs = spy(FileSystem.getLocalFileSystem());
    FsCheckpointStateOutputStream stream1 = new FsCheckpointStateOutputStream(Path.fromLocalFile(directory), fs, 1024, 1, relativePaths);
    FsCheckpointStateOutputStream stream2 = new FsCheckpointStateOutputStream(Path.fromLocalFile(directory), fs, 1024, 1, relativePaths);
    stream1.write(new byte[61]);
    stream2.write(new byte[61]);
    try {
        stream1.closeAndGetHandle();
        fail("this should fail with an exception");
    } catch (IOException ignored) {
    }
    stream2.close();
    // no delete call must have happened
    verify(fs, times(0)).delete(any(Path.class), anyBoolean());
    // the directory must still exist as a proper directory
    assertTrue(directory.exists());
    assertTrue(directory.isDirectory());
}

Example 4 with FsCheckpointStateOutputStream

use of org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory.FsCheckpointStateOutputStream in project flink by apache.

the class FsCheckpointStateOutputStreamTest method testWriteFailsFastWhenClosed.

@Test
public void testWriteFailsFastWhenClosed() throws Exception {
    FsCheckpointStateOutputStream stream = new FsCheckpointStateOutputStream(Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 1024, 512, relativePaths);
    assertFalse(stream.isClosed());
    stream.close();
    assertTrue(stream.isClosed());
    try {
        stream.write(1);
        fail();
    } catch (IOException e) {
    // expected
    }
    try {
        stream.write(new byte[4], 1, 2);
        fail();
    } catch (IOException e) {
    // expected
    }
}

Example 5 with FsCheckpointStateOutputStream

use of org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory.FsCheckpointStateOutputStream in project flink by apache.

the class FsCheckpointStorageAccessTest method testTaskOwnedStateStream.

@Test
public void testTaskOwnedStateStream() throws Exception {
    final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter");
    // we chose a small size threshold here to force the state to disk
    final FsCheckpointStorageAccess storage = new FsCheckpointStorageAccess(Path.fromLocalFile(tmp.newFolder()), null, new JobID(), 10, WRITE_BUFFER_SIZE);
    final StreamStateHandle stateHandle;
    try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
        assertTrue(stream instanceof FsCheckpointStateOutputStream);
        new ObjectOutputStream(stream).writeObject(state);
        stateHandle = stream.closeAndGetHandle();
    }
    // the state must have gone to disk
    FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
    // check that the state is in the correct directory
    String parentDirName = fileStateHandle.getFilePath().getParent().getName();
    assertEquals(FsCheckpointStorageAccess.CHECKPOINT_TASK_OWNED_STATE_DIR, parentDirName);
    // validate the contents
    try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) {
        assertEquals(state, in.readObject());
    }
}