Examples with CheckpointStateOutputStream org.apache.flink.runtime.state.CheckpointStateOutputStream used on opensource projects

Example 6 with CheckpointStateOutputStream

use of org.apache.flink.runtime.state.CheckpointStateOutputStream in project flink by apache.

the class FsCheckpointStateOutputStreamTest method testCleanupWhenFailingCloseAndGetHandle.

/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
    final FileSystem fs = mock(FileSystem.class);
    final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);
    final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
    when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);
    doThrow(new IOException("Test IOException.")).when(outputStream).close();
    CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(Path.fromLocalFile(tempDir.newFolder()), fs, 4, 0, relativePaths);
    // this should create the underlying file stream
    stream.write(new byte[] { 1, 2, 3, 4, 5 });
    verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));
    try {
        stream.closeAndGetHandle();
        fail("Expected IOException");
    } catch (IOException ioE) {
    // expected exception
    }
    verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}

Example 7 with CheckpointStateOutputStream

use of org.apache.flink.runtime.state.CheckpointStateOutputStream in project flink by apache.

the class FsCheckpointStateOutputStreamTest method testEmptyState.

@Test
public void testEmptyState() throws Exception {
    CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 1024, 512, relativePaths);
    StreamStateHandle handle = stream.closeAndGetHandle();
    assertNull(handle);
}

Example 8 with CheckpointStateOutputStream

use of org.apache.flink.runtime.state.CheckpointStateOutputStream in project flink by apache.

the class FsStateBackendEntropyTest method testEntropyInjection.

@Test
public void testEntropyInjection() throws Exception {
    final int fileSizeThreshold = 1024;
    final FileSystem fs = new TestEntropyAwareFs();
    final Path checkpointDir = new Path(Path.fromLocalFile(tmp.newFolder()), ENTROPY_MARKER + "/checkpoints");
    final String checkpointDirStr = checkpointDir.toString();
    final FsCheckpointStorageAccess storage = new FsCheckpointStorageAccess(fs, checkpointDir, null, new JobID(), fileSizeThreshold, 4096);
    storage.initializeBaseLocationsForCheckpoint();
    final FsCheckpointStorageLocation location = (FsCheckpointStorageLocation) storage.initializeLocationForCheckpoint(96562);
    assertThat(location.getCheckpointDirectory().toString(), startsWith(checkpointDirStr));
    assertThat(location.getSharedStateDirectory().toString(), startsWith(checkpointDirStr));
    assertThat(location.getTaskOwnedStateDirectory().toString(), startsWith(checkpointDirStr));
    assertThat(location.getMetadataFilePath().toString(), not(containsString(ENTROPY_MARKER)));
    // check entropy in task-owned state
    try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
        stream.write(new byte[fileSizeThreshold + 1], 0, fileSizeThreshold + 1);
        FileStateHandle handle = (FileStateHandle) stream.closeAndGetHandle();
        assertNotNull(handle);
        assertThat(handle.getFilePath().toString(), not(containsString(ENTROPY_MARKER)));
        assertThat(handle.getFilePath().toString(), containsString(RESOLVED_MARKER));
    }
    // check entropy in the exclusive/shared state
    try (CheckpointStateOutputStream stream = location.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE)) {
        stream.write(new byte[fileSizeThreshold + 1], 0, fileSizeThreshold + 1);
        FileStateHandle handle = (FileStateHandle) stream.closeAndGetHandle();
        assertNotNull(handle);
        assertThat(handle.getFilePath().toString(), not(containsString(ENTROPY_MARKER)));
        assertThat(handle.getFilePath().toString(), containsString(RESOLVED_MARKER));
    }
    // check entropy in the exclusive/shared state
    try (CheckpointMetadataOutputStream stream = location.createMetadataOutputStream()) {
        stream.flush();
        FsCompletedCheckpointStorageLocation handle = (FsCompletedCheckpointStorageLocation) stream.closeAndFinalizeCheckpoint();
        assertNotNull(handle);
        // metadata files have no entropy
        assertThat(handle.getMetadataHandle().getFilePath().toString(), not(containsString(ENTROPY_MARKER)));
        assertThat(handle.getMetadataHandle().getFilePath().toString(), not(containsString(RESOLVED_MARKER)));
        // external location is the same as metadata, without the file name
        assertEquals(handle.getMetadataHandle().getFilePath().getParent().toString(), handle.getExternalPointer());
    }
}

Example 9 with CheckpointStateOutputStream

use of org.apache.flink.runtime.state.CheckpointStateOutputStream in project flink by apache.

the class RocksDBStateUploaderTest method testMultiThreadUploadThreadPoolExceptionRethrow.

/**
 * Test that the exception arose in the thread pool will rethrow to the main thread.
 */
@Test
public void testMultiThreadUploadThreadPoolExceptionRethrow() throws IOException {
    SpecifiedException expectedException = new SpecifiedException("throw exception while multi thread upload states.");
    CheckpointStateOutputStream outputStream = createFailingCheckpointStateOutputStream(expectedException);
    CheckpointStreamFactory checkpointStreamFactory = new CheckpointStreamFactory() {

        @Override
        public CheckpointStateOutputStream createCheckpointStateOutputStream(CheckpointedStateScope scope) throws IOException {
            return outputStream;
        }

        @Override
        public boolean canFastDuplicate(StreamStateHandle stateHandle, CheckpointedStateScope scope) throws IOException {
            return false;
        }

        @Override
        public List<StreamStateHandle> duplicate(List<StreamStateHandle> stateHandles, CheckpointedStateScope scope) throws IOException {
            return null;
        }
    };
    File file = temporaryFolder.newFile(String.valueOf(UUID.randomUUID()));
    generateRandomFileContent(file.getPath(), 20);
    Map<StateHandleID, Path> filePaths = new HashMap<>(1);
    filePaths.put(new StateHandleID("mockHandleID"), file.toPath());
    try (RocksDBStateUploader rocksDBStateUploader = new RocksDBStateUploader(5)) {
        rocksDBStateUploader.uploadFilesToCheckpointFs(filePaths, checkpointStreamFactory, CheckpointedStateScope.SHARED, new CloseableRegistry());
        fail();
    } catch (Exception e) {
        assertEquals(expectedException, e);
    }
}

Example 10 with CheckpointStateOutputStream

use of org.apache.flink.runtime.state.CheckpointStateOutputStream in project flink by apache.

the class BlockingCheckpointOutputStream method closeAndGetHandle.

@Nullable
@Override
public StreamStateHandle closeAndGetHandle() throws IOException {
    if (!closed.compareAndSet(false, true)) {
        throw new IOException("Stream was already closed!");
    }
    if (delegate instanceof CheckpointStateOutputStream) {
        StreamStateHandle streamStateHandle = ((CheckpointStateOutputStream) delegate).closeAndGetHandle();
        unblockAll();
        return streamStateHandle;
    } else {
        unblockAll();
        throw new IOException("Delegate is not a CheckpointStateOutputStream!");
    }
}