Examples with CheckpointMetadataOutputStream org.apache.flink.runtime.state.CheckpointMetadataOutputStream used on opensource projects

Example 1 with CheckpointMetadataOutputStream

use of org.apache.flink.runtime.state.CheckpointMetadataOutputStream in project flink by apache.

the class DispatcherTest method createTestingSavepoint.

@Nonnull
private URI createTestingSavepoint() throws IOException, URISyntaxException {
    final CheckpointStorage storage = Checkpoints.loadCheckpointStorage(configuration, Thread.currentThread().getContextClassLoader(), log);
    final CheckpointStorageCoordinatorView checkpointStorage = storage.createCheckpointStorage(jobGraph.getJobID());
    final File savepointFile = temporaryFolder.newFolder();
    final long checkpointId = 1L;
    final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath());
    final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream();
    Checkpoints.storeCheckpointMetadata(new CheckpointMetadata(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream);
    final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint();
    return new URI(completedCheckpointStorageLocation.getExternalPointer());
}

Example 2 with CheckpointMetadataOutputStream

use of org.apache.flink.runtime.state.CheckpointMetadataOutputStream in project flink by apache.

the class SavepointOutputFormat method writeRecord.

@Override
public void writeRecord(CheckpointMetadata metadata) throws IOException {
    String path = LambdaUtil.withContextClassLoader(getRuntimeContext().getUserCodeClassLoader(), () -> {
        try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
            Checkpoints.storeCheckpointMetadata(metadata, out);
            CompletedCheckpointStorageLocation finalizedLocation = out.closeAndFinalizeCheckpoint();
            return finalizedLocation.getExternalPointer();
        }
    });
    LOG.info("Savepoint written to " + path);
}

Example 3 with CheckpointMetadataOutputStream

use of org.apache.flink.runtime.state.CheckpointMetadataOutputStream in project flink by apache.

the class FsStateBackendEntropyTest method testEntropyInjection.

@Test
public void testEntropyInjection() throws Exception {
    final int fileSizeThreshold = 1024;
    final FileSystem fs = new TestEntropyAwareFs();
    final Path checkpointDir = new Path(Path.fromLocalFile(tmp.newFolder()), ENTROPY_MARKER + "/checkpoints");
    final String checkpointDirStr = checkpointDir.toString();
    final FsCheckpointStorageAccess storage = new FsCheckpointStorageAccess(fs, checkpointDir, null, new JobID(), fileSizeThreshold, 4096);
    storage.initializeBaseLocationsForCheckpoint();
    final FsCheckpointStorageLocation location = (FsCheckpointStorageLocation) storage.initializeLocationForCheckpoint(96562);
    assertThat(location.getCheckpointDirectory().toString(), startsWith(checkpointDirStr));
    assertThat(location.getSharedStateDirectory().toString(), startsWith(checkpointDirStr));
    assertThat(location.getTaskOwnedStateDirectory().toString(), startsWith(checkpointDirStr));
    assertThat(location.getMetadataFilePath().toString(), not(containsString(ENTROPY_MARKER)));
    // check entropy in task-owned state
    try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
        stream.write(new byte[fileSizeThreshold + 1], 0, fileSizeThreshold + 1);
        FileStateHandle handle = (FileStateHandle) stream.closeAndGetHandle();
        assertNotNull(handle);
        assertThat(handle.getFilePath().toString(), not(containsString(ENTROPY_MARKER)));
        assertThat(handle.getFilePath().toString(), containsString(RESOLVED_MARKER));
    }
    // check entropy in the exclusive/shared state
    try (CheckpointStateOutputStream stream = location.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE)) {
        stream.write(new byte[fileSizeThreshold + 1], 0, fileSizeThreshold + 1);
        FileStateHandle handle = (FileStateHandle) stream.closeAndGetHandle();
        assertNotNull(handle);
        assertThat(handle.getFilePath().toString(), not(containsString(ENTROPY_MARKER)));
        assertThat(handle.getFilePath().toString(), containsString(RESOLVED_MARKER));
    }
    // check entropy in the exclusive/shared state
    try (CheckpointMetadataOutputStream stream = location.createMetadataOutputStream()) {
        stream.flush();
        FsCompletedCheckpointStorageLocation handle = (FsCompletedCheckpointStorageLocation) stream.closeAndFinalizeCheckpoint();
        assertNotNull(handle);
        // metadata files have no entropy
        assertThat(handle.getMetadataHandle().getFilePath().toString(), not(containsString(ENTROPY_MARKER)));
        assertThat(handle.getMetadataHandle().getFilePath().toString(), not(containsString(RESOLVED_MARKER)));
        // external location is the same as metadata, without the file name
        assertEquals(handle.getMetadataHandle().getFilePath().getParent().toString(), handle.getExternalPointer());
    }
}

Example 4 with CheckpointMetadataOutputStream

use of org.apache.flink.runtime.state.CheckpointMetadataOutputStream in project flink by apache.

the class AbstractFileCheckpointStorageAccessTestBase method testSavepoint.

private void testSavepoint(@Nullable Path savepointDir, @Nullable Path customDir, Path expectedParent) throws Exception {
    final CheckpointStorageAccess storage = savepointDir == null ? createCheckpointStorage(randomTempPath()) : createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);
    final String customLocation = customDir == null ? null : customDir.toString();
    final CheckpointStorageLocation savepointLocation = storage.initializeLocationForSavepoint(52452L, customLocation);
    final byte[] data = { 77, 66, 55, 99, 88 };
    final CompletedCheckpointStorageLocation completed;
    try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
        out.write(data);
        completed = out.closeAndFinalizeCheckpoint();
    }
    // we need to do this step to make sure we have a slash (or not) in the same way as the
    // expected path has it
    final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));
    assertEquals(expectedParent, normalizedWithSlash);
    validateContents(completed.getMetadataHandle(), data);
    // validate that the correct directory was used
    FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();
    // we need to recreate that path in the same way as the "expected path" (via File and URI)
    // to make
    // sure the either both use '/' suffixes, or neither uses them (a bit of an annoying
    // ambiguity)
    Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());
    assertEquals(expectedParent, usedSavepointDir);
}

Example 5 with CheckpointMetadataOutputStream

use of org.apache.flink.runtime.state.CheckpointMetadataOutputStream in project flink by apache.

the class PendingCheckpoint method finalizeCheckpoint.

public CompletedCheckpoint finalizeCheckpoint(CheckpointsCleaner checkpointsCleaner, Runnable postCleanup, Executor executor, @Nullable PendingCheckpointStats statsCallback) throws IOException {
    synchronized (lock) {
        checkState(!isDisposed(), "checkpoint is discarded");
        checkState(isFullyAcknowledged(), "Pending checkpoint has not been fully acknowledged yet");
        // make sure we fulfill the promise with an exception if something fails
        try {
            checkpointPlan.fulfillFinishedTaskStatus(operatorStates);
            // write out the metadata
            final CheckpointMetadata savepoint = new CheckpointMetadata(checkpointId, operatorStates.values(), masterStates);
            final CompletedCheckpointStorageLocation finalizedLocation;
            try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
                Checkpoints.storeCheckpointMetadata(savepoint, out);
                finalizedLocation = out.closeAndFinalizeCheckpoint();
            }
            CompletedCheckpoint completed = new CompletedCheckpoint(jobId, checkpointId, checkpointTimestamp, System.currentTimeMillis(), operatorStates, masterStates, props, finalizedLocation);
            onCompletionPromise.complete(completed);
            if (statsCallback != null) {
                LOG.trace("Checkpoint {} size: {}Kb, duration: {}ms", checkpointId, statsCallback.getStateSize() == 0 ? 0 : statsCallback.getStateSize() / 1024, statsCallback.getEndToEndDuration());
                // Finalize the statsCallback and give the completed checkpoint a
                // callback for discards.
                CompletedCheckpointStats.DiscardCallback discardCallback = statsCallback.reportCompletedCheckpoint(finalizedLocation.getExternalPointer());
                completed.setDiscardCallback(discardCallback);
            }
            // mark this pending checkpoint as disposed, but do NOT drop the state
            dispose(false, checkpointsCleaner, postCleanup, executor);
            return completed;
        } catch (Throwable t) {
            onCompletionPromise.completeExceptionally(t);
            ExceptionUtils.rethrowIOException(t);
            // silence the compiler
            return null;
        }
    }
}