Examples with CompletedCheckpointStorageLocation org.apache.flink.runtime.state.CompletedCheckpointStorageLocation used on opensource projects

Example 6 with CompletedCheckpointStorageLocation

use of org.apache.flink.runtime.state.CompletedCheckpointStorageLocation in project flink by apache.

the class AbstractFileCheckpointStorageAccessTestBase method testSavepoint.

private void testSavepoint(@Nullable Path savepointDir, @Nullable Path customDir, Path expectedParent) throws Exception {
    final CheckpointStorageAccess storage = savepointDir == null ? createCheckpointStorage(randomTempPath()) : createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);
    final String customLocation = customDir == null ? null : customDir.toString();
    final CheckpointStorageLocation savepointLocation = storage.initializeLocationForSavepoint(52452L, customLocation);
    final byte[] data = { 77, 66, 55, 99, 88 };
    final CompletedCheckpointStorageLocation completed;
    try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
        out.write(data);
        completed = out.closeAndFinalizeCheckpoint();
    }
    // we need to do this step to make sure we have a slash (or not) in the same way as the
    // expected path has it
    final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));
    assertEquals(expectedParent, normalizedWithSlash);
    validateContents(completed.getMetadataHandle(), data);
    // validate that the correct directory was used
    FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();
    // we need to recreate that path in the same way as the "expected path" (via File and URI)
    // to make
    // sure the either both use '/' suffixes, or neither uses them (a bit of an annoying
    // ambiguity)
    Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());
    assertEquals(expectedParent, usedSavepointDir);
}

Example 7 with CompletedCheckpointStorageLocation

use of org.apache.flink.runtime.state.CompletedCheckpointStorageLocation in project flink by apache.

the class Checkpoints method disposeSavepoint.

// ------------------------------------------------------------------------
// Savepoint Disposal Hooks
// ------------------------------------------------------------------------
public static void disposeSavepoint(String pointer, CheckpointStorage checkpointStorage, ClassLoader classLoader) throws IOException, FlinkException {
    checkNotNull(pointer, "location");
    checkNotNull(checkpointStorage, "stateBackend");
    checkNotNull(classLoader, "classLoader");
    final CompletedCheckpointStorageLocation checkpointLocation = checkpointStorage.resolveCheckpoint(pointer);
    final StreamStateHandle metadataHandle = checkpointLocation.getMetadataHandle();
    // load the savepoint object (the metadata) to have all the state handles that we need
    // to dispose of all state
    final CheckpointMetadata metadata;
    try (InputStream in = metadataHandle.openInputStream();
        DataInputStream dis = new DataInputStream(in)) {
        metadata = loadCheckpointMetadata(dis, classLoader, pointer);
    }
    Exception exception = null;
    // addressable any more even if the following disposal fails
    try {
        metadataHandle.discardState();
    } catch (Exception e) {
        exception = e;
    }
    // now dispose the savepoint data
    try {
        metadata.dispose();
    } catch (Exception e) {
        exception = ExceptionUtils.firstOrSuppressed(e, exception);
    }
    // now dispose the location (directory, table, whatever)
    try {
        checkpointLocation.disposeStorageLocation();
    } catch (Exception e) {
        exception = ExceptionUtils.firstOrSuppressed(e, exception);
    }
    // forward exceptions caught in the process
    if (exception != null) {
        ExceptionUtils.rethrowIOException(exception);
    }
}

Example 8 with CompletedCheckpointStorageLocation

use of org.apache.flink.runtime.state.CompletedCheckpointStorageLocation in project flink by apache.

the class PendingCheckpoint method finalizeCheckpoint.

public CompletedCheckpoint finalizeCheckpoint(CheckpointsCleaner checkpointsCleaner, Runnable postCleanup, Executor executor, @Nullable PendingCheckpointStats statsCallback) throws IOException {
    synchronized (lock) {
        checkState(!isDisposed(), "checkpoint is discarded");
        checkState(isFullyAcknowledged(), "Pending checkpoint has not been fully acknowledged yet");
        // make sure we fulfill the promise with an exception if something fails
        try {
            checkpointPlan.fulfillFinishedTaskStatus(operatorStates);
            // write out the metadata
            final CheckpointMetadata savepoint = new CheckpointMetadata(checkpointId, operatorStates.values(), masterStates);
            final CompletedCheckpointStorageLocation finalizedLocation;
            try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
                Checkpoints.storeCheckpointMetadata(savepoint, out);
                finalizedLocation = out.closeAndFinalizeCheckpoint();
            }
            CompletedCheckpoint completed = new CompletedCheckpoint(jobId, checkpointId, checkpointTimestamp, System.currentTimeMillis(), operatorStates, masterStates, props, finalizedLocation);
            onCompletionPromise.complete(completed);
            if (statsCallback != null) {
                LOG.trace("Checkpoint {} size: {}Kb, duration: {}ms", checkpointId, statsCallback.getStateSize() == 0 ? 0 : statsCallback.getStateSize() / 1024, statsCallback.getEndToEndDuration());
                // Finalize the statsCallback and give the completed checkpoint a
                // callback for discards.
                CompletedCheckpointStats.DiscardCallback discardCallback = statsCallback.reportCompletedCheckpoint(finalizedLocation.getExternalPointer());
                completed.setDiscardCallback(discardCallback);
            }
            // mark this pending checkpoint as disposed, but do NOT drop the state
            dispose(false, checkpointsCleaner, postCleanup, executor);
            return completed;
        } catch (Throwable t) {
            onCompletionPromise.completeExceptionally(t);
            ExceptionUtils.rethrowIOException(t);
            // silence the compiler
            return null;
        }
    }
}

Example 9 with CompletedCheckpointStorageLocation

use of org.apache.flink.runtime.state.CompletedCheckpointStorageLocation in project flink by apache.

the class MemoryCheckpointStorageAccessTest method testNonPersistentCheckpointLocation.

@Test
public void testNonPersistentCheckpointLocation() throws Exception {
    MemoryBackendCheckpointStorageAccess storage = new MemoryBackendCheckpointStorageAccess(new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);
    CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);
    CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
    stream.write(99);
    CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
    StreamStateHandle handle = completed.getMetadataHandle();
    assertTrue(handle instanceof ByteStreamStateHandle);
    // the reference is not valid in that case
    try {
        storage.resolveCheckpoint(completed.getExternalPointer());
        fail("should fail with an exception");
    } catch (Exception e) {
    // expected
    }
}

Example 10 with CompletedCheckpointStorageLocation

use of org.apache.flink.runtime.state.CompletedCheckpointStorageLocation in project flink by apache.

the class AbstractFileCheckpointStorageAccessTestBase method testPersistMultipleMetadataOnlyCheckpoints.

// ------------------------------------------------------------------------
// checkpoints
// ------------------------------------------------------------------------
/**
 * Validates that multiple checkpoints from different jobs with the same checkpoint ID do not
 * interfere with each other.
 */
@Test
public void testPersistMultipleMetadataOnlyCheckpoints() throws Exception {
    final FileSystem fs = FileSystem.getLocalFileSystem();
    final Path checkpointDir = new Path(tmp.newFolder().toURI());
    final long checkpointId = 177;
    final CheckpointStorageAccess storage1 = createCheckpointStorage(checkpointDir);
    storage1.initializeBaseLocationsForCheckpoint();
    final CheckpointStorageAccess storage2 = createCheckpointStorage(checkpointDir);
    storage2.initializeBaseLocationsForCheckpoint();
    final CheckpointStorageLocation loc1 = storage1.initializeLocationForCheckpoint(checkpointId);
    final CheckpointStorageLocation loc2 = storage2.initializeLocationForCheckpoint(checkpointId);
    final byte[] data1 = { 77, 66, 55, 99, 88 };
    final byte[] data2 = { 1, 3, 2, 5, 4 };
    final CompletedCheckpointStorageLocation completedLocation1;
    try (CheckpointMetadataOutputStream out = loc1.createMetadataOutputStream()) {
        out.write(data1);
        completedLocation1 = out.closeAndFinalizeCheckpoint();
    }
    final String result1 = completedLocation1.getExternalPointer();
    final CompletedCheckpointStorageLocation completedLocation2;
    try (CheckpointMetadataOutputStream out = loc2.createMetadataOutputStream()) {
        out.write(data2);
        completedLocation2 = out.closeAndFinalizeCheckpoint();
    }
    final String result2 = completedLocation2.getExternalPointer();
    // check that this went to a file, but in a nested directory structure
    // one directory per storage
    FileStatus[] files = fs.listStatus(checkpointDir);
    assertEquals(2, files.length);
    // in each per-storage directory, one for the checkpoint
    FileStatus[] job1Files = fs.listStatus(files[0].getPath());
    FileStatus[] job2Files = fs.listStatus(files[1].getPath());
    assertTrue(job1Files.length >= 1);
    assertTrue(job2Files.length >= 1);
    assertTrue(fs.exists(new Path(result1, AbstractFsCheckpointStorageAccess.METADATA_FILE_NAME)));
    assertTrue(fs.exists(new Path(result2, AbstractFsCheckpointStorageAccess.METADATA_FILE_NAME)));
    // check that both storages can resolve each others contents
    validateContents(storage1.resolveCheckpoint(result1).getMetadataHandle(), data1);
    validateContents(storage1.resolveCheckpoint(result2).getMetadataHandle(), data2);
    validateContents(storage2.resolveCheckpoint(result1).getMetadataHandle(), data1);
    validateContents(storage2.resolveCheckpoint(result2).getMetadataHandle(), data2);
}