Examples with CompletedCheckpointStorageLocation org.apache.flink.runtime.state.CompletedCheckpointStorageLocation used on opensource projects

Example 1 with CompletedCheckpointStorageLocation

use of org.apache.flink.runtime.state.CompletedCheckpointStorageLocation in project flink by apache.

the class DispatcherTest method createTestingSavepoint.

@Nonnull
private URI createTestingSavepoint() throws IOException, URISyntaxException {
    final CheckpointStorage storage = Checkpoints.loadCheckpointStorage(configuration, Thread.currentThread().getContextClassLoader(), log);
    final CheckpointStorageCoordinatorView checkpointStorage = storage.createCheckpointStorage(jobGraph.getJobID());
    final File savepointFile = temporaryFolder.newFolder();
    final long checkpointId = 1L;
    final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath());
    final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream();
    Checkpoints.storeCheckpointMetadata(new CheckpointMetadata(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream);
    final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint();
    return new URI(completedCheckpointStorageLocation.getExternalPointer());
}

Example 2 with CompletedCheckpointStorageLocation

use of org.apache.flink.runtime.state.CompletedCheckpointStorageLocation in project flink by apache.

the class SavepointOutputFormat method writeRecord.

@Override
public void writeRecord(CheckpointMetadata metadata) throws IOException {
    String path = LambdaUtil.withContextClassLoader(getRuntimeContext().getUserCodeClassLoader(), () -> {
        try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
            Checkpoints.storeCheckpointMetadata(metadata, out);
            CompletedCheckpointStorageLocation finalizedLocation = out.closeAndFinalizeCheckpoint();
            return finalizedLocation.getExternalPointer();
        }
    });
    LOG.info("Savepoint written to " + path);
}

Example 3 with CompletedCheckpointStorageLocation

use of org.apache.flink.runtime.state.CompletedCheckpointStorageLocation in project flink by apache.

the class CheckpointMetadataLoadingTest method testAllStateRestored.

/**
 * Tests correct savepoint loading.
 */
@Test
public void testAllStateRestored() throws Exception {
    final JobID jobId = new JobID();
    final OperatorID operatorId = new OperatorID();
    final long checkpointId = Integer.MAX_VALUE + 123123L;
    final int parallelism = 128128;
    final CompletedCheckpointStorageLocation testSavepoint = createSavepointWithOperatorSubtaskState(checkpointId, operatorId, parallelism);
    final Map<JobVertexID, ExecutionJobVertex> tasks = createTasks(operatorId, parallelism, parallelism);
    final CompletedCheckpoint loaded = Checkpoints.loadAndValidateCheckpoint(jobId, tasks, testSavepoint, cl, false, CheckpointProperties.forSavepoint(false, SavepointFormatType.CANONICAL), RestoreMode.NO_CLAIM);
    assertEquals(jobId, loaded.getJobId());
    assertEquals(checkpointId, loaded.getCheckpointID());
}

Example 4 with CompletedCheckpointStorageLocation

use of org.apache.flink.runtime.state.CompletedCheckpointStorageLocation in project flink by apache.

the class CheckpointCoordinator method restoreSavepoint.

/**
 * Restore the state with given savepoint.
 *
 * @param restoreSettings Settings for a snapshot to restore from. Includes the path and
 *     parameters for the restore process.
 * @param tasks Map of job vertices to restore. State for these vertices is restored via {@link
 *     Execution#setInitialState(JobManagerTaskRestore)}.
 * @param userClassLoader The class loader to resolve serialized classes in legacy savepoint
 *     versions.
 */
public boolean restoreSavepoint(SavepointRestoreSettings restoreSettings, Map<JobVertexID, ExecutionJobVertex> tasks, ClassLoader userClassLoader) throws Exception {
    final String savepointPointer = restoreSettings.getRestorePath();
    final boolean allowNonRestored = restoreSettings.allowNonRestoredState();
    Preconditions.checkNotNull(savepointPointer, "The savepoint path cannot be null.");
    LOG.info("Starting job {} from savepoint {} ({})", job, savepointPointer, (allowNonRestored ? "allowing non restored state" : ""));
    final CompletedCheckpointStorageLocation checkpointLocation = checkpointStorageView.resolveCheckpoint(savepointPointer);
    // convert to checkpoint so the system can fall back to it
    final CheckpointProperties checkpointProperties;
    switch(restoreSettings.getRestoreMode()) {
        case CLAIM:
            checkpointProperties = this.checkpointProperties;
            break;
        case LEGACY:
            checkpointProperties = CheckpointProperties.forSavepoint(false, // necessary when triggering a savepoint
            SavepointFormatType.CANONICAL);
            break;
        case NO_CLAIM:
            checkpointProperties = CheckpointProperties.forUnclaimedSnapshot();
            break;
        default:
            throw new IllegalArgumentException("Unknown snapshot restore mode");
    }
    // Load the savepoint as a checkpoint into the system
    CompletedCheckpoint savepoint = Checkpoints.loadAndValidateCheckpoint(job, tasks, checkpointLocation, userClassLoader, allowNonRestored, checkpointProperties, restoreSettings.getRestoreMode());
    // register shared state - even before adding the checkpoint to the store
    // because the latter might trigger subsumption so the ref counts must be up-to-date
    savepoint.registerSharedStatesAfterRestored(completedCheckpointStore.getSharedStateRegistry());
    completedCheckpointStore.addCheckpointAndSubsumeOldestOne(savepoint, checkpointsCleaner, this::scheduleTriggerRequest);
    // Reset the checkpoint ID counter
    long nextCheckpointId = savepoint.getCheckpointID() + 1;
    checkpointIdCounter.setCount(nextCheckpointId);
    LOG.info("Reset the checkpoint ID of job {} to {}.", job, nextCheckpointId);
    final OptionalLong restoredCheckpointId = restoreLatestCheckpointedStateInternal(new HashSet<>(tasks.values()), OperatorCoordinatorRestoreBehavior.RESTORE_IF_CHECKPOINT_PRESENT, true, allowNonRestored, true);
    return restoredCheckpointId.isPresent();
}

Example 5 with CompletedCheckpointStorageLocation

use of org.apache.flink.runtime.state.CompletedCheckpointStorageLocation in project flink by apache.

the class AbstractFileCheckpointStorageAccessTestBase method testPointerPathResolution.

// ------------------------------------------------------------------------
// pointers
// ------------------------------------------------------------------------
@Test
public void testPointerPathResolution() throws Exception {
    final FileSystem fs = FileSystem.getLocalFileSystem();
    final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorageAccess.METADATA_FILE_NAME);
    final String basePointer = metadataFile.getParent().toString();
    final String pointer1 = metadataFile.toString();
    final String pointer2 = metadataFile.getParent().toString();
    final String pointer3 = metadataFile.getParent().toString() + '/';
    // create the storage for some random checkpoint directory
    final CheckpointStorageAccess storage = createCheckpointStorage(randomTempPath());
    final byte[] data = new byte[23686];
    new Random().nextBytes(data);
    try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
        out.write(data);
    }
    CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
    CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
    CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);
    assertEquals(basePointer, completed1.getExternalPointer());
    assertEquals(basePointer, completed2.getExternalPointer());
    assertEquals(basePointer, completed3.getExternalPointer());
    StreamStateHandle handle1 = completed1.getMetadataHandle();
    StreamStateHandle handle2 = completed2.getMetadataHandle();
    StreamStateHandle handle3 = completed3.getMetadataHandle();
    assertNotNull(handle1);
    assertNotNull(handle2);
    assertNotNull(handle3);
    validateContents(handle1, data);
    validateContents(handle2, data);
    validateContents(handle3, data);
}