Examples with CheckpointStorageLocation org.apache.flink.runtime.state.CheckpointStorageLocation used on opensource projects

Example 1 with CheckpointStorageLocation

use of org.apache.flink.runtime.state.CheckpointStorageLocation in project flink by apache.

the class DispatcherTest method createTestingSavepoint.

@Nonnull
private URI createTestingSavepoint() throws IOException, URISyntaxException {
    final CheckpointStorage storage = Checkpoints.loadCheckpointStorage(configuration, Thread.currentThread().getContextClassLoader(), log);
    final CheckpointStorageCoordinatorView checkpointStorage = storage.createCheckpointStorage(jobGraph.getJobID());
    final File savepointFile = temporaryFolder.newFolder();
    final long checkpointId = 1L;
    final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath());
    final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream();
    Checkpoints.storeCheckpointMetadata(new CheckpointMetadata(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream);
    final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint();
    return new URI(completedCheckpointStorageLocation.getExternalPointer());
}

Example 2 with CheckpointStorageLocation

use of org.apache.flink.runtime.state.CheckpointStorageLocation in project flink by apache.

the class AbstractFileCheckpointStorageAccessTestBase method testSavepoint.

private void testSavepoint(@Nullable Path savepointDir, @Nullable Path customDir, Path expectedParent) throws Exception {
    final CheckpointStorageAccess storage = savepointDir == null ? createCheckpointStorage(randomTempPath()) : createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);
    final String customLocation = customDir == null ? null : customDir.toString();
    final CheckpointStorageLocation savepointLocation = storage.initializeLocationForSavepoint(52452L, customLocation);
    final byte[] data = { 77, 66, 55, 99, 88 };
    final CompletedCheckpointStorageLocation completed;
    try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
        out.write(data);
        completed = out.closeAndFinalizeCheckpoint();
    }
    // we need to do this step to make sure we have a slash (or not) in the same way as the
    // expected path has it
    final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));
    assertEquals(expectedParent, normalizedWithSlash);
    validateContents(completed.getMetadataHandle(), data);
    // validate that the correct directory was used
    FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();
    // we need to recreate that path in the same way as the "expected path" (via File and URI)
    // to make
    // sure the either both use '/' suffixes, or neither uses them (a bit of an annoying
    // ambiguity)
    Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());
    assertEquals(expectedParent, usedSavepointDir);
}

Example 3 with CheckpointStorageLocation

use of org.apache.flink.runtime.state.CheckpointStorageLocation in project flink by apache.

the class CheckpointCoordinator method startTriggeringCheckpoint.

private void startTriggeringCheckpoint(CheckpointTriggerRequest request) {
    try {
        synchronized (lock) {
            preCheckGlobalState(request.isPeriodic);
        }
        // we will actually trigger this checkpoint!
        Preconditions.checkState(!isTriggering);
        isTriggering = true;
        final long timestamp = System.currentTimeMillis();
        CompletableFuture<CheckpointPlan> checkpointPlanFuture = checkpointPlanCalculator.calculateCheckpointPlan();
        boolean initializeBaseLocations = !baseLocationsForCheckpointInitialized;
        baseLocationsForCheckpointInitialized = true;
        final CompletableFuture<PendingCheckpoint> pendingCheckpointCompletableFuture = checkpointPlanFuture.thenApplyAsync(plan -> {
            try {
                CheckpointIdAndStorageLocation checkpointIdAndStorageLocation = initializeCheckpoint(request.props, request.externalSavepointLocation, initializeBaseLocations);
                return new Tuple2<>(plan, checkpointIdAndStorageLocation);
            } catch (Throwable e) {
                throw new CompletionException(e);
            }
        }, executor).thenApplyAsync((checkpointInfo) -> createPendingCheckpoint(timestamp, request.props, checkpointInfo.f0, request.isPeriodic, checkpointInfo.f1.checkpointId, checkpointInfo.f1.checkpointStorageLocation, request.getOnCompletionFuture()), timer);
        final CompletableFuture<?> coordinatorCheckpointsComplete = pendingCheckpointCompletableFuture.thenComposeAsync((pendingCheckpoint) -> OperatorCoordinatorCheckpoints.triggerAndAcknowledgeAllCoordinatorCheckpointsWithCompletion(coordinatorsToCheckpoint, pendingCheckpoint, timer), timer);
        // We have to take the snapshot of the master hooks after the coordinator checkpoints
        // has completed.
        // This is to ensure the tasks are checkpointed after the OperatorCoordinators in case
        // ExternallyInducedSource is used.
        final CompletableFuture<?> masterStatesComplete = coordinatorCheckpointsComplete.thenComposeAsync(ignored -> {
            // If the code reaches here, the pending checkpoint is guaranteed to
            // be not null.
            // We use FutureUtils.getWithoutException() to make compiler happy
            // with checked
            // exceptions in the signature.
            PendingCheckpoint checkpoint = FutureUtils.getWithoutException(pendingCheckpointCompletableFuture);
            return snapshotMasterState(checkpoint);
        }, timer);
        FutureUtils.assertNoException(CompletableFuture.allOf(masterStatesComplete, coordinatorCheckpointsComplete).handleAsync((ignored, throwable) -> {
            final PendingCheckpoint checkpoint = FutureUtils.getWithoutException(pendingCheckpointCompletableFuture);
            Preconditions.checkState(checkpoint != null || throwable != null, "Either the pending checkpoint needs to be created or an error must have occurred.");
            if (throwable != null) {
                // the initialization might not be finished yet
                if (checkpoint == null) {
                    onTriggerFailure(request, throwable);
                } else {
                    onTriggerFailure(checkpoint, throwable);
                }
            } else {
                triggerCheckpointRequest(request, timestamp, checkpoint);
            }
            return null;
        }, timer).exceptionally(error -> {
            if (!isShutdown()) {
                throw new CompletionException(error);
            } else if (findThrowable(error, RejectedExecutionException.class).isPresent()) {
                LOG.debug("Execution rejected during shutdown");
            } else {
                LOG.warn("Error encountered during shutdown", error);
            }
            return null;
        }));
    } catch (Throwable throwable) {
        onTriggerFailure(request, throwable);
    }
}

Example 4 with CheckpointStorageLocation

use of org.apache.flink.runtime.state.CheckpointStorageLocation in project flink by apache.

the class CheckpointCoordinator method initializeCheckpoint.

/**
 * Initialize the checkpoint trigger asynchronously. It will expected to be executed in io
 * thread due to it might be time-consuming.
 *
 * @param props checkpoint properties
 * @param externalSavepointLocation the external savepoint location, it might be null
 * @return the initialized result, checkpoint id and checkpoint location
 */
private CheckpointIdAndStorageLocation initializeCheckpoint(CheckpointProperties props, @Nullable String externalSavepointLocation, boolean initializeBaseLocations) throws Exception {
    // this must happen outside the coordinator-wide lock, because it
    // communicates
    // with external services (in HA mode) and may block for a while.
    long checkpointID = checkpointIdCounter.getAndIncrement();
    final CheckpointStorageLocation checkpointStorageLocation;
    if (props.isSavepoint()) {
        checkpointStorageLocation = checkpointStorageView.initializeLocationForSavepoint(checkpointID, externalSavepointLocation);
    } else {
        if (initializeBaseLocations) {
            checkpointStorageView.initializeBaseLocationsForCheckpoint();
        }
        checkpointStorageLocation = checkpointStorageView.initializeLocationForCheckpoint(checkpointID);
    }
    return new CheckpointIdAndStorageLocation(checkpointID, checkpointStorageLocation);
}

Example 5 with CheckpointStorageLocation

use of org.apache.flink.runtime.state.CheckpointStorageLocation in project flink by apache.

the class MemoryCheckpointStorageAccessTest method testNonPersistentCheckpointLocation.

@Test
public void testNonPersistentCheckpointLocation() throws Exception {
    MemoryBackendCheckpointStorageAccess storage = new MemoryBackendCheckpointStorageAccess(new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);
    CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);
    CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
    stream.write(99);
    CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
    StreamStateHandle handle = completed.getMetadataHandle();
    assertTrue(handle instanceof ByteStreamStateHandle);
    // the reference is not valid in that case
    try {
        storage.resolveCheckpoint(completed.getExternalPointer());
        fail("should fail with an exception");
    } catch (Exception e) {
    // expected
    }
}