Examples with SavepointRestoreSettings org.apache.flink.runtime.jobgraph.SavepointRestoreSettings used on opensource projects

Example 16 with SavepointRestoreSettings

use of org.apache.flink.runtime.jobgraph.SavepointRestoreSettings in project beam by apache.

the class FlinkRequiresStableInputTest method restoreFromSavepoint.

private JobID restoreFromSavepoint(Pipeline pipeline, String savepointDir) throws ExecutionException, InterruptedException {
    JobGraph jobGraph = getJobGraph(pipeline);
    SavepointRestoreSettings savepointSettings = SavepointRestoreSettings.forPath(savepointDir);
    jobGraph.setSavepointRestoreSettings(savepointSettings);
    return flinkCluster.submitJob(jobGraph).get().getJobID();
}

Example 17 with SavepointRestoreSettings

use of org.apache.flink.runtime.jobgraph.SavepointRestoreSettings in project flink by apache.

the class StreamingJobGraphGeneratorTest method generatorForwardsSavepointRestoreSettings.

@Test
public void generatorForwardsSavepointRestoreSettings() {
    StreamGraph streamGraph = new StreamGraph(new ExecutionConfig(), new CheckpointConfig(), SavepointRestoreSettings.forPath("hello"));
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);
    SavepointRestoreSettings savepointRestoreSettings = jobGraph.getSavepointRestoreSettings();
    assertThat(savepointRestoreSettings.getRestorePath(), is("hello"));
}

Example 18 with SavepointRestoreSettings

use of org.apache.flink.runtime.jobgraph.SavepointRestoreSettings in project flink by apache.

the class JobMasterTest method testCheckpointPrecedesSavepointRecovery.

/**
 * Tests that an existing checkpoint will have precedence over an savepoint.
 */
@Test
public void testCheckpointPrecedesSavepointRecovery() throws Exception {
    // create savepoint data
    final long savepointId = 42L;
    final File savepointFile = createSavepoint(savepointId);
    // set savepoint settings
    final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath("" + savepointFile.getAbsolutePath(), true);
    final JobGraph jobGraph = createJobGraphWithCheckpointing(savepointRestoreSettings);
    final long checkpointId = 1L;
    final CompletedCheckpoint completedCheckpoint = new CompletedCheckpoint(jobGraph.getJobID(), checkpointId, 1L, 1L, Collections.emptyMap(), null, CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION), new DummyCheckpointStorageLocation());
    final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
    completedCheckpointStore.addCheckpointAndSubsumeOldestOne(completedCheckpoint, new CheckpointsCleaner(), () -> {
    });
    final CheckpointRecoveryFactory testingCheckpointRecoveryFactory = PerJobCheckpointRecoveryFactory.withoutCheckpointStoreRecovery(maxCheckpoints -> completedCheckpointStore);
    haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).createJobMaster();
    try {
        // starting the JobMaster should have read the savepoint
        final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint();
        assertThat(savepointCheckpoint, Matchers.notNullValue());
        assertThat(savepointCheckpoint.getCheckpointID(), is(checkpointId));
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}

Example 19 with SavepointRestoreSettings

use of org.apache.flink.runtime.jobgraph.SavepointRestoreSettings in project flink by apache.

the class JobMasterTest method testRestoringFromSavepoint.

/**
 * Tests that a JobMaster will restore the given JobGraph from its savepoint upon initial
 * submission.
 */
@Test
public void testRestoringFromSavepoint() throws Exception {
    // create savepoint data
    final long savepointId = 42L;
    final File savepointFile = createSavepoint(savepointId);
    // set savepoint settings
    final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(savepointFile.getAbsolutePath(), true);
    final JobGraph jobGraph = createJobGraphWithCheckpointing(savepointRestoreSettings);
    final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
    final CheckpointRecoveryFactory testingCheckpointRecoveryFactory = PerJobCheckpointRecoveryFactory.withoutCheckpointStoreRecovery(maxCheckpoints -> completedCheckpointStore);
    haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);
    final JobMaster jobMaster = new JobMasterBuilder(jobGraph, rpcService).withHighAvailabilityServices(haServices).createJobMaster();
    try {
        // we need to start and register the required slots to let the adaptive scheduler
        // restore from the savepoint
        jobMaster.start();
        final OneShotLatch taskSubmitLatch = new OneShotLatch();
        registerSlotsAtJobMaster(1, jobMaster.getSelfGateway(JobMasterGateway.class), jobGraph.getJobID(), new TestingTaskExecutorGatewayBuilder().setSubmitTaskConsumer((taskDeploymentDescriptor, jobMasterId) -> {
            taskSubmitLatch.trigger();
            return CompletableFuture.completedFuture(Acknowledge.get());
        }).createTestingTaskExecutorGateway(), new LocalUnresolvedTaskManagerLocation());
        // wait until a task has submitted because this guarantees that the ExecutionGraph has
        // been created
        taskSubmitLatch.await();
        final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint();
        assertThat(savepointCheckpoint, Matchers.notNullValue());
        assertThat(savepointCheckpoint.getCheckpointID(), is(savepointId));
    } finally {
        RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
    }
}

Example 20 with SavepointRestoreSettings

use of org.apache.flink.runtime.jobgraph.SavepointRestoreSettings in project flink by apache.

the class MiniCluster method checkRestoreModeForRandomizedChangelogStateBackend.

// HACK: temporary hack to make the randomized changelog state backend tests work with forced
// full snapshots. This option should be removed once changelog state backend supports forced
// full snapshots
private void checkRestoreModeForRandomizedChangelogStateBackend(JobGraph jobGraph) {
    final SavepointRestoreSettings savepointRestoreSettings = jobGraph.getSavepointRestoreSettings();
    if (overrideRestoreModeForRandomizedChangelogStateBackend && savepointRestoreSettings.getRestoreMode() == RestoreMode.NO_CLAIM) {
        final Configuration conf = new Configuration();
        SavepointRestoreSettings.toConfiguration(savepointRestoreSettings, conf);
        conf.set(SavepointConfigOptions.RESTORE_MODE, RestoreMode.LEGACY);
        jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.fromConfiguration(conf));
    }
}