Examples with JobCheckpointingSettings org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings used on opensource projects

Example 16 with JobCheckpointingSettings

use of org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings in project flink by apache.

the class TestUtils method createJobGraphFromJobVerticesWithCheckpointing.

@Nonnull
public static JobGraph createJobGraphFromJobVerticesWithCheckpointing(SavepointRestoreSettings savepointRestoreSettings, JobVertex... jobVertices) {
    // enable checkpointing which is required to resume from a savepoint
    final CheckpointCoordinatorConfiguration checkpointCoordinatorConfiguration = CheckpointCoordinatorConfiguration.builder().setCheckpointInterval(1000L).setCheckpointTimeout(1000L).setMinPauseBetweenCheckpoints(1000L).setMaxConcurrentCheckpoints(1).setCheckpointRetentionPolicy(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION).setExactlyOnce(true).setUnalignedCheckpointsEnabled(false).setTolerableCheckpointFailureNumber(0).build();
    final JobCheckpointingSettings checkpointingSettings = new JobCheckpointingSettings(checkpointCoordinatorConfiguration, null);
    return JobGraphBuilder.newStreamingJobGraphBuilder().addJobVertices(Arrays.asList(jobVertices)).setJobCheckpointingSettings(checkpointingSettings).setSavepointRestoreSettings(savepointRestoreSettings).build();
}

Example 17 with JobCheckpointingSettings

use of org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings in project flink by apache.

the class AdaptiveSchedulerTest method testArchivedCheckpointingSettingsNotNullIfCheckpointingIsEnabled.

@Test
public void testArchivedCheckpointingSettingsNotNullIfCheckpointingIsEnabled() throws Exception {
    final JobGraph jobGraph = createJobGraph();
    jobGraph.setSnapshotSettings(new JobCheckpointingSettings(CheckpointCoordinatorConfiguration.builder().build(), null));
    final ArchivedExecutionGraph archivedExecutionGraph = new AdaptiveSchedulerBuilder(jobGraph, mainThreadExecutor).build().getArchivedExecutionGraph(JobStatus.INITIALIZING, null);
    ArchivedExecutionGraphTest.assertContainsCheckpointSettings(archivedExecutionGraph);
}

Example 18 with JobCheckpointingSettings

use of org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings in project flink by apache.

the class AdaptiveSchedulerTest method testCloseShutsDownCheckpointingComponents.

@Test
public void testCloseShutsDownCheckpointingComponents() throws Exception {
    final CompletableFuture<JobStatus> completedCheckpointStoreShutdownFuture = new CompletableFuture<>();
    final CompletedCheckpointStore completedCheckpointStore = TestingCompletedCheckpointStore.createStoreWithShutdownCheckAndNoCompletedCheckpoints(completedCheckpointStoreShutdownFuture);
    final CompletableFuture<JobStatus> checkpointIdCounterShutdownFuture = new CompletableFuture<>();
    final CheckpointIDCounter checkpointIdCounter = TestingCheckpointIDCounter.createStoreWithShutdownCheckAndNoStartAction(checkpointIdCounterShutdownFuture);
    final JobGraph jobGraph = createJobGraph();
    // checkpointing components are only created if checkpointing is enabled
    jobGraph.setSnapshotSettings(new JobCheckpointingSettings(CheckpointCoordinatorConfiguration.builder().build(), null));
    final AdaptiveScheduler scheduler = new AdaptiveSchedulerBuilder(jobGraph, singleThreadMainThreadExecutor).setCheckpointRecoveryFactory(new TestingCheckpointRecoveryFactory(completedCheckpointStore, checkpointIdCounter)).build();
    singleThreadMainThreadExecutor.execute(() -> {
        scheduler.startScheduling();
        // transition into the FAILED state
        scheduler.handleGlobalFailure(new FlinkException("Test exception"));
        scheduler.closeAsync();
    });
    assertThat(completedCheckpointStoreShutdownFuture.get()).isEqualTo(JobStatus.FAILED);
    assertThat(checkpointIdCounterShutdownFuture.get()).isEqualTo(JobStatus.FAILED);
}

Example 19 with JobCheckpointingSettings

use of org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings in project flink by apache.

the class JobMasterStopWithSavepointITCase method setUpJobGraph.

private void setUpJobGraph(final Class<? extends TaskInvokable> invokable, final RestartStrategies.RestartStrategyConfiguration restartStrategy) throws Exception {
    finishingLatch = new OneShotLatch();
    invokeLatch = new CountDownLatch(PARALLELISM);
    numberOfRestarts = new CountDownLatch(2);
    checkpointsToWaitFor = new CountDownLatch(10);
    syncSavepointId.set(-1);
    savepointDirectory = temporaryFolder.newFolder().toPath();
    Assume.assumeTrue("ClusterClient is not an instance of MiniClusterClient", MINI_CLUSTER_RESOURCE.getClusterClient() instanceof MiniClusterClient);
    clusterClient = (MiniClusterClient) MINI_CLUSTER_RESOURCE.getClusterClient();
    final ExecutionConfig config = new ExecutionConfig();
    config.setRestartStrategy(restartStrategy);
    final JobVertex vertex = new JobVertex("testVertex");
    vertex.setInvokableClass(invokable);
    vertex.setParallelism(PARALLELISM);
    final JobCheckpointingSettings jobCheckpointingSettings = new JobCheckpointingSettings(new CheckpointCoordinatorConfiguration(CHECKPOINT_INTERVAL, 60_000, 10, 1, CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION, true, false, 0, 0), null);
    jobGraph = JobGraphBuilder.newStreamingJobGraphBuilder().setExecutionConfig(config).addJobVertex(vertex).setJobCheckpointingSettings(jobCheckpointingSettings).build();
    clusterClient.submitJob(jobGraph).get();
    assertTrue(invokeLatch.await(60, TimeUnit.SECONDS));
    waitForJob();
}

Example 20 with JobCheckpointingSettings

use of org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings in project flink by apache.

the class JobMasterTriggerSavepointITCase method setUpWithCheckpointInterval.

private void setUpWithCheckpointInterval(long checkpointInterval) throws Exception {
    invokeLatch = new CountDownLatch(1);
    triggerCheckpointLatch = new CountDownLatch(1);
    savepointDirectory = temporaryFolder.newFolder().toPath();
    Assume.assumeTrue("ClusterClient is not an instance of MiniClusterClient", MINI_CLUSTER_RESOURCE.getClusterClient() instanceof MiniClusterClient);
    clusterClient = (MiniClusterClient) MINI_CLUSTER_RESOURCE.getClusterClient();
    final JobVertex vertex = new JobVertex("testVertex");
    vertex.setInvokableClass(NoOpBlockingInvokable.class);
    vertex.setParallelism(1);
    final JobCheckpointingSettings jobCheckpointingSettings = new JobCheckpointingSettings(new CheckpointCoordinatorConfiguration(checkpointInterval, 60_000, 10, 1, CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION, true, false, 0, 0), null);
    jobGraph = JobGraphBuilder.newStreamingJobGraphBuilder().addJobVertex(vertex).setJobCheckpointingSettings(jobCheckpointingSettings).build();
    clusterClient.submitJob(jobGraph).get();
    assertTrue(invokeLatch.await(60, TimeUnit.SECONDS));
    waitForJob();
}