Examples with CheckpointCoordinatorConfiguration org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration used on opensource projects

Example 11 with CheckpointCoordinatorConfiguration

use of org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration in project flink by apache.

the class DefaultExecutionGraphDeploymentTest method createExecutionGraph.

private ExecutionGraph createExecutionGraph(Configuration configuration) throws Exception {
    final JobGraph jobGraph = JobGraphTestUtils.emptyJobGraph();
    jobGraph.setSnapshotSettings(new JobCheckpointingSettings(new CheckpointCoordinatorConfiguration(100, 10 * 60 * 1000, 0, 1, CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION, false, false, 0, 0), null));
    return TestingDefaultExecutionGraphBuilder.newBuilder().setJobGraph(jobGraph).setJobMasterConfig(configuration).setBlobWriter(blobWriter).build();
}

Example 12 with CheckpointCoordinatorConfiguration

use of org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration in project flink by apache.

the class JobDispatcherITCase method generateAndPersistJobGraph.

private JobID generateAndPersistJobGraph(Configuration configuration, long checkpointInterval, Path tmpPath) throws Exception {
    final JobVertex jobVertex = new JobVertex("jobVertex");
    jobVertex.setInvokableClass(AtLeastOneCheckpointInvokable.class);
    jobVertex.setParallelism(1);
    final CheckpointCoordinatorConfiguration checkpointCoordinatorConfiguration = CheckpointCoordinatorConfiguration.builder().setCheckpointInterval(checkpointInterval).build();
    final JobCheckpointingSettings checkpointingSettings = new JobCheckpointingSettings(checkpointCoordinatorConfiguration, null);
    final JobGraph jobGraph = JobGraphBuilder.newStreamingJobGraphBuilder().addJobVertex(jobVertex).setJobCheckpointingSettings(checkpointingSettings).build();
    final Path jobGraphPath = tmpPath.resolve(JOB_GRAPH_FILE_PATH.defaultValue());
    try (ObjectOutputStream objectOutputStream = new ObjectOutputStream(Files.newOutputStream(jobGraphPath, CREATE))) {
        objectOutputStream.writeObject(jobGraph);
    }
    configuration.setString(JOB_GRAPH_FILE_PATH.key(), jobGraphPath.toString());
    return jobGraph.getJobID();
}

Example 13 with CheckpointCoordinatorConfiguration

use of org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration in project flink by apache.

the class JMXJobManagerMetricTest method testJobManagerJMXMetricAccess.

/**
 * Tests that metrics registered on the JobManager are actually accessible via JMX.
 */
@Test
void testJobManagerJMXMetricAccess(@InjectClusterClient ClusterClient<?> client) throws Exception {
    Deadline deadline = Deadline.now().plus(Duration.ofMinutes(2));
    try {
        JobVertex sourceJobVertex = new JobVertex("Source");
        sourceJobVertex.setInvokableClass(BlockingInvokable.class);
        sourceJobVertex.setParallelism(1);
        final JobCheckpointingSettings jobCheckpointingSettings = new JobCheckpointingSettings(new CheckpointCoordinatorConfiguration(500, 500, 50, 5, CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION, true, false, 0, 0), null);
        final JobGraph jobGraph = JobGraphBuilder.newStreamingJobGraphBuilder().setJobName("TestingJob").addJobVertex(sourceJobVertex).setJobCheckpointingSettings(jobCheckpointingSettings).build();
        client.submitJob(jobGraph).get();
        FutureUtils.retrySuccessfulWithDelay(() -> client.getJobStatus(jobGraph.getJobID()), Time.milliseconds(10), deadline, status -> status == JobStatus.RUNNING, TestingUtils.defaultScheduledExecutor()).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
        MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer();
        Set<ObjectName> nameSet = mBeanServer.queryNames(new ObjectName("org.apache.flink.jobmanager.job.lastCheckpointSize:job_name=TestingJob,*"), null);
        assertThat(nameSet).hasSize(1);
        assertThat(mBeanServer.getAttribute(nameSet.iterator().next(), "Value")).isEqualTo(-1L);
        BlockingInvokable.unblock();
    } finally {
        BlockingInvokable.unblock();
    }
}

Example 14 with CheckpointCoordinatorConfiguration

use of org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration in project flink by apache.

the class SchedulerTestingUtils method enableCheckpointing.

public static void enableCheckpointing(final JobGraph jobGraph, @Nullable StateBackend stateBackend, @Nullable CheckpointStorage checkpointStorage) {
    final CheckpointCoordinatorConfiguration config = new CheckpointCoordinatorConfiguration(// disable periodical checkpointing
    Long.MAX_VALUE, DEFAULT_CHECKPOINT_TIMEOUT_MS, 0, 1, CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION, false, false, 0, 0);
    SerializedValue<StateBackend> serializedStateBackend = null;
    if (stateBackend != null) {
        try {
            serializedStateBackend = new SerializedValue<>(stateBackend);
        } catch (IOException e) {
            throw new RuntimeException("could not serialize state backend", e);
        }
    }
    SerializedValue<CheckpointStorage> serializedCheckpointStorage = null;
    if (checkpointStorage != null) {
        try {
            serializedCheckpointStorage = new SerializedValue<>(checkpointStorage);
        } catch (IOException e) {
            throw new RuntimeException("could not serialize checkpoint storage", e);
        }
    }
    jobGraph.setSnapshotSettings(new JobCheckpointingSettings(config, serializedStateBackend, TernaryBoolean.UNDEFINED, serializedCheckpointStorage, null));
}

Example 15 with CheckpointCoordinatorConfiguration

use of org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration in project flink by apache.

the class DefaultSchedulerCheckpointCoordinatorTest method createSchedulerAndEnableCheckpointing.

private DefaultScheduler createSchedulerAndEnableCheckpointing(CheckpointIDCounter counter, CompletedCheckpointStore store) throws Exception {
    final Time timeout = Time.days(1L);
    final JobVertex jobVertex = new JobVertex("MockVertex");
    jobVertex.setInvokableClass(AbstractInvokable.class);
    final CheckpointCoordinatorConfiguration chkConfig = CheckpointCoordinatorConfiguration.builder().setCheckpointInterval(100L).setCheckpointTimeout(100L).build();
    final JobCheckpointingSettings checkpointingSettings = new JobCheckpointingSettings(chkConfig, null);
    final JobGraph jobGraph = JobGraphBuilder.newStreamingJobGraphBuilder().addJobVertex(jobVertex).setJobCheckpointingSettings(checkpointingSettings).build();
    return SchedulerTestingUtils.newSchedulerBuilder(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).setCheckpointRecoveryFactory(new TestingCheckpointRecoveryFactory(store, counter)).setRpcTimeout(timeout).build();
}