Examples with JobManagerCheckpointStorage org.apache.flink.runtime.state.storage.JobManagerCheckpointStorage used on opensource projects

Example 1 with JobManagerCheckpointStorage

use of org.apache.flink.runtime.state.storage.JobManagerCheckpointStorage in project flink by apache.

the class Checkpoints method loadCheckpointStorage.

@Nonnull
public static CheckpointStorage loadCheckpointStorage(Configuration configuration, ClassLoader classLoader, @Nullable Logger logger) {
    StateBackend backend = loadStateBackend(configuration, classLoader, logger);
    if (logger != null) {
        logger.info("Attempting to load configured checkpoint storage for savepoint disposal");
    }
    CheckpointStorage checkpointStorage = null;
    try {
        checkpointStorage = CheckpointStorageLoader.load(null, null, backend, configuration, classLoader, null);
    } catch (Throwable t) {
        // catches exceptions and errors (like linking errors)
        if (logger != null) {
            logger.info("Could not load configured state backend.");
            logger.debug("Detailed exception:", t);
        }
    }
    if (checkpointStorage == null) {
        // FileSystem-based for metadata
        return new JobManagerCheckpointStorage();
    }
    return checkpointStorage;
}

Example 2 with JobManagerCheckpointStorage

use of org.apache.flink.runtime.state.storage.JobManagerCheckpointStorage in project flink by apache.

the class CheckpointStorageLoaderTest method testConfigureJobManagerStorageWithParameters.

/**
 * Tests that job parameters take precedence over cluster configurations.
 */
@Test
public void testConfigureJobManagerStorageWithParameters() throws Exception {
    final String savepointDirConfig = new Path(tmp.newFolder().toURI()).toString();
    final Path savepointDirJob = new Path(tmp.newFolder().toURI());
    final Configuration config = new Configuration();
    config.set(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDirConfig);
    CheckpointStorage storage = CheckpointStorageLoader.load(new JobManagerCheckpointStorage(), savepointDirJob, new ModernStateBackend(), config, cl, log);
    Assert.assertThat(storage, Matchers.instanceOf(JobManagerCheckpointStorage.class));
    JobManagerCheckpointStorage jmStorage = (JobManagerCheckpointStorage) storage;
    Assert.assertThat(jmStorage.getSavepointPath(), normalizedPath(savepointDirJob));
}

Example 3 with JobManagerCheckpointStorage

use of org.apache.flink.runtime.state.storage.JobManagerCheckpointStorage in project flink by apache.

the class CheckpointStorageLoaderTest method testLoadJobManagerStorageWithParameters.

/**
 * Validates loading a job manager checkpoint storage with additional parameters from the
 * cluster configuration.
 */
@Test
public void testLoadJobManagerStorageWithParameters() throws Exception {
    final String savepointDir = new Path(tmp.newFolder().toURI()).toString();
    final Path expectedSavepointPath = new Path(savepointDir);
    // we configure with the explicit string (rather than
    // AbstractStateBackend#X_STATE_BACKEND_NAME)
    // to guard against config-breaking changes of the name
    final Configuration config1 = new Configuration();
    config1.set(CheckpointingOptions.CHECKPOINT_STORAGE, "jobmanager");
    config1.set(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
    CheckpointStorage storage1 = CheckpointStorageLoader.fromConfig(config1, cl, null).get();
    Assert.assertThat(storage1, Matchers.instanceOf(JobManagerCheckpointStorage.class));
    Assert.assertEquals(expectedSavepointPath, ((JobManagerCheckpointStorage) storage1).getSavepointPath());
}

Example 4 with JobManagerCheckpointStorage

use of org.apache.flink.runtime.state.storage.JobManagerCheckpointStorage in project flink by apache.

the class UnalignedCheckpointFailureHandlingITCase method testCheckpointSuccessAfterFailure.

@Test
public void testCheckpointSuccessAfterFailure() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    TestCheckpointStorage storage = new TestCheckpointStorage(new JobManagerCheckpointStorage(), sharedObjects, temporaryFolder);
    configure(env, storage);
    buildGraph(env);
    JobClient jobClient = env.executeAsync();
    JobID jobID = jobClient.getJobID();
    MiniCluster miniCluster = miniClusterResource.getMiniCluster();
    waitForJobStatus(jobClient, singletonList(RUNNING), fromNow(Duration.ofSeconds(30)));
    waitForAllTaskRunning(miniCluster, jobID, false);
    triggerFailingCheckpoint(jobID, TestException.class, miniCluster);
    miniCluster.triggerCheckpoint(jobID).get();
}

Example 5 with JobManagerCheckpointStorage

use of org.apache.flink.runtime.state.storage.JobManagerCheckpointStorage in project flink by apache.

the class CheckpointStorageLoaderTest method testMemoryBackendHighAvailabilityDefault.

private void testMemoryBackendHighAvailabilityDefault(String haPersistenceDir, Path checkpointPath) throws Exception {
    final Configuration config1 = new Configuration();
    config1.set(HighAvailabilityOptions.HA_MODE, "zookeeper");
    config1.set(HighAvailabilityOptions.HA_CLUSTER_ID, "myCluster");
    config1.set(HighAvailabilityOptions.HA_STORAGE_PATH, haPersistenceDir);
    final Configuration config2 = new Configuration();
    config2.set(CheckpointingOptions.CHECKPOINT_STORAGE, "jobmanager");
    config2.set(HighAvailabilityOptions.HA_MODE, "zookeeper");
    config2.set(HighAvailabilityOptions.HA_CLUSTER_ID, "myCluster");
    config2.set(HighAvailabilityOptions.HA_STORAGE_PATH, haPersistenceDir);
    if (checkpointPath != null) {
        config1.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointPath.toUri().toString());
        config2.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointPath.toUri().toString());
    }
    final JobManagerCheckpointStorage storage = new JobManagerCheckpointStorage();
    final CheckpointStorage loaded1 = CheckpointStorageLoader.load(storage, null, new ModernStateBackend(), config1, cl, log);
    final CheckpointStorage loaded2 = CheckpointStorageLoader.load(null, null, new ModernStateBackend(), config2, cl, log);
    Assert.assertThat(loaded1, Matchers.instanceOf(JobManagerCheckpointStorage.class));
    Assert.assertThat(loaded2, Matchers.instanceOf(JobManagerCheckpointStorage.class));
    final JobManagerCheckpointStorage memStorage1 = (JobManagerCheckpointStorage) loaded1;
    final JobManagerCheckpointStorage memStorage2 = (JobManagerCheckpointStorage) loaded2;
    Assert.assertNull(memStorage1.getSavepointPath());
    Assert.assertNull(memStorage2.getSavepointPath());
    if (checkpointPath != null) {
        Assert.assertThat(memStorage1.getCheckpointPath(), normalizedPath(checkpointPath));
        Assert.assertThat(memStorage2.getCheckpointPath(), normalizedPath(checkpointPath));
    } else {
        Assert.assertNull(memStorage1.getCheckpointPath());
        Assert.assertNull(memStorage2.getCheckpointPath());
    }
}