Examples with CheckpointStorageAccess org.apache.flink.runtime.state.CheckpointStorageAccess used on opensource projects

Example 6 with CheckpointStorageAccess

use of org.apache.flink.runtime.state.CheckpointStorageAccess in project flink by apache.

the class AbstractFileCheckpointStorageAccessTestBase method testPersistMultipleMetadataOnlyCheckpoints.

// ------------------------------------------------------------------------
// checkpoints
// ------------------------------------------------------------------------
/**
 * Validates that multiple checkpoints from different jobs with the same checkpoint ID do not
 * interfere with each other.
 */
@Test
public void testPersistMultipleMetadataOnlyCheckpoints() throws Exception {
    final FileSystem fs = FileSystem.getLocalFileSystem();
    final Path checkpointDir = new Path(tmp.newFolder().toURI());
    final long checkpointId = 177;
    final CheckpointStorageAccess storage1 = createCheckpointStorage(checkpointDir);
    storage1.initializeBaseLocationsForCheckpoint();
    final CheckpointStorageAccess storage2 = createCheckpointStorage(checkpointDir);
    storage2.initializeBaseLocationsForCheckpoint();
    final CheckpointStorageLocation loc1 = storage1.initializeLocationForCheckpoint(checkpointId);
    final CheckpointStorageLocation loc2 = storage2.initializeLocationForCheckpoint(checkpointId);
    final byte[] data1 = { 77, 66, 55, 99, 88 };
    final byte[] data2 = { 1, 3, 2, 5, 4 };
    final CompletedCheckpointStorageLocation completedLocation1;
    try (CheckpointMetadataOutputStream out = loc1.createMetadataOutputStream()) {
        out.write(data1);
        completedLocation1 = out.closeAndFinalizeCheckpoint();
    }
    final String result1 = completedLocation1.getExternalPointer();
    final CompletedCheckpointStorageLocation completedLocation2;
    try (CheckpointMetadataOutputStream out = loc2.createMetadataOutputStream()) {
        out.write(data2);
        completedLocation2 = out.closeAndFinalizeCheckpoint();
    }
    final String result2 = completedLocation2.getExternalPointer();
    // check that this went to a file, but in a nested directory structure
    // one directory per storage
    FileStatus[] files = fs.listStatus(checkpointDir);
    assertEquals(2, files.length);
    // in each per-storage directory, one for the checkpoint
    FileStatus[] job1Files = fs.listStatus(files[0].getPath());
    FileStatus[] job2Files = fs.listStatus(files[1].getPath());
    assertTrue(job1Files.length >= 1);
    assertTrue(job2Files.length >= 1);
    assertTrue(fs.exists(new Path(result1, AbstractFsCheckpointStorageAccess.METADATA_FILE_NAME)));
    assertTrue(fs.exists(new Path(result2, AbstractFsCheckpointStorageAccess.METADATA_FILE_NAME)));
    // check that both storages can resolve each others contents
    validateContents(storage1.resolveCheckpoint(result1).getMetadataHandle(), data1);
    validateContents(storage1.resolveCheckpoint(result2).getMetadataHandle(), data2);
    validateContents(storage2.resolveCheckpoint(result1).getMetadataHandle(), data1);
    validateContents(storage2.resolveCheckpoint(result2).getMetadataHandle(), data2);
}

Example 7 with CheckpointStorageAccess

use of org.apache.flink.runtime.state.CheckpointStorageAccess in project flink by apache.

the class AbstractFileCheckpointStorageAccessTestBase method testFailingPointerPathResolution.

@Test
public void testFailingPointerPathResolution() throws Exception {
    // create the storage for some random checkpoint directory
    final CheckpointStorageAccess storage = createCheckpointStorage(randomTempPath());
    // null value
    try {
        storage.resolveCheckpoint(null);
        fail("expected exception");
    } catch (NullPointerException ignored) {
    }
    // empty string
    try {
        storage.resolveCheckpoint("");
        fail("expected exception");
    } catch (IllegalArgumentException ignored) {
    }
    // not a file path at all
    try {
        storage.resolveCheckpoint("this-is_not/a#filepath.at.all");
        fail("expected exception");
    } catch (IOException ignored) {
    }
    // non-existing file
    try {
        storage.resolveCheckpoint(tmp.newFile().toURI().toString() + "_not_existing");
        fail("expected exception");
    } catch (IOException ignored) {
    }
}

Example 8 with CheckpointStorageAccess

use of org.apache.flink.runtime.state.CheckpointStorageAccess in project flink by apache.

the class AbstractFileCheckpointStorageAccessTestBase method writeToAlreadyExistingCheckpointFails.

@Test
public void writeToAlreadyExistingCheckpointFails() throws Exception {
    final byte[] data = { 8, 8, 4, 5, 2, 6, 3 };
    final long checkpointId = 177;
    final CheckpointStorageAccess storage = createCheckpointStorage(randomTempPath());
    storage.initializeBaseLocationsForCheckpoint();
    final CheckpointStorageLocation loc = storage.initializeLocationForCheckpoint(checkpointId);
    try (CheckpointMetadataOutputStream out = loc.createMetadataOutputStream()) {
        out.write(data);
        out.closeAndFinalizeCheckpoint();
    }
    // create another writer to the metadata file for the checkpoint
    try {
        loc.createMetadataOutputStream();
        fail("this should fail with an exception");
    } catch (IOException ignored) {
    }
}