Examples with CheckpointMetadataOutputStream org.apache.flink.runtime.state.CheckpointMetadataOutputStream used on opensource projects

Example 6 with CheckpointMetadataOutputStream

use of org.apache.flink.runtime.state.CheckpointMetadataOutputStream in project flink by apache.

the class MemoryCheckpointStorageAccessTest method testNonPersistentCheckpointLocation.

@Test
public void testNonPersistentCheckpointLocation() throws Exception {
    MemoryBackendCheckpointStorageAccess storage = new MemoryBackendCheckpointStorageAccess(new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);
    CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);
    CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
    stream.write(99);
    CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
    StreamStateHandle handle = completed.getMetadataHandle();
    assertTrue(handle instanceof ByteStreamStateHandle);
    // the reference is not valid in that case
    try {
        storage.resolveCheckpoint(completed.getExternalPointer());
        fail("should fail with an exception");
    } catch (Exception e) {
    // expected
    }
}

Example 7 with CheckpointMetadataOutputStream

use of org.apache.flink.runtime.state.CheckpointMetadataOutputStream in project flink by apache.

the class AbstractFileCheckpointStorageAccessTestBase method testPersistMultipleMetadataOnlyCheckpoints.

// ------------------------------------------------------------------------
// checkpoints
// ------------------------------------------------------------------------
/**
 * Validates that multiple checkpoints from different jobs with the same checkpoint ID do not
 * interfere with each other.
 */
@Test
public void testPersistMultipleMetadataOnlyCheckpoints() throws Exception {
    final FileSystem fs = FileSystem.getLocalFileSystem();
    final Path checkpointDir = new Path(tmp.newFolder().toURI());
    final long checkpointId = 177;
    final CheckpointStorageAccess storage1 = createCheckpointStorage(checkpointDir);
    storage1.initializeBaseLocationsForCheckpoint();
    final CheckpointStorageAccess storage2 = createCheckpointStorage(checkpointDir);
    storage2.initializeBaseLocationsForCheckpoint();
    final CheckpointStorageLocation loc1 = storage1.initializeLocationForCheckpoint(checkpointId);
    final CheckpointStorageLocation loc2 = storage2.initializeLocationForCheckpoint(checkpointId);
    final byte[] data1 = { 77, 66, 55, 99, 88 };
    final byte[] data2 = { 1, 3, 2, 5, 4 };
    final CompletedCheckpointStorageLocation completedLocation1;
    try (CheckpointMetadataOutputStream out = loc1.createMetadataOutputStream()) {
        out.write(data1);
        completedLocation1 = out.closeAndFinalizeCheckpoint();
    }
    final String result1 = completedLocation1.getExternalPointer();
    final CompletedCheckpointStorageLocation completedLocation2;
    try (CheckpointMetadataOutputStream out = loc2.createMetadataOutputStream()) {
        out.write(data2);
        completedLocation2 = out.closeAndFinalizeCheckpoint();
    }
    final String result2 = completedLocation2.getExternalPointer();
    // check that this went to a file, but in a nested directory structure
    // one directory per storage
    FileStatus[] files = fs.listStatus(checkpointDir);
    assertEquals(2, files.length);
    // in each per-storage directory, one for the checkpoint
    FileStatus[] job1Files = fs.listStatus(files[0].getPath());
    FileStatus[] job2Files = fs.listStatus(files[1].getPath());
    assertTrue(job1Files.length >= 1);
    assertTrue(job2Files.length >= 1);
    assertTrue(fs.exists(new Path(result1, AbstractFsCheckpointStorageAccess.METADATA_FILE_NAME)));
    assertTrue(fs.exists(new Path(result2, AbstractFsCheckpointStorageAccess.METADATA_FILE_NAME)));
    // check that both storages can resolve each others contents
    validateContents(storage1.resolveCheckpoint(result1).getMetadataHandle(), data1);
    validateContents(storage1.resolveCheckpoint(result2).getMetadataHandle(), data2);
    validateContents(storage2.resolveCheckpoint(result1).getMetadataHandle(), data1);
    validateContents(storage2.resolveCheckpoint(result2).getMetadataHandle(), data2);
}

Example 8 with CheckpointMetadataOutputStream

use of org.apache.flink.runtime.state.CheckpointMetadataOutputStream in project flink by apache.

the class AbstractFileCheckpointStorageAccessTestBase method writeToAlreadyExistingCheckpointFails.

@Test
public void writeToAlreadyExistingCheckpointFails() throws Exception {
    final byte[] data = { 8, 8, 4, 5, 2, 6, 3 };
    final long checkpointId = 177;
    final CheckpointStorageAccess storage = createCheckpointStorage(randomTempPath());
    storage.initializeBaseLocationsForCheckpoint();
    final CheckpointStorageLocation loc = storage.initializeLocationForCheckpoint(checkpointId);
    try (CheckpointMetadataOutputStream out = loc.createMetadataOutputStream()) {
        out.write(data);
        out.closeAndFinalizeCheckpoint();
    }
    // create another writer to the metadata file for the checkpoint
    try {
        loc.createMetadataOutputStream();
        fail("this should fail with an exception");
    } catch (IOException ignored) {
    }
}