Examples with CheckpointStorageAccess org.apache.flink.runtime.state.CheckpointStorageAccess used on opensource projects

Example 1 with CheckpointStorageAccess

use of org.apache.flink.runtime.state.CheckpointStorageAccess in project flink by apache.

the class FsCheckpointStorageAccessTest method testNotDuplicationCheckpointStateToolset.

@Test
public void testNotDuplicationCheckpointStateToolset() throws Exception {
    CheckpointStorageAccess checkpointStorage = createCheckpointStorage(randomTempPath());
    assertThat(checkpointStorage.createTaskOwnedCheckpointStateToolset(), instanceOf(NotDuplicatingCheckpointStateToolset.class));
}

Example 2 with CheckpointStorageAccess

use of org.apache.flink.runtime.state.CheckpointStorageAccess in project flink by apache.

the class AbstractFileCheckpointStorageAccessTestBase method testPointerPathResolution.

// ------------------------------------------------------------------------
// pointers
// ------------------------------------------------------------------------
@Test
public void testPointerPathResolution() throws Exception {
    final FileSystem fs = FileSystem.getLocalFileSystem();
    final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorageAccess.METADATA_FILE_NAME);
    final String basePointer = metadataFile.getParent().toString();
    final String pointer1 = metadataFile.toString();
    final String pointer2 = metadataFile.getParent().toString();
    final String pointer3 = metadataFile.getParent().toString() + '/';
    // create the storage for some random checkpoint directory
    final CheckpointStorageAccess storage = createCheckpointStorage(randomTempPath());
    final byte[] data = new byte[23686];
    new Random().nextBytes(data);
    try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
        out.write(data);
    }
    CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
    CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
    CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);
    assertEquals(basePointer, completed1.getExternalPointer());
    assertEquals(basePointer, completed2.getExternalPointer());
    assertEquals(basePointer, completed3.getExternalPointer());
    StreamStateHandle handle1 = completed1.getMetadataHandle();
    StreamStateHandle handle2 = completed2.getMetadataHandle();
    StreamStateHandle handle3 = completed3.getMetadataHandle();
    assertNotNull(handle1);
    assertNotNull(handle2);
    assertNotNull(handle3);
    validateContents(handle1, data);
    validateContents(handle2, data);
    validateContents(handle3, data);
}

Example 3 with CheckpointStorageAccess

use of org.apache.flink.runtime.state.CheckpointStorageAccess in project flink by apache.

the class AbstractFileCheckpointStorageAccessTestBase method testNoSavepointPathConfiguredNoTarget.

@Test
public void testNoSavepointPathConfiguredNoTarget() throws Exception {
    final CheckpointStorageAccess storage = createCheckpointStorage(randomTempPath());
    try {
        storage.initializeLocationForSavepoint(1337, null);
        fail("this should fail with an exception");
    } catch (IllegalArgumentException ignored) {
    }
}

Example 4 with CheckpointStorageAccess

use of org.apache.flink.runtime.state.CheckpointStorageAccess in project flink by apache.

the class AbstractFileCheckpointStorageAccessTestBase method testSavepoint.

private void testSavepoint(@Nullable Path savepointDir, @Nullable Path customDir, Path expectedParent) throws Exception {
    final CheckpointStorageAccess storage = savepointDir == null ? createCheckpointStorage(randomTempPath()) : createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);
    final String customLocation = customDir == null ? null : customDir.toString();
    final CheckpointStorageLocation savepointLocation = storage.initializeLocationForSavepoint(52452L, customLocation);
    final byte[] data = { 77, 66, 55, 99, 88 };
    final CompletedCheckpointStorageLocation completed;
    try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
        out.write(data);
        completed = out.closeAndFinalizeCheckpoint();
    }
    // we need to do this step to make sure we have a slash (or not) in the same way as the
    // expected path has it
    final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));
    assertEquals(expectedParent, normalizedWithSlash);
    validateContents(completed.getMetadataHandle(), data);
    // validate that the correct directory was used
    FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();
    // we need to recreate that path in the same way as the "expected path" (via File and URI)
    // to make
    // sure the either both use '/' suffixes, or neither uses them (a bit of an annoying
    // ambiguity)
    Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());
    assertEquals(expectedParent, usedSavepointDir);
}

Example 5 with CheckpointStorageAccess

use of org.apache.flink.runtime.state.CheckpointStorageAccess in project flink by apache.

the class FsCheckpointStorageAccessTest method testDuplicationCheckpointStateToolset.

@Test
public void testDuplicationCheckpointStateToolset() throws Exception {
    CheckpointStorageAccess checkpointStorage = new FsCheckpointStorageAccess(new TestDuplicatingFileSystem(), randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);
    assertThat(checkpointStorage.createTaskOwnedCheckpointStateToolset(), instanceOf(FsCheckpointStateToolset.class));
}