Examples with FsCheckpointStreamFactory org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory used on opensource projects

Example 1 with FsCheckpointStreamFactory

use of org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory in project flink by apache.

the class ChannelStateCheckpointWriterTest method testSmallFilesNotWritten.

@Test
@SuppressWarnings("ConstantConditions")
public void testSmallFilesNotWritten() throws Exception {
    int threshold = 100;
    File checkpointsDir = temporaryFolder.newFolder("checkpointsDir");
    File sharedStateDir = temporaryFolder.newFolder("sharedStateDir");
    FsCheckpointStreamFactory checkpointStreamFactory = new FsCheckpointStreamFactory(getSharedInstance(), fromLocalFile(checkpointsDir), fromLocalFile(sharedStateDir), threshold, threshold);
    ChannelStateWriteResult result = new ChannelStateWriteResult();
    ChannelStateCheckpointWriter writer = createWriter(result, checkpointStreamFactory.createCheckpointStateOutputStream(EXCLUSIVE));
    NetworkBuffer buffer = new NetworkBuffer(MemorySegmentFactory.allocateUnpooledSegment(threshold / 2), FreeingBufferRecycler.INSTANCE);
    writer.writeInput(new InputChannelInfo(1, 2), buffer);
    writer.completeOutput();
    writer.completeInput();
    assertTrue(result.isDone());
    assertEquals(0, checkpointsDir.list().length);
    assertEquals(0, sharedStateDir.list().length);
}

Example 2 with FsCheckpointStreamFactory

use of org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory in project flink by apache.

the class ChannelStateCheckpointWriterTest method testFileHandleSize.

@Test
public void testFileHandleSize() throws Exception {
    int numChannels = 3;
    int numWritesPerChannel = 4;
    int numBytesPerWrite = 5;
    ChannelStateWriteResult result = new ChannelStateWriteResult();
    ChannelStateCheckpointWriter writer = createWriter(result, new FsCheckpointStreamFactory(getSharedInstance(), fromLocalFile(temporaryFolder.newFolder("checkpointsDir")), fromLocalFile(temporaryFolder.newFolder("sharedStateDir")), numBytesPerWrite - 1, numBytesPerWrite - 1).createCheckpointStateOutputStream(EXCLUSIVE));
    InputChannelInfo[] channels = IntStream.range(0, numChannels).mapToObj(i -> new InputChannelInfo(0, i)).toArray(InputChannelInfo[]::new);
    for (int call = 0; call < numWritesPerChannel; call++) {
        for (int channel = 0; channel < numChannels; channel++) {
            write(writer, channels[channel], getData(numBytesPerWrite));
        }
    }
    writer.completeInput();
    writer.completeOutput();
    for (InputChannelStateHandle handle : result.inputChannelStateHandles.get()) {
        assertEquals((Integer.BYTES + numBytesPerWrite) * numWritesPerChannel, handle.getStateSize());
    }
}

Example 3 with FsCheckpointStreamFactory

use of org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory in project flink by apache.

the class RocksDBStateUploaderTest method testMultiThreadUploadCorrectly.

/**
 * Test that upload files with multi-thread correctly.
 */
@Test
public void testMultiThreadUploadCorrectly() throws Exception {
    File checkpointPrivateFolder = temporaryFolder.newFolder("private");
    org.apache.flink.core.fs.Path checkpointPrivateDirectory = org.apache.flink.core.fs.Path.fromLocalFile(checkpointPrivateFolder);
    File checkpointSharedFolder = temporaryFolder.newFolder("shared");
    org.apache.flink.core.fs.Path checkpointSharedDirectory = org.apache.flink.core.fs.Path.fromLocalFile(checkpointSharedFolder);
    FileSystem fileSystem = checkpointPrivateDirectory.getFileSystem();
    int fileStateSizeThreshold = 1024;
    int writeBufferSize = 4096;
    FsCheckpointStreamFactory checkpointStreamFactory = new FsCheckpointStreamFactory(fileSystem, checkpointPrivateDirectory, checkpointSharedDirectory, fileStateSizeThreshold, writeBufferSize);
    String localFolder = "local";
    temporaryFolder.newFolder(localFolder);
    int sstFileCount = 6;
    Map<StateHandleID, Path> sstFilePaths = generateRandomSstFiles(localFolder, sstFileCount, fileStateSizeThreshold);
    try (RocksDBStateUploader rocksDBStateUploader = new RocksDBStateUploader(5)) {
        Map<StateHandleID, StreamStateHandle> sstFiles = rocksDBStateUploader.uploadFilesToCheckpointFs(sstFilePaths, checkpointStreamFactory, CheckpointedStateScope.SHARED, new CloseableRegistry());
        for (Map.Entry<StateHandleID, Path> entry : sstFilePaths.entrySet()) {
            assertStateContentEqual(entry.getValue(), sstFiles.get(entry.getKey()).openInputStream());
        }
    }
}

Example 4 with FsCheckpointStreamFactory

use of org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory in project flink by apache.

the class RocksIncrementalSnapshotStrategyTest method testCheckpointIsIncremental.

// Verify the next checkpoint is still incremental after a savepoint completed.
@Test
public void testCheckpointIsIncremental() throws Exception {
    try (CloseableRegistry closeableRegistry = new CloseableRegistry();
        RocksIncrementalSnapshotStrategy checkpointSnapshotStrategy = createSnapshotStrategy(closeableRegistry)) {
        FsCheckpointStreamFactory checkpointStreamFactory = createFsCheckpointStreamFactory();
        // make and notify checkpoint with id 1
        snapshot(1L, checkpointSnapshotStrategy, checkpointStreamFactory, closeableRegistry);
        checkpointSnapshotStrategy.notifyCheckpointComplete(1L);
        // notify savepoint with id 2
        checkpointSnapshotStrategy.notifyCheckpointComplete(2L);
        // make checkpoint with id 3
        IncrementalRemoteKeyedStateHandle incrementalRemoteKeyedStateHandle3 = snapshot(3L, checkpointSnapshotStrategy, checkpointStreamFactory, closeableRegistry);
        // If 3rd checkpoint's placeholderStateHandleCount > 0,it means 3rd checkpoint is
        // incremental.
        Map<StateHandleID, StreamStateHandle> sharedState3 = incrementalRemoteKeyedStateHandle3.getSharedState();
        long placeholderStateHandleCount = sharedState3.entrySet().stream().filter(e -> e.getValue() instanceof PlaceholderStreamStateHandle).count();
        Assert.assertTrue(placeholderStateHandleCount > 0);
    }
}

Example 5 with FsCheckpointStreamFactory

use of org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory in project flink by apache.

the class RocksIncrementalSnapshotStrategyTest method createFsCheckpointStreamFactory.

public FsCheckpointStreamFactory createFsCheckpointStreamFactory() throws IOException {
    int threshold = 100;
    File checkpointsDir = tmp.newFolder("checkpointsDir");
    File sharedStateDir = tmp.newFolder("sharedStateDir");
    FsCheckpointStreamFactory checkpointStreamFactory = new FsCheckpointStreamFactory(getSharedInstance(), fromLocalFile(checkpointsDir), fromLocalFile(sharedStateDir), threshold, threshold);
    return checkpointStreamFactory;
}