Examples with StreamStateHandle org.apache.flink.runtime.state.StreamStateHandle used on opensource projects

Example 71 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class FsCheckpointStreamFactoryTest method testExclusiveStateHasRelativePathHandles.

@Test
public void testExclusiveStateHasRelativePathHandles() throws IOException {
    final FsCheckpointStreamFactory factory = createFactory(FileSystem.getLocalFileSystem(), 0);
    final FsCheckpointStreamFactory.FsCheckpointStateOutputStream stream = factory.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE);
    stream.write(1657);
    final StreamStateHandle handle = stream.closeAndGetHandle();
    assertThat(handle, instanceOf(RelativeFileStateHandle.class));
    assertPathsEqual(exclusiveStateDir, ((RelativeFileStateHandle) handle).getFilePath().getParent());
}

Example 72 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class FsCheckpointStorageAccessTest method testTaskOwnedStateStream.

@Test
public void testTaskOwnedStateStream() throws Exception {
    final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter");
    // we chose a small size threshold here to force the state to disk
    final FsCheckpointStorageAccess storage = new FsCheckpointStorageAccess(Path.fromLocalFile(tmp.newFolder()), null, new JobID(), 10, WRITE_BUFFER_SIZE);
    final StreamStateHandle stateHandle;
    try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
        assertTrue(stream instanceof FsCheckpointStateOutputStream);
        new ObjectOutputStream(stream).writeObject(state);
        stateHandle = stream.closeAndGetHandle();
    }
    // the state must have gone to disk
    FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
    // check that the state is in the correct directory
    String parentDirName = fileStateHandle.getFilePath().getParent().getName();
    assertEquals(FsCheckpointStorageAccess.CHECKPOINT_TASK_OWNED_STATE_DIR, parentDirName);
    // validate the contents
    try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) {
        assertEquals(state, in.readObject());
    }
}

Example 73 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class FsCheckpointStorageAccessTest method testDirectoriesForExclusiveAndSharedState.

@Test
public void testDirectoriesForExclusiveAndSharedState() throws Exception {
    final FileSystem fs = LocalFileSystem.getSharedInstance();
    final Path checkpointDir = randomTempPath();
    final Path sharedStateDir = randomTempPath();
    FsCheckpointStorageLocation storageLocation = new FsCheckpointStorageLocation(fs, checkpointDir, sharedStateDir, randomTempPath(), CheckpointStorageLocationReference.getDefault(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);
    assertNotEquals(storageLocation.getCheckpointDirectory(), storageLocation.getSharedStateDirectory());
    assertEquals(0, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
    assertEquals(0, fs.listStatus(storageLocation.getSharedStateDirectory()).length);
    // create exclusive state
    FsCheckpointStateOutputStream exclusiveStream = storageLocation.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE);
    exclusiveStream.write(42);
    exclusiveStream.flushToFile();
    StreamStateHandle exclusiveHandle = exclusiveStream.closeAndGetHandle();
    assertEquals(1, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
    assertEquals(0, fs.listStatus(storageLocation.getSharedStateDirectory()).length);
    // create shared state
    FsCheckpointStateOutputStream sharedStream = storageLocation.createCheckpointStateOutputStream(CheckpointedStateScope.SHARED);
    sharedStream.write(42);
    sharedStream.flushToFile();
    StreamStateHandle sharedHandle = sharedStream.closeAndGetHandle();
    assertEquals(1, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
    assertEquals(1, fs.listStatus(storageLocation.getSharedStateDirectory()).length);
    // drop state
    exclusiveHandle.discardState();
    sharedHandle.discardState();
}

Example 74 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class RocksDBStateDownloader method createDownloadRunnables.

private List<Runnable> createDownloadRunnables(Map<StateHandleID, StreamStateHandle> stateHandleMap, Path restoreInstancePath, CloseableRegistry closeableRegistry) {
    List<Runnable> runnables = new ArrayList<>(stateHandleMap.size());
    for (Map.Entry<StateHandleID, StreamStateHandle> entry : stateHandleMap.entrySet()) {
        StateHandleID stateHandleID = entry.getKey();
        StreamStateHandle remoteFileHandle = entry.getValue();
        Path path = restoreInstancePath.resolve(stateHandleID.toString());
        runnables.add(ThrowingRunnable.unchecked(() -> downloadDataForStateHandle(path, remoteFileHandle, closeableRegistry)));
    }
    return runnables;
}

Example 75 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class RocksDBStateUploader method uploadLocalFileToCheckpointFs.

private StreamStateHandle uploadLocalFileToCheckpointFs(Path filePath, CheckpointStreamFactory checkpointStreamFactory, CheckpointedStateScope stateScope, CloseableRegistry closeableRegistry) throws IOException {
    InputStream inputStream = null;
    CheckpointStateOutputStream outputStream = null;
    try {
        final byte[] buffer = new byte[READ_BUFFER_SIZE];
        inputStream = Files.newInputStream(filePath);
        closeableRegistry.registerCloseable(inputStream);
        outputStream = checkpointStreamFactory.createCheckpointStateOutputStream(stateScope);
        closeableRegistry.registerCloseable(outputStream);
        while (true) {
            int numBytes = inputStream.read(buffer);
            if (numBytes == -1) {
                break;
            }
            outputStream.write(buffer, 0, numBytes);
        }
        StreamStateHandle result = null;
        if (closeableRegistry.unregisterCloseable(outputStream)) {
            result = outputStream.closeAndGetHandle();
            outputStream = null;
        }
        return result;
    } finally {
        if (closeableRegistry.unregisterCloseable(inputStream)) {
            IOUtils.closeQuietly(inputStream);
        }
        if (closeableRegistry.unregisterCloseable(outputStream)) {
            IOUtils.closeQuietly(outputStream);
        }
    }
}