Examples with FileStateHandle org.apache.flink.runtime.state.filesystem.FileStateHandle used on opensource projects

Example 6 with FileStateHandle

use of org.apache.flink.runtime.state.filesystem.FileStateHandle in project flink by apache.

the class SavepointV1Serializer method deserializeStreamStateHandle.

private static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException {
    int type = dis.read();
    if (NULL_HANDLE == type) {
        return null;
    } else if (FILE_STREAM_STATE_HANDLE == type) {
        long size = dis.readLong();
        String pathString = dis.readUTF();
        return new FileStateHandle(new Path(pathString), size);
    } else if (BYTE_STREAM_STATE_HANDLE == type) {
        String handleName = dis.readUTF();
        int numBytes = dis.readInt();
        byte[] data = new byte[numBytes];
        dis.readFully(data);
        return new ByteStreamStateHandle(handleName, data);
    } else {
        throw new IOException("Unknown implementation of StreamStateHandle, code: " + type);
    }
}

Example 7 with FileStateHandle

use of org.apache.flink.runtime.state.filesystem.FileStateHandle in project flink by apache.

the class SavepointV1Serializer method serializeStreamStateHandle.

private static void serializeStreamStateHandle(StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {
    if (stateHandle == null) {
        dos.writeByte(NULL_HANDLE);
    } else if (stateHandle instanceof FileStateHandle) {
        dos.writeByte(FILE_STREAM_STATE_HANDLE);
        FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
        dos.writeLong(stateHandle.getStateSize());
        dos.writeUTF(fileStateHandle.getFilePath().toString());
    } else if (stateHandle instanceof ByteStreamStateHandle) {
        dos.writeByte(BYTE_STREAM_STATE_HANDLE);
        ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
        dos.writeUTF(byteStreamStateHandle.getHandleName());
        byte[] internalData = byteStreamStateHandle.getData();
        dos.writeInt(internalData.length);
        dos.write(byteStreamStateHandle.getData());
    } else {
        throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
    }
    dos.flush();
}

Example 8 with FileStateHandle

use of org.apache.flink.runtime.state.filesystem.FileStateHandle in project flink by apache.

the class SavepointStore method storeSavepointToHandle.

/**
	 * Stores the savepoint metadata file to a state handle.
	 *
	 * @param directory Target directory to store savepoint in
	 * @param savepoint Savepoint to be stored
	 *
	 * @return State handle to the checkpoint metadata
	 * @throws IOException Failures during store are forwarded
	 */
static <T extends Savepoint> FileStateHandle storeSavepointToHandle(String directory, String filename, T savepoint) throws IOException {
    checkNotNull(directory, "Target directory");
    checkNotNull(savepoint, "Savepoint");
    final Path basePath = new Path(directory);
    final Path metadataFilePath = new Path(basePath, filename);
    final FileSystem fs = FileSystem.get(basePath.toUri());
    boolean success = false;
    try (FSDataOutputStream fdos = fs.create(metadataFilePath, WriteMode.NO_OVERWRITE);
        DataOutputStream dos = new DataOutputStream(fdos)) {
        // Write header
        dos.writeInt(MAGIC_NUMBER);
        dos.writeInt(savepoint.getVersion());
        // Write savepoint
        SavepointSerializer<T> serializer = SavepointSerializers.getSerializer(savepoint);
        serializer.serialize(savepoint, dos);
        // construct result handle
        FileStateHandle handle = new FileStateHandle(metadataFilePath, dos.size());
        // all good!
        success = true;
        return handle;
    } finally {
        if (!success && fs.exists(metadataFilePath)) {
            if (!fs.delete(metadataFilePath, true)) {
                LOG.warn("Failed to delete file {} after failed metadata write.", metadataFilePath);
            }
        }
    }
}

Example 9 with FileStateHandle

use of org.apache.flink.runtime.state.filesystem.FileStateHandle in project flink by apache.

the class CompletedCheckpointTest method testDiscard.

/**
	 * Tests that persistent checkpoints discard their header file.
	 */
@Test
public void testDiscard() throws Exception {
    File file = tmpFolder.newFile();
    assertEquals(true, file.exists());
    TaskState state = mock(TaskState.class);
    Map<JobVertexID, TaskState> taskStates = new HashMap<>();
    taskStates.put(new JobVertexID(), state);
    // Verify discard call is forwarded to state
    CompletedCheckpoint checkpoint = new CompletedCheckpoint(new JobID(), 0, 0, 1, taskStates, CheckpointProperties.forStandardCheckpoint(), new FileStateHandle(new Path(file.toURI()), file.length()), file.getAbsolutePath());
    checkpoint.discard(JobStatus.FAILED);
    assertEquals(false, file.exists());
}

Example 10 with FileStateHandle

use of org.apache.flink.runtime.state.filesystem.FileStateHandle in project flink by apache.

the class CompletedCheckpointTest method testCleanUpOnShutdown.

/**
	 * Tests that the garbage collection properties are respected when shutting down.
	 */
@Test
public void testCleanUpOnShutdown() throws Exception {
    File file = tmpFolder.newFile();
    String externalPath = file.getAbsolutePath();
    JobStatus[] terminalStates = new JobStatus[] { JobStatus.FINISHED, JobStatus.CANCELED, JobStatus.FAILED, JobStatus.SUSPENDED };
    TaskState state = mock(TaskState.class);
    Map<JobVertexID, TaskState> taskStates = new HashMap<>();
    taskStates.put(new JobVertexID(), state);
    for (JobStatus status : terminalStates) {
        Mockito.reset(state);
        // Keep
        CheckpointProperties props = new CheckpointProperties(false, true, false, false, false, false, false);
        CompletedCheckpoint checkpoint = new CompletedCheckpoint(new JobID(), 0, 0, 1, new HashMap<>(taskStates), props, new FileStateHandle(new Path(file.toURI()), file.length()), externalPath);
        checkpoint.discard(status);
        verify(state, times(0)).discardState();
        assertEquals(true, file.exists());
        // Discard
        props = new CheckpointProperties(false, false, true, true, true, true, true);
        checkpoint = new CompletedCheckpoint(new JobID(), 0, 0, 1, new HashMap<>(taskStates), props);
        checkpoint.discard(status);
        verify(state, times(1)).discardState();
    }
}