Examples with StreamStateHandle org.apache.flink.runtime.state.StreamStateHandle used on opensource projects

Example 66 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class RoundRobinOperatorStateRepartitioner method repartitionUnionState.

/**
 * Repartition UNION state.
 */
private void repartitionUnionState(Map<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> unionState, List<Map<StreamStateHandle, OperatorStateHandle>> mergeMapList) {
    for (Map<StreamStateHandle, OperatorStateHandle> mergeMap : mergeMapList) {
        for (Map.Entry<String, List<Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo>>> e : unionState.entrySet()) {
            for (Tuple2<StreamStateHandle, OperatorStateHandle.StateMetaInfo> handleWithMetaInfo : e.getValue()) {
                OperatorStateHandle operatorStateHandle = mergeMap.get(handleWithMetaInfo.f0);
                if (operatorStateHandle == null) {
                    operatorStateHandle = new OperatorStreamStateHandle(new HashMap<>(unionState.size()), handleWithMetaInfo.f0);
                    mergeMap.put(handleWithMetaInfo.f0, operatorStateHandle);
                }
                operatorStateHandle.getStateNameToPartitionOffsets().put(e.getKey(), handleWithMetaInfo.f1);
            }
        }
    }
}

Example 67 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class ChannelStateCheckpointWriter method finishWriteAndResult.

private void finishWriteAndResult() throws IOException {
    if (inputChannelOffsets.isEmpty() && resultSubpartitionOffsets.isEmpty()) {
        dataStream.close();
        result.inputChannelStateHandles.complete(emptyList());
        result.resultSubpartitionStateHandles.complete(emptyList());
        return;
    }
    dataStream.flush();
    StreamStateHandle underlying = checkpointStream.closeAndGetHandle();
    complete(underlying, result.inputChannelStateHandles, inputChannelOffsets, HandleFactory.INPUT_CHANNEL);
    complete(underlying, result.resultSubpartitionStateHandles, resultSubpartitionOffsets, HandleFactory.RESULT_SUBPARTITION);
}

Example 68 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class MemoryCheckpointOutputStreamTest method testStateStream.

@Test
public void testStateStream() throws Exception {
    HashMap<String, Integer> state = new HashMap<>();
    state.put("hey there", 2);
    state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);
    CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(MemoryStateBackend.DEFAULT_MAX_STATE_SIZE);
    ObjectOutputStream oos = new ObjectOutputStream(outStream);
    oos.writeObject(state);
    oos.flush();
    StreamStateHandle handle = outStream.closeAndGetHandle();
    assertNotNull(handle);
    try (ObjectInputStream ois = new ObjectInputStream(handle.openInputStream())) {
        assertEquals(state, ois.readObject());
        assertTrue(ois.available() <= 0);
    }
}

Example 69 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class MemoryCheckpointStorageAccessTest method testTaskOwnedStateStream.

@Test
public void testTaskOwnedStateStream() throws Exception {
    final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter");
    final MemoryBackendCheckpointStorageAccess storage = new MemoryBackendCheckpointStorageAccess(new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);
    StreamStateHandle stateHandle;
    try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
        assertTrue(stream instanceof MemoryCheckpointOutputStream);
        new ObjectOutputStream(stream).writeObject(state);
        stateHandle = stream.closeAndGetHandle();
    }
    try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) {
        assertEquals(state, in.readObject());
    }
}

Example 70 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class MemoryCheckpointStorageAccessTest method testNonPersistentCheckpointLocation.

@Test
public void testNonPersistentCheckpointLocation() throws Exception {
    MemoryBackendCheckpointStorageAccess storage = new MemoryBackendCheckpointStorageAccess(new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);
    CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);
    CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
    stream.write(99);
    CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
    StreamStateHandle handle = completed.getMetadataHandle();
    assertTrue(handle instanceof ByteStreamStateHandle);
    // the reference is not valid in that case
    try {
        storage.resolveCheckpoint(completed.getExternalPointer());
        fail("should fail with an exception");
    } catch (Exception e) {
    // expected
    }
}