Examples with MemCheckpointStreamFactory org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory used on opensource projects

Example 11 with MemCheckpointStreamFactory

use of org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory in project flink by apache.

the class OperatorStateBackendTest method testSnapshotBroadcastStateWithEmptyOperatorState.

@Test
public void testSnapshotBroadcastStateWithEmptyOperatorState() throws Exception {
    final AbstractStateBackend abstractStateBackend = new MemoryStateBackend(4096);
    OperatorStateBackend operatorStateBackend = abstractStateBackend.createOperatorStateBackend(createMockEnvironment(), "testOperator", emptyStateHandles, new CloseableRegistry());
    final MapStateDescriptor<Integer, Integer> broadcastStateDesc = new MapStateDescriptor<>("test-broadcast", BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);
    final Map<Integer, Integer> expected = new HashMap<>(3);
    expected.put(1, 2);
    expected.put(3, 4);
    expected.put(5, 6);
    final BroadcastState<Integer, Integer> broadcastState = operatorStateBackend.getBroadcastState(broadcastStateDesc);
    broadcastState.putAll(expected);
    final CheckpointStreamFactory streamFactory = new MemCheckpointStreamFactory(4096);
    OperatorStateHandle stateHandle = null;
    try {
        RunnableFuture<SnapshotResult<OperatorStateHandle>> snapshot = operatorStateBackend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
        SnapshotResult<OperatorStateHandle> snapshotResult = FutureUtils.runIfNotDoneAndGet(snapshot);
        stateHandle = snapshotResult.getJobManagerOwnedSnapshot();
        assertNotNull(stateHandle);
        final Map<Integer, Integer> retrieved = new HashMap<>();
        operatorStateBackend = recreateOperatorStateBackend(operatorStateBackend, abstractStateBackend, StateObjectCollection.singleton(stateHandle));
        BroadcastState<Integer, Integer> retrievedState = operatorStateBackend.getBroadcastState(broadcastStateDesc);
        for (Map.Entry<Integer, Integer> e : retrievedState.entries()) {
            retrieved.put(e.getKey(), e.getValue());
        }
        assertEquals(expected, retrieved);
        // remove an element from both expected and stored state.
        retrievedState.remove(1);
        expected.remove(1);
        snapshot = operatorStateBackend.snapshot(1L, 1L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
        snapshotResult = FutureUtils.runIfNotDoneAndGet(snapshot);
        stateHandle.discardState();
        stateHandle = snapshotResult.getJobManagerOwnedSnapshot();
        retrieved.clear();
        operatorStateBackend = recreateOperatorStateBackend(operatorStateBackend, abstractStateBackend, StateObjectCollection.singleton(stateHandle));
        retrievedState = operatorStateBackend.getBroadcastState(broadcastStateDesc);
        for (Map.Entry<Integer, Integer> e : retrievedState.immutableEntries()) {
            retrieved.put(e.getKey(), e.getValue());
        }
        assertEquals(expected, retrieved);
        // remove all elements from both expected and stored state.
        retrievedState.clear();
        expected.clear();
        snapshot = operatorStateBackend.snapshot(2L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
        snapshotResult = FutureUtils.runIfNotDoneAndGet(snapshot);
        if (stateHandle != null) {
            stateHandle.discardState();
        }
        stateHandle = snapshotResult.getJobManagerOwnedSnapshot();
        retrieved.clear();
        operatorStateBackend = recreateOperatorStateBackend(operatorStateBackend, abstractStateBackend, StateObjectCollection.singleton(stateHandle));
        retrievedState = operatorStateBackend.getBroadcastState(broadcastStateDesc);
        for (Map.Entry<Integer, Integer> e : retrievedState.immutableEntries()) {
            retrieved.put(e.getKey(), e.getValue());
        }
        assertTrue(expected.isEmpty());
        assertEquals(expected, retrieved);
        if (stateHandle != null) {
            stateHandle.discardState();
            stateHandle = null;
        }
    } finally {
        operatorStateBackend.close();
        operatorStateBackend.dispose();
        if (stateHandle != null) {
            stateHandle.discardState();
        }
    }
}

Example 12 with MemCheckpointStreamFactory

use of org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory in project flink by apache.

the class OperatorStateBackendTest method testSnapshotEmpty.

@Test
public void testSnapshotEmpty() throws Exception {
    final AbstractStateBackend abstractStateBackend = new MemoryStateBackend(4096);
    CloseableRegistry cancelStreamRegistry = new CloseableRegistry();
    final OperatorStateBackend operatorStateBackend = abstractStateBackend.createOperatorStateBackend(createMockEnvironment(), "testOperator", emptyStateHandles, cancelStreamRegistry);
    CheckpointStreamFactory streamFactory = new MemCheckpointStreamFactory(4096);
    RunnableFuture<SnapshotResult<OperatorStateHandle>> snapshot = operatorStateBackend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
    SnapshotResult<OperatorStateHandle> snapshotResult = FutureUtils.runIfNotDoneAndGet(snapshot);
    OperatorStateHandle stateHandle = snapshotResult.getJobManagerOwnedSnapshot();
    assertNull(stateHandle);
}