Examples with MemCheckpointStreamFactory org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory used on opensource projects

Example 6 with MemCheckpointStreamFactory

use of org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory in project flink by apache.

the class BackendRestorerProcedureTest method testRestoreProcedureOrderAndFailure.

/**
 * Tests that the restore procedure follows the order of the iterator and will retries failed
 * attempts if there are more options.
 */
@Test
public void testRestoreProcedureOrderAndFailure() throws Exception {
    CloseableRegistry closeableRegistry = new CloseableRegistry();
    CheckpointStreamFactory checkpointStreamFactory = new MemCheckpointStreamFactory(1024);
    ListStateDescriptor<Integer> stateDescriptor = new ListStateDescriptor<>("test-state", Integer.class);
    OperatorStateBackend originalBackend = backendSupplier.apply(Collections.emptyList());
    SnapshotResult<OperatorStateHandle> snapshotResult;
    try {
        ListState<Integer> listState = originalBackend.getListState(stateDescriptor);
        listState.add(0);
        listState.add(1);
        listState.add(2);
        listState.add(3);
        RunnableFuture<SnapshotResult<OperatorStateHandle>> snapshot = originalBackend.snapshot(0L, 0L, checkpointStreamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
        snapshot.run();
        snapshotResult = snapshot.get();
    } finally {
        originalBackend.close();
        originalBackend.dispose();
    }
    OperatorStateHandle firstFailHandle = mock(OperatorStateHandle.class);
    OperatorStateHandle secondSuccessHandle = spy(snapshotResult.getJobManagerOwnedSnapshot());
    OperatorStateHandle thirdNotUsedHandle = mock(OperatorStateHandle.class);
    List<StateObjectCollection<OperatorStateHandle>> sortedRestoreOptions = Arrays.asList(new StateObjectCollection<>(Collections.singletonList(firstFailHandle)), new StateObjectCollection<>(Collections.singletonList(secondSuccessHandle)), new StateObjectCollection<>(Collections.singletonList(thirdNotUsedHandle)));
    BackendRestorerProcedure<OperatorStateBackend, OperatorStateHandle> restorerProcedure = new BackendRestorerProcedure<>(backendSupplier, closeableRegistry, "test op state backend");
    OperatorStateBackend restoredBackend = restorerProcedure.createAndRestore(sortedRestoreOptions);
    Assert.assertNotNull(restoredBackend);
    try {
        verify(firstFailHandle).openInputStream();
        verify(secondSuccessHandle).openInputStream();
        verifyZeroInteractions(thirdNotUsedHandle);
        ListState<Integer> listState = restoredBackend.getListState(stateDescriptor);
        Iterator<Integer> stateIterator = listState.get().iterator();
        Assert.assertEquals(0, (int) stateIterator.next());
        Assert.assertEquals(1, (int) stateIterator.next());
        Assert.assertEquals(2, (int) stateIterator.next());
        Assert.assertEquals(3, (int) stateIterator.next());
        Assert.assertFalse(stateIterator.hasNext());
    } finally {
        restoredBackend.close();
        restoredBackend.dispose();
    }
}

Example 7 with MemCheckpointStreamFactory

use of org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory in project flink by apache.

the class SavepointStateBackendSwitchTestBase method takeSavepoint.

private void takeSavepoint(CheckpointableKeyedStateBackend<String> keyedBackend, File pathToWrite, MapStateDescriptor<Long, Long> stateDescr, ValueStateDescriptor<Long> valueStateDescriptor, ListStateDescriptor<Long> listStateDescriptor, Integer namespace1, Integer namespace2, Integer namespace3, Integer namespace4) throws Exception {
    InternalMapState<String, Integer, Long, Long> mapState = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);
    InternalValueState<String, Integer, Long> valueState = keyedBackend.createInternalState(IntSerializer.INSTANCE, valueStateDescriptor);
    InternalListState<String, Integer, Long> listState = keyedBackend.createInternalState(IntSerializer.INSTANCE, listStateDescriptor);
    keyedBackend.setCurrentKey("abc");
    mapState.setCurrentNamespace(namespace1);
    mapState.put(33L, 33L);
    mapState.put(55L, 55L);
    mapState.setCurrentNamespace(namespace2);
    mapState.put(22L, 22L);
    mapState.put(11L, 11L);
    listState.setCurrentNamespace(namespace2);
    listState.add(4L);
    listState.add(5L);
    listState.add(6L);
    mapState.setCurrentNamespace(namespace3);
    mapState.put(44L, 44L);
    keyedBackend.setCurrentKey("mno");
    mapState.setCurrentNamespace(namespace3);
    mapState.put(11L, 11L);
    mapState.put(22L, 22L);
    mapState.put(33L, 33L);
    mapState.put(44L, 44L);
    mapState.put(55L, 55L);
    valueState.setCurrentNamespace(namespace3);
    valueState.update(1239L);
    listState.setCurrentNamespace(namespace3);
    listState.add(1L);
    listState.add(2L);
    listState.add(3L);
    mapState.setCurrentNamespace(namespace4);
    mapState.put(1L, 1L);
    // HEAP state backend will keep an empty map as an entry in the underlying State Table
    // we should skip such entries when serializing
    Iterator<Map.Entry<Long, Long>> iterator = mapState.iterator();
    while (iterator.hasNext()) {
        iterator.next();
        iterator.remove();
    }
    KeyGroupedInternalPriorityQueue<TimerHeapInternalTimer<String, Integer>> priorityQueue = keyedBackend.create("event-time", new TimerSerializer<>(keyedBackend.getKeySerializer(), IntSerializer.INSTANCE));
    priorityQueue.add(new TimerHeapInternalTimer<>(1234L, "mno", namespace3));
    priorityQueue.add(new TimerHeapInternalTimer<>(2345L, "mno", namespace2));
    priorityQueue.add(new TimerHeapInternalTimer<>(3456L, "mno", namespace3));
    SnapshotStrategyRunner<KeyedStateHandle, ? extends FullSnapshotResources<?>> savepointRunner = StreamOperatorStateHandler.prepareCanonicalSavepoint(keyedBackend, new CloseableRegistry());
    RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = savepointRunner.snapshot(0L, 0L, new MemCheckpointStreamFactory(4 * 1024 * 1024), new CheckpointOptions(SavepointType.savepoint(SavepointFormatType.CANONICAL), CheckpointStorageLocationReference.getDefault()));
    snapshot.run();
    try (BufferedOutputStream bis = new BufferedOutputStream(new FileOutputStream(pathToWrite))) {
        InstantiationUtil.serializeObject(bis, snapshot.get());
    }
}

Example 8 with MemCheckpointStreamFactory

use of org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory in project flink by apache.

the class StateSnapshotCompressionTest method snapshotRestoreRoundtrip.

private void snapshotRestoreRoundtrip(boolean useCompression) throws Exception {
    ExecutionConfig executionConfig = new ExecutionConfig();
    executionConfig.setUseSnapshotCompression(useCompression);
    KeyedStateHandle stateHandle;
    ValueStateDescriptor<String> stateDescriptor = new ValueStateDescriptor<>("test", String.class);
    stateDescriptor.initializeSerializerUnlessSet(executionConfig);
    AbstractKeyedStateBackend<String> stateBackend = getStringHeapKeyedStateBackend(executionConfig);
    try {
        InternalValueState<String, VoidNamespace, String> state = stateBackend.createInternalState(new VoidNamespaceSerializer(), stateDescriptor);
        stateBackend.setCurrentKey("A");
        state.setCurrentNamespace(VoidNamespace.INSTANCE);
        state.update("42");
        stateBackend.setCurrentKey("B");
        state.setCurrentNamespace(VoidNamespace.INSTANCE);
        state.update("43");
        stateBackend.setCurrentKey("C");
        state.setCurrentNamespace(VoidNamespace.INSTANCE);
        state.update("44");
        stateBackend.setCurrentKey("D");
        state.setCurrentNamespace(VoidNamespace.INSTANCE);
        state.update("45");
        CheckpointStreamFactory streamFactory = new MemCheckpointStreamFactory(4 * 1024 * 1024);
        RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = stateBackend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
        snapshot.run();
        SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();
        stateHandle = snapshotResult.getJobManagerOwnedSnapshot();
    } finally {
        IOUtils.closeQuietly(stateBackend);
        stateBackend.dispose();
    }
    executionConfig = new ExecutionConfig();
    stateBackend = getStringHeapKeyedStateBackend(executionConfig, StateObjectCollection.singleton(stateHandle));
    try {
        InternalValueState<String, VoidNamespace, String> state = stateBackend.createInternalState(new VoidNamespaceSerializer(), stateDescriptor);
        stateBackend.setCurrentKey("A");
        state.setCurrentNamespace(VoidNamespace.INSTANCE);
        Assert.assertEquals("42", state.value());
        stateBackend.setCurrentKey("B");
        state.setCurrentNamespace(VoidNamespace.INSTANCE);
        Assert.assertEquals("43", state.value());
        stateBackend.setCurrentKey("C");
        state.setCurrentNamespace(VoidNamespace.INSTANCE);
        Assert.assertEquals("44", state.value());
        stateBackend.setCurrentKey("D");
        state.setCurrentNamespace(VoidNamespace.INSTANCE);
        Assert.assertEquals("45", state.value());
    } finally {
        IOUtils.closeQuietly(stateBackend);
        stateBackend.dispose();
    }
}

Example 9 with MemCheckpointStreamFactory

use of org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory in project flink by apache.

the class OperatorStateBackendTest method testCorrectClassLoaderUsedOnSnapshot.

@SuppressWarnings("unchecked")
@Test
public void testCorrectClassLoaderUsedOnSnapshot() throws Exception {
    AbstractStateBackend abstractStateBackend = new MemoryStateBackend(4096);
    final Environment env = createMockEnvironment();
    CloseableRegistry cancelStreamRegistry = new CloseableRegistry();
    OperatorStateBackend operatorStateBackend = abstractStateBackend.createOperatorStateBackend(env, "test-op-name", emptyStateHandles, cancelStreamRegistry);
    AtomicInteger copyCounter = new AtomicInteger(0);
    TypeSerializer<Integer> serializer = new VerifyingIntSerializer(env.getUserCodeClassLoader().asClassLoader(), copyCounter);
    // write some state
    ListStateDescriptor<Integer> stateDescriptor = new ListStateDescriptor<>("test", serializer);
    ListState<Integer> listState = operatorStateBackend.getListState(stateDescriptor);
    listState.add(42);
    AtomicInteger keyCopyCounter = new AtomicInteger(0);
    AtomicInteger valueCopyCounter = new AtomicInteger(0);
    TypeSerializer<Integer> keySerializer = new VerifyingIntSerializer(env.getUserCodeClassLoader().asClassLoader(), keyCopyCounter);
    TypeSerializer<Integer> valueSerializer = new VerifyingIntSerializer(env.getUserCodeClassLoader().asClassLoader(), valueCopyCounter);
    MapStateDescriptor<Integer, Integer> broadcastStateDesc = new MapStateDescriptor<>("test-broadcast", keySerializer, valueSerializer);
    BroadcastState<Integer, Integer> broadcastState = operatorStateBackend.getBroadcastState(broadcastStateDesc);
    broadcastState.put(1, 2);
    broadcastState.put(3, 4);
    broadcastState.put(5, 6);
    CheckpointStreamFactory streamFactory = new MemCheckpointStreamFactory(4096);
    RunnableFuture<SnapshotResult<OperatorStateHandle>> runnableFuture = operatorStateBackend.snapshot(1, 1, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
    FutureUtils.runIfNotDoneAndGet(runnableFuture);
    // make sure that the copy method has been called
    assertTrue(copyCounter.get() > 0);
    assertTrue(keyCopyCounter.get() > 0);
    assertTrue(valueCopyCounter.get() > 0);
}

Example 10 with MemCheckpointStreamFactory

use of org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory in project flink by apache.

the class OperatorStateBackendTest method testSnapshotRestoreSync.

@Test
public void testSnapshotRestoreSync() throws Exception {
    AbstractStateBackend abstractStateBackend = new MemoryStateBackend(2 * 4096);
    OperatorStateBackend operatorStateBackend = abstractStateBackend.createOperatorStateBackend(createMockEnvironment(), "test-op-name", emptyStateHandles, new CloseableRegistry());
    ListStateDescriptor<Serializable> stateDescriptor1 = new ListStateDescriptor<>("test1", new JavaSerializer<>());
    ListStateDescriptor<Serializable> stateDescriptor2 = new ListStateDescriptor<>("test2", new JavaSerializer<>());
    ListStateDescriptor<Serializable> stateDescriptor3 = new ListStateDescriptor<>("test3", new JavaSerializer<>());
    MapStateDescriptor<Serializable, Serializable> broadcastStateDescriptor1 = new MapStateDescriptor<>("test4", new JavaSerializer<>(), new JavaSerializer<>());
    MapStateDescriptor<Serializable, Serializable> broadcastStateDescriptor2 = new MapStateDescriptor<>("test5", new JavaSerializer<>(), new JavaSerializer<>());
    MapStateDescriptor<Serializable, Serializable> broadcastStateDescriptor3 = new MapStateDescriptor<>("test6", new JavaSerializer<>(), new JavaSerializer<>());
    ListState<Serializable> listState1 = operatorStateBackend.getListState(stateDescriptor1);
    ListState<Serializable> listState2 = operatorStateBackend.getListState(stateDescriptor2);
    ListState<Serializable> listState3 = operatorStateBackend.getUnionListState(stateDescriptor3);
    BroadcastState<Serializable, Serializable> broadcastState1 = operatorStateBackend.getBroadcastState(broadcastStateDescriptor1);
    BroadcastState<Serializable, Serializable> broadcastState2 = operatorStateBackend.getBroadcastState(broadcastStateDescriptor2);
    BroadcastState<Serializable, Serializable> broadcastState3 = operatorStateBackend.getBroadcastState(broadcastStateDescriptor3);
    listState1.add(42);
    listState1.add(4711);
    listState2.add(7);
    listState2.add(13);
    listState2.add(23);
    listState3.add(17);
    listState3.add(18);
    listState3.add(19);
    listState3.add(20);
    broadcastState1.put(1, 2);
    broadcastState1.put(2, 5);
    broadcastState2.put(2, 5);
    CheckpointStreamFactory streamFactory = new MemCheckpointStreamFactory(2 * 4096);
    RunnableFuture<SnapshotResult<OperatorStateHandle>> snapshot = operatorStateBackend.snapshot(1L, 1L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
    SnapshotResult<OperatorStateHandle> snapshotResult = FutureUtils.runIfNotDoneAndGet(snapshot);
    OperatorStateHandle stateHandle = snapshotResult.getJobManagerOwnedSnapshot();
    try {
        operatorStateBackend.close();
        operatorStateBackend.dispose();
        operatorStateBackend = abstractStateBackend.createOperatorStateBackend(createMockEnvironment(), "testOperator", StateObjectCollection.singleton(stateHandle), new CloseableRegistry());
        assertEquals(3, operatorStateBackend.getRegisteredStateNames().size());
        assertEquals(3, operatorStateBackend.getRegisteredBroadcastStateNames().size());
        listState1 = operatorStateBackend.getListState(stateDescriptor1);
        listState2 = operatorStateBackend.getListState(stateDescriptor2);
        listState3 = operatorStateBackend.getUnionListState(stateDescriptor3);
        broadcastState1 = operatorStateBackend.getBroadcastState(broadcastStateDescriptor1);
        broadcastState2 = operatorStateBackend.getBroadcastState(broadcastStateDescriptor2);
        broadcastState3 = operatorStateBackend.getBroadcastState(broadcastStateDescriptor3);
        assertEquals(3, operatorStateBackend.getRegisteredStateNames().size());
        assertEquals(3, operatorStateBackend.getRegisteredBroadcastStateNames().size());
        Iterator<Serializable> it = listState1.get().iterator();
        assertEquals(42, it.next());
        assertEquals(4711, it.next());
        assertFalse(it.hasNext());
        it = listState2.get().iterator();
        assertEquals(7, it.next());
        assertEquals(13, it.next());
        assertEquals(23, it.next());
        assertFalse(it.hasNext());
        it = listState3.get().iterator();
        assertEquals(17, it.next());
        assertEquals(18, it.next());
        assertEquals(19, it.next());
        assertEquals(20, it.next());
        assertFalse(it.hasNext());
        Iterator<Map.Entry<Serializable, Serializable>> bIt = broadcastState1.iterator();
        assertTrue(bIt.hasNext());
        Map.Entry<Serializable, Serializable> entry = bIt.next();
        assertEquals(1, entry.getKey());
        assertEquals(2, entry.getValue());
        assertTrue(bIt.hasNext());
        entry = bIt.next();
        assertEquals(2, entry.getKey());
        assertEquals(5, entry.getValue());
        assertFalse(bIt.hasNext());
        bIt = broadcastState2.iterator();
        assertTrue(bIt.hasNext());
        entry = bIt.next();
        assertEquals(2, entry.getKey());
        assertEquals(5, entry.getValue());
        assertFalse(bIt.hasNext());
        bIt = broadcastState3.iterator();
        assertFalse(bIt.hasNext());
        operatorStateBackend.close();
        operatorStateBackend.dispose();
    } finally {
        stateHandle.discardState();
    }
}