Examples with MemCheckpointStreamFactory org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory used on opensource projects

Example 1 with MemCheckpointStreamFactory

use of org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory in project flink by apache.

the class SavepointV0Serializer method convertKeyedBackendState.

/**
	 * This is public so that we can use it when restoring a legacy snapshot
	 * in {@code AbstractStreamOperatorTestHarness}.
	 */
public static KeyGroupsStateHandle convertKeyedBackendState(HashMap<String, KvStateSnapshot<?, ?, ?, ?>> oldKeyedState, int parallelInstanceIdx, long checkpointID) throws Exception {
    if (null != oldKeyedState) {
        CheckpointStreamFactory checkpointStreamFactory = new MemCheckpointStreamFactory(MAX_SIZE);
        CheckpointStreamFactory.CheckpointStateOutputStream keyedStateOut = checkpointStreamFactory.createCheckpointStateOutputStream(checkpointID, 0L);
        try {
            final long offset = keyedStateOut.getPos();
            InstantiationUtil.serializeObject(keyedStateOut, oldKeyedState);
            StreamStateHandle streamStateHandle = keyedStateOut.closeAndGetHandle();
            // makes IOUtils.closeQuietly(...) ignore this
            keyedStateOut = null;
            if (null != streamStateHandle) {
                KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(parallelInstanceIdx, parallelInstanceIdx, new long[] { offset });
                return new MigrationKeyGroupStateHandle(keyGroupRangeOffsets, streamStateHandle);
            }
        } finally {
            IOUtils.closeQuietly(keyedStateOut);
        }
    }
    return null;
}

Example 2 with MemCheckpointStreamFactory

use of org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory in project flink by apache.

the class BatchExecutionStateBackendVerificationTest method verifySnapshotNotSupported.

@Test
public void verifySnapshotNotSupported() {
    expectedException.expect(UnsupportedOperationException.class);
    expectedException.expectMessage("Snapshotting is not supported in BATCH runtime mode.");
    BatchExecutionKeyedStateBackend<Long> stateBackend = new BatchExecutionKeyedStateBackend<>(LONG_SERIALIZER, new KeyGroupRange(0, 9));
    long checkpointId = 0L;
    CheckpointStreamFactory streamFactory = new MemCheckpointStreamFactory(10);
    stateBackend.snapshot(checkpointId, 0L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
}

Example 3 with MemCheckpointStreamFactory

use of org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory in project flink by apache.

the class HeapKeyedStateBackendSnapshotMigrationTest method testMapStateMigrationAfterHashMapSerRemoval.

@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
    ClassLoader cl = getClass().getClassLoader();
    URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");
    Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");
    final SnapshotResult<KeyedStateHandle> stateHandles;
    try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
        stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
    }
    final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
    try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
        final Integer namespace1 = 1;
        final Integer namespace2 = 2;
        final Integer namespace3 = 3;
        final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
        stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
        InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);
        keyedBackend.setCurrentKey("abc");
        state.setCurrentNamespace(namespace1);
        assertEquals(33L, (long) state.get(33L));
        assertEquals(55L, (long) state.get(55L));
        assertEquals(2, getStateSize(state));
        state.setCurrentNamespace(namespace2);
        assertEquals(22L, (long) state.get(22L));
        assertEquals(11L, (long) state.get(11L));
        assertEquals(2, getStateSize(state));
        state.setCurrentNamespace(namespace3);
        assertEquals(44L, (long) state.get(44L));
        assertEquals(1, getStateSize(state));
        keyedBackend.setCurrentKey("def");
        state.setCurrentNamespace(namespace1);
        assertEquals(11L, (long) state.get(11L));
        assertEquals(44L, (long) state.get(44L));
        assertEquals(2, getStateSize(state));
        state.setCurrentNamespace(namespace3);
        assertEquals(22L, (long) state.get(22L));
        assertEquals(55L, (long) state.get(55L));
        assertEquals(33L, (long) state.get(33L));
        assertEquals(3, getStateSize(state));
        keyedBackend.setCurrentKey("jkl");
        state.setCurrentNamespace(namespace1);
        assertEquals(11L, (long) state.get(11L));
        assertEquals(22L, (long) state.get(22L));
        assertEquals(33L, (long) state.get(33L));
        assertEquals(44L, (long) state.get(44L));
        assertEquals(55L, (long) state.get(55L));
        assertEquals(5, getStateSize(state));
        keyedBackend.setCurrentKey("mno");
        state.setCurrentNamespace(namespace3);
        assertEquals(11L, (long) state.get(11L));
        assertEquals(22L, (long) state.get(22L));
        assertEquals(33L, (long) state.get(33L));
        assertEquals(44L, (long) state.get(44L));
        assertEquals(55L, (long) state.get(55L));
        assertEquals(5, getStateSize(state));
        RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(1L, 1L, new MemCheckpointStreamFactory(4 * 1024 * 1024), CheckpointOptions.forCheckpointWithDefaultLocation());
        snapshot.run();
    }
}

Example 4 with MemCheckpointStreamFactory

use of org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory in project flink by apache.

the class StateSnapshotContextSynchronousImplTest method setUp.

@Before
public void setUp() throws Exception {
    CloseableRegistry closableRegistry = new CloseableRegistry();
    CheckpointStreamFactory streamFactory = new MemCheckpointStreamFactory(1024);
    KeyGroupRange keyGroupRange = new KeyGroupRange(0, 2);
    this.snapshotContext = new StateSnapshotContextSynchronousImpl(42, 4711, streamFactory, keyGroupRange, closableRegistry);
}

Example 5 with MemCheckpointStreamFactory

use of org.apache.flink.runtime.state.memory.MemCheckpointStreamFactory in project flink by apache.

the class StreamOperatorStateHandlerTest method testFailingBackendSnapshotMethod.

/**
 * Tests that a failing snapshot method call to the keyed state backend will trigger the closing
 * of the StateSnapshotContextSynchronousImpl and the cancellation of the
 * OperatorSnapshotResult. The latter is supposed to also cancel all assigned futures.
 */
@Test
public void testFailingBackendSnapshotMethod() throws Exception {
    final long checkpointId = 42L;
    final long timestamp = 1L;
    try (CloseableRegistry closeableRegistry = new CloseableRegistry()) {
        RunnableFuture<SnapshotResult<KeyedStateHandle>> keyedStateManagedFuture = new CancelableFuture<>();
        RunnableFuture<SnapshotResult<KeyedStateHandle>> keyedStateRawFuture = new CancelableFuture<>();
        RunnableFuture<SnapshotResult<OperatorStateHandle>> operatorStateManagedFuture = new CancelableFuture<>();
        RunnableFuture<SnapshotResult<OperatorStateHandle>> operatorStateRawFuture = new CancelableFuture<>();
        RunnableFuture<SnapshotResult<StateObjectCollection<InputChannelStateHandle>>> inputChannelStateFuture = new CancelableFuture<>();
        RunnableFuture<SnapshotResult<StateObjectCollection<ResultSubpartitionStateHandle>>> resultSubpartitionStateFuture = new CancelableFuture<>();
        OperatorSnapshotFutures operatorSnapshotResult = new OperatorSnapshotFutures(keyedStateManagedFuture, keyedStateRawFuture, operatorStateManagedFuture, operatorStateRawFuture, inputChannelStateFuture, resultSubpartitionStateFuture);
        StateSnapshotContextSynchronousImpl context = new TestStateSnapshotContextSynchronousImpl(checkpointId, timestamp, closeableRegistry);
        context.getRawKeyedOperatorStateOutput();
        context.getRawOperatorStateOutput();
        StreamTaskStateInitializerImpl stateInitializer = new StreamTaskStateInitializerImpl(new MockEnvironmentBuilder().build(), new MemoryStateBackend());
        StreamOperatorStateContext stateContext = stateInitializer.streamOperatorStateContext(new OperatorID(), "whatever", new TestProcessingTimeService(), new UnUsedKeyContext(), IntSerializer.INSTANCE, closeableRegistry, new InterceptingOperatorMetricGroup(), 1.0, false);
        StreamOperatorStateHandler stateHandler = new StreamOperatorStateHandler(stateContext, new ExecutionConfig(), closeableRegistry);
        final String keyedStateField = "keyedStateField";
        final String operatorStateField = "operatorStateField";
        CheckpointedStreamOperator checkpointedStreamOperator = new CheckpointedStreamOperator() {

            @Override
            public void initializeState(StateInitializationContext context) throws Exception {
                context.getKeyedStateStore().getState(new ValueStateDescriptor<>(keyedStateField, LongSerializer.INSTANCE)).update(42L);
                context.getOperatorStateStore().getListState(new ListStateDescriptor<>(operatorStateField, LongSerializer.INSTANCE)).add(42L);
            }

            @Override
            public void snapshotState(StateSnapshotContext context) throws Exception {
                throw new ExpectedTestException();
            }
        };
        stateHandler.setCurrentKey("44");
        stateHandler.initializeOperatorState(checkpointedStreamOperator);
        assertThat(stateContext.operatorStateBackend().getRegisteredStateNames(), is(not(empty())));
        assertThat(((AbstractKeyedStateBackend<?>) stateContext.keyedStateBackend()).numKeyValueStatesByName(), equalTo(1));
        try {
            stateHandler.snapshotState(checkpointedStreamOperator, Optional.of(stateContext.internalTimerServiceManager()), "42", 42, 42, CheckpointOptions.forCheckpointWithDefaultLocation(), new MemCheckpointStreamFactory(1024), operatorSnapshotResult, context, false);
            fail("Exception expected.");
        } catch (CheckpointException e) {
            // as CheckpointException is wrapping the cause with SerializedThrowable
            if (!ExceptionUtils.findThrowableWithMessage(e, ExpectedTestException.MESSAGE).isPresent()) {
                throw e;
            }
        }
        assertTrue(keyedStateManagedFuture.isCancelled());
        assertTrue(keyedStateRawFuture.isCancelled());
        assertTrue(context.getKeyedStateStreamFuture().isCancelled());
        assertTrue(operatorStateManagedFuture.isCancelled());
        assertTrue(operatorStateRawFuture.isCancelled());
        assertTrue(context.getOperatorStateStreamFuture().isCancelled());
        assertTrue(inputChannelStateFuture.isCancelled());
        assertTrue(resultSubpartitionStateFuture.isCancelled());
        stateHandler.dispose();
        assertThat(stateContext.operatorStateBackend().getRegisteredBroadcastStateNames(), is(empty()));
        assertThat(stateContext.operatorStateBackend().getRegisteredStateNames(), is(empty()));
        assertThat(((AbstractKeyedStateBackend<?>) stateContext.keyedStateBackend()).numKeyValueStatesByName(), equalTo(0));
    }
}