Examples with MapStateDescriptor org.apache.flink.api.common.state.MapStateDescriptor used on opensource projects

Example 21 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class OperatorStateBackendTest method testSnapshotBroadcastStateWithEmptyOperatorState.

@Test
public void testSnapshotBroadcastStateWithEmptyOperatorState() throws Exception {
    final AbstractStateBackend abstractStateBackend = new MemoryStateBackend(4096);
    OperatorStateBackend operatorStateBackend = abstractStateBackend.createOperatorStateBackend(createMockEnvironment(), "testOperator", emptyStateHandles, new CloseableRegistry());
    final MapStateDescriptor<Integer, Integer> broadcastStateDesc = new MapStateDescriptor<>("test-broadcast", BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);
    final Map<Integer, Integer> expected = new HashMap<>(3);
    expected.put(1, 2);
    expected.put(3, 4);
    expected.put(5, 6);
    final BroadcastState<Integer, Integer> broadcastState = operatorStateBackend.getBroadcastState(broadcastStateDesc);
    broadcastState.putAll(expected);
    final CheckpointStreamFactory streamFactory = new MemCheckpointStreamFactory(4096);
    OperatorStateHandle stateHandle = null;
    try {
        RunnableFuture<SnapshotResult<OperatorStateHandle>> snapshot = operatorStateBackend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
        SnapshotResult<OperatorStateHandle> snapshotResult = FutureUtils.runIfNotDoneAndGet(snapshot);
        stateHandle = snapshotResult.getJobManagerOwnedSnapshot();
        assertNotNull(stateHandle);
        final Map<Integer, Integer> retrieved = new HashMap<>();
        operatorStateBackend = recreateOperatorStateBackend(operatorStateBackend, abstractStateBackend, StateObjectCollection.singleton(stateHandle));
        BroadcastState<Integer, Integer> retrievedState = operatorStateBackend.getBroadcastState(broadcastStateDesc);
        for (Map.Entry<Integer, Integer> e : retrievedState.entries()) {
            retrieved.put(e.getKey(), e.getValue());
        }
        assertEquals(expected, retrieved);
        // remove an element from both expected and stored state.
        retrievedState.remove(1);
        expected.remove(1);
        snapshot = operatorStateBackend.snapshot(1L, 1L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
        snapshotResult = FutureUtils.runIfNotDoneAndGet(snapshot);
        stateHandle.discardState();
        stateHandle = snapshotResult.getJobManagerOwnedSnapshot();
        retrieved.clear();
        operatorStateBackend = recreateOperatorStateBackend(operatorStateBackend, abstractStateBackend, StateObjectCollection.singleton(stateHandle));
        retrievedState = operatorStateBackend.getBroadcastState(broadcastStateDesc);
        for (Map.Entry<Integer, Integer> e : retrievedState.immutableEntries()) {
            retrieved.put(e.getKey(), e.getValue());
        }
        assertEquals(expected, retrieved);
        // remove all elements from both expected and stored state.
        retrievedState.clear();
        expected.clear();
        snapshot = operatorStateBackend.snapshot(2L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());
        snapshotResult = FutureUtils.runIfNotDoneAndGet(snapshot);
        if (stateHandle != null) {
            stateHandle.discardState();
        }
        stateHandle = snapshotResult.getJobManagerOwnedSnapshot();
        retrieved.clear();
        operatorStateBackend = recreateOperatorStateBackend(operatorStateBackend, abstractStateBackend, StateObjectCollection.singleton(stateHandle));
        retrievedState = operatorStateBackend.getBroadcastState(broadcastStateDesc);
        for (Map.Entry<Integer, Integer> e : retrievedState.immutableEntries()) {
            retrieved.put(e.getKey(), e.getValue());
        }
        assertTrue(expected.isEmpty());
        assertEquals(expected, retrieved);
        if (stateHandle != null) {
            stateHandle.discardState();
            stateHandle = null;
        }
    } finally {
        operatorStateBackend.close();
        operatorStateBackend.dispose();
        if (stateHandle != null) {
            stateHandle.discardState();
        }
    }
}

Example 22 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class StatefulJobWBroadcastStateMigrationITCase method testSavepoint.

@Test
public void testSavepoint() throws Exception {
    final int parallelism = 4;
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setRestartStrategy(RestartStrategies.noRestart());
    switch(snapshotSpec.getStateBackendType()) {
        case StateBackendLoader.ROCKSDB_STATE_BACKEND_NAME:
            env.setStateBackend(new EmbeddedRocksDBStateBackend());
            break;
        case StateBackendLoader.MEMORY_STATE_BACKEND_NAME:
            env.setStateBackend(new MemoryStateBackend());
            break;
        case StateBackendLoader.HASHMAP_STATE_BACKEND_NAME:
            env.setStateBackend(new HashMapStateBackend());
            break;
        default:
            throw new UnsupportedOperationException();
    }
    env.enableChangelogStateBackend(false);
    env.enableCheckpointing(500);
    env.setParallelism(parallelism);
    env.setMaxParallelism(parallelism);
    SourceFunction<Tuple2<Long, Long>> nonParallelSource;
    SourceFunction<Tuple2<Long, Long>> nonParallelSourceB;
    SourceFunction<Tuple2<Long, Long>> parallelSource;
    SourceFunction<Tuple2<Long, Long>> parallelSourceB;
    KeyedBroadcastProcessFunction<Long, Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> firstBroadcastFunction;
    KeyedBroadcastProcessFunction<Long, Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> secondBroadcastFunction;
    final Map<Long, Long> expectedFirstState = new HashMap<>();
    expectedFirstState.put(0L, 0L);
    expectedFirstState.put(1L, 1L);
    expectedFirstState.put(2L, 2L);
    expectedFirstState.put(3L, 3L);
    final Map<String, Long> expectedSecondState = new HashMap<>();
    expectedSecondState.put("0", 0L);
    expectedSecondState.put("1", 1L);
    expectedSecondState.put("2", 2L);
    expectedSecondState.put("3", 3L);
    final Map<Long, String> expectedThirdState = new HashMap<>();
    expectedThirdState.put(0L, "0");
    expectedThirdState.put(1L, "1");
    expectedThirdState.put(2L, "2");
    expectedThirdState.put(3L, "3");
    if (executionMode == ExecutionMode.CREATE_SNAPSHOT) {
        nonParallelSource = new MigrationTestUtils.CheckpointingNonParallelSourceWithListState(NUM_SOURCE_ELEMENTS);
        nonParallelSourceB = new MigrationTestUtils.CheckpointingNonParallelSourceWithListState(NUM_SOURCE_ELEMENTS);
        parallelSource = new MigrationTestUtils.CheckpointingParallelSourceWithUnionListState(NUM_SOURCE_ELEMENTS);
        parallelSourceB = new MigrationTestUtils.CheckpointingParallelSourceWithUnionListState(NUM_SOURCE_ELEMENTS);
        firstBroadcastFunction = new CheckpointingKeyedBroadcastFunction();
        secondBroadcastFunction = new CheckpointingKeyedSingleBroadcastFunction();
    } else if (executionMode == ExecutionMode.VERIFY_SNAPSHOT) {
        nonParallelSource = new MigrationTestUtils.CheckingNonParallelSourceWithListState(NUM_SOURCE_ELEMENTS);
        nonParallelSourceB = new MigrationTestUtils.CheckingNonParallelSourceWithListState(NUM_SOURCE_ELEMENTS);
        parallelSource = new MigrationTestUtils.CheckingParallelSourceWithUnionListState(NUM_SOURCE_ELEMENTS);
        parallelSourceB = new MigrationTestUtils.CheckingParallelSourceWithUnionListState(NUM_SOURCE_ELEMENTS);
        firstBroadcastFunction = new CheckingKeyedBroadcastFunction(expectedFirstState, expectedSecondState);
        secondBroadcastFunction = new CheckingKeyedSingleBroadcastFunction(expectedThirdState);
    } else {
        throw new IllegalStateException("Unknown ExecutionMode " + executionMode);
    }
    KeyedStream<Tuple2<Long, Long>, Long> npStream = env.addSource(nonParallelSource).uid("CheckpointingSource1").keyBy(new KeySelector<Tuple2<Long, Long>, Long>() {

        private static final long serialVersionUID = -4514793867774977152L;

        @Override
        public Long getKey(Tuple2<Long, Long> value) throws Exception {
            return value.f0;
        }
    });
    KeyedStream<Tuple2<Long, Long>, Long> pStream = env.addSource(parallelSource).uid("CheckpointingSource2").keyBy(new KeySelector<Tuple2<Long, Long>, Long>() {

        private static final long serialVersionUID = 4940496713319948104L;

        @Override
        public Long getKey(Tuple2<Long, Long> value) throws Exception {
            return value.f0;
        }
    });
    final MapStateDescriptor<Long, Long> firstBroadcastStateDesc = new MapStateDescriptor<>("broadcast-state-1", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
    final MapStateDescriptor<String, Long> secondBroadcastStateDesc = new MapStateDescriptor<>("broadcast-state-2", BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
    final MapStateDescriptor<Long, String> thirdBroadcastStateDesc = new MapStateDescriptor<>("broadcast-state-3", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    BroadcastStream<Tuple2<Long, Long>> npBroadcastStream = env.addSource(nonParallelSourceB).uid("BrCheckpointingSource1").broadcast(firstBroadcastStateDesc, secondBroadcastStateDesc);
    BroadcastStream<Tuple2<Long, Long>> pBroadcastStream = env.addSource(parallelSourceB).uid("BrCheckpointingSource2").broadcast(thirdBroadcastStateDesc);
    npStream.connect(npBroadcastStream).process(firstBroadcastFunction).uid("BrProcess1").addSink(new MigrationTestUtils.AccumulatorCountingSink<>());
    pStream.connect(pBroadcastStream).process(secondBroadcastFunction).uid("BrProcess2").addSink(new MigrationTestUtils.AccumulatorCountingSink<>());
    if (executionMode == ExecutionMode.CREATE_SNAPSHOT) {
        executeAndSnapshot(env, "src/test/resources/" + getSnapshotPath(snapshotSpec), snapshotSpec.getSnapshotType(), new Tuple2<>(MigrationTestUtils.AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, 2 * NUM_SOURCE_ELEMENTS));
    } else {
        restoreAndExecute(env, getResourceFilename(getSnapshotPath(snapshotSpec)), new Tuple2<>(MigrationTestUtils.CheckingNonParallelSourceWithListState.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, // we have 2 sources
        2), new Tuple2<>(MigrationTestUtils.CheckingParallelSourceWithUnionListState.SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, // we have 2 sources
        2 * parallelism), new Tuple2<>(MigrationTestUtils.AccumulatorCountingSink.NUM_ELEMENTS_ACCUMULATOR, NUM_SOURCE_ELEMENTS * 2));
    }
}

Example 23 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class KvStateRequestSerializerTest method testMapSerialization.

/**
	 * Tests map serialization utils.
	 */
@Test
public void testMapSerialization() throws Exception {
    final long key = 0L;
    // objects for heap state list serialisation
    final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = new HeapKeyedStateBackend<>(mock(TaskKvStateRegistry.class), LongSerializer.INSTANCE, ClassLoader.getSystemClassLoader(), 1, new KeyGroupRange(0, 0), async, new ExecutionConfig());
    longHeapKeyedStateBackend.setCurrentKey(key);
    final InternalMapState<VoidNamespace, Long, String> mapState = (InternalMapState<VoidNamespace, Long, String>) longHeapKeyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));
    testMapSerialization(key, mapState);
}

Example 24 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class StateBackendTestBase method testMapStateDefaultValue.

/**
	 * Verify that an empty {@code MapState} yields {@code null}.
	 */
@Test
public void testMapStateDefaultValue() throws Exception {
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    MapStateDescriptor<String, String> kvId = new MapStateDescriptor<>("id", String.class, String.class);
    kvId.initializeSerializerUnlessSet(new ExecutionConfig());
    MapState<String, String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    backend.setCurrentKey(1);
    assertNull(state.entries());
    state.put("Ciao", "Hello");
    state.put("Bello", "Nice");
    assertNotNull(state.entries());
    assertEquals(state.get("Ciao"), "Hello");
    assertEquals(state.get("Bello"), "Nice");
    state.clear();
    assertNull(state.entries());
    backend.dispose();
}

Example 25 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class KVStateRequestSerializerRocksDBTest method testMapSerialization.

/**
	 * Tests map serialization and deserialization match.
	 *
	 * @see KvStateRequestSerializerTest#testMapSerialization()
	 * KvStateRequestSerializerTest#testMapSerialization() using the heap state back-end
	 * test
	 */
@Test
public void testMapSerialization() throws Exception {
    final long key = 0L;
    // objects for RocksDB state list serialisation
    DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
    dbOptions.setCreateIfMissing(true);
    ColumnFamilyOptions columnFamilyOptions = PredefinedOptions.DEFAULT.createColumnOptions();
    final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend = new RocksDBKeyedStateBackend<>(new JobID(), "no-op", ClassLoader.getSystemClassLoader(), temporaryFolder.getRoot(), dbOptions, columnFamilyOptions, mock(TaskKvStateRegistry.class), LongSerializer.INSTANCE, 1, new KeyGroupRange(0, 0), new ExecutionConfig());
    longHeapKeyedStateBackend.setCurrentKey(key);
    final InternalMapState<VoidNamespace, Long, String> mapState = (InternalMapState<VoidNamespace, Long, String>) longHeapKeyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));
    KvStateRequestSerializerTest.testMapSerialization(key, mapState);
}