Examples with MapStateDescriptor org.apache.flink.api.common.state.MapStateDescriptor used on opensource projects

Example 31 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class SavepointReader method readBroadcastState.

/**
 * Read operator {@code BroadcastState} from a {@code Savepoint} when a custom serializer was
 * used; e.g., a different serializer than the one returned by {@code
 * TypeInformation#createSerializer}.
 *
 * @param uid The uid of the operator.
 * @param name The (unique) name for the state.
 * @param keyTypeInfo The type information for the keys in the state.
 * @param valueTypeInfo The type information for the values in the state.
 * @param keySerializer The type serializer used to write keys into the state.
 * @param valueSerializer The type serializer used to write values into the state.
 * @param <K> The type of keys in state.
 * @param <V> The type of values in state.
 * @return A {@code DataStream} of key-value pairs from state.
 * @throws IOException If the savepoint path is invalid or the uid does not exist.
 */
public <K, V> DataStream<Tuple2<K, V>> readBroadcastState(String uid, String name, TypeInformation<K> keyTypeInfo, TypeInformation<V> valueTypeInfo, TypeSerializer<K> keySerializer, TypeSerializer<V> valueSerializer) throws IOException {
    OperatorState operatorState = metadata.getOperatorState(uid);
    MapStateDescriptor<K, V> descriptor = new MapStateDescriptor<>(name, keySerializer, valueSerializer);
    BroadcastStateInputFormat<K, V> inputFormat = new BroadcastStateInputFormat<>(operatorState, MutableConfig.of(env.getConfiguration()), stateBackend, descriptor);
    return SourceBuilder.fromFormat(env, inputFormat, new TupleTypeInfo<>(keyTypeInfo, valueTypeInfo));
}

Example 32 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class DataStreamTest method testFailedTranslationOnNonKeyed.

/**
 * Tests that with a non-keyed stream we have to provide a {@link BroadcastProcessFunction}.
 */
@Test
public void testFailedTranslationOnNonKeyed() {
    final MapStateDescriptor<Long, String> descriptor = new MapStateDescriptor<>("broadcast", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final DataStream<Long> srcOne = env.generateSequence(0L, 5L).assignTimestampsAndWatermarks(new CustomWmEmitter<Long>() {

        @Override
        public long extractTimestamp(Long element, long previousElementTimestamp) {
            return element;
        }
    });
    final DataStream<String> srcTwo = env.fromElements("Test:0", "Test:1", "Test:2", "Test:3", "Test:4", "Test:5").assignTimestampsAndWatermarks(new CustomWmEmitter<String>() {

        @Override
        public long extractTimestamp(String element, long previousElementTimestamp) {
            return Long.parseLong(element.split(":")[1]);
        }
    });
    BroadcastStream<String> broadcast = srcTwo.broadcast(descriptor);
    BroadcastConnectedStream<Long, String> bcStream = srcOne.connect(broadcast);
    expectedException.expect(IllegalArgumentException.class);
    bcStream.process(new KeyedBroadcastProcessFunction<String, Long, String, String>() {

        @Override
        public void processBroadcastElement(String value, Context ctx, Collector<String> out) throws Exception {
        // do nothing
        }

        @Override
        public void processElement(Long value, ReadOnlyContext ctx, Collector<String> out) throws Exception {
        // do nothing
        }
    });
}

Example 33 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class DataStreamTest method testFailedTranslationOnKeyed.

/**
 * Tests that with a {@link KeyedStream} we have to provide a {@link
 * KeyedBroadcastProcessFunction}.
 */
@Test
public void testFailedTranslationOnKeyed() {
    final MapStateDescriptor<Long, String> descriptor = new MapStateDescriptor<>("broadcast", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final DataStream<Long> srcOne = env.generateSequence(0L, 5L).assignTimestampsAndWatermarks(new CustomWmEmitter<Long>() {

        @Override
        public long extractTimestamp(Long element, long previousElementTimestamp) {
            return element;
        }
    }).keyBy((KeySelector<Long, Long>) value -> value);
    final DataStream<String> srcTwo = env.fromElements("Test:0", "Test:1", "Test:2", "Test:3", "Test:4", "Test:5").assignTimestampsAndWatermarks(new CustomWmEmitter<String>() {

        @Override
        public long extractTimestamp(String element, long previousElementTimestamp) {
            return Long.parseLong(element.split(":")[1]);
        }
    });
    BroadcastStream<String> broadcast = srcTwo.broadcast(descriptor);
    BroadcastConnectedStream<Long, String> bcStream = srcOne.connect(broadcast);
    expectedException.expect(IllegalArgumentException.class);
    bcStream.process(new BroadcastProcessFunction<Long, String, String>() {

        @Override
        public void processBroadcastElement(String value, Context ctx, Collector<String> out) throws Exception {
        // do nothing
        }

        @Override
        public void processElement(Long value, ReadOnlyContext ctx, Collector<String> out) throws Exception {
        // do nothing
        }
    });
}

Example 34 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class StateBackendTestBase method testMapState.

@Test
@SuppressWarnings("unchecked,rawtypes")
public void testMapState() throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    MapStateDescriptor<Integer, String> kvId = new MapStateDescriptor<>("id", Integer.class, String.class);
    TypeSerializer<String> keySerializer = StringSerializer.INSTANCE;
    TypeSerializer<VoidNamespace> namespaceSerializer = VoidNamespaceSerializer.INSTANCE;
    CheckpointableKeyedStateBackend<String> backend = createKeyedBackend(StringSerializer.INSTANCE);
    try {
        MapState<Integer, String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        @SuppressWarnings("unchecked") InternalKvState<String, VoidNamespace, Map<Integer, String>> kvState = (InternalKvState<String, VoidNamespace, Map<Integer, String>>) state;
        // these are only available after the backend initialized the serializer
        TypeSerializer<Integer> userKeySerializer = kvId.getKeySerializer();
        TypeSerializer<String> userValueSerializer = kvId.getValueSerializer();
        // some modifications to the state
        backend.setCurrentKey("1");
        assertNull(state.get(1));
        assertNull(getSerializedMap(kvState, "1", keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        state.put(1, "1");
        backend.setCurrentKey("2");
        assertNull(state.get(2));
        assertNull(getSerializedMap(kvState, "2", keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        state.put(2, "2");
        // put entry with different userKeyOffset
        backend.setCurrentKey("11");
        state.put(11, "11");
        backend.setCurrentKey("1");
        assertTrue(state.contains(1));
        assertEquals("1", state.get(1));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(1, "1");
            }
        }, getSerializedMap(kvState, "1", keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(11, "11");
            }
        }, getSerializedMap(kvState, "11", keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        // draw a snapshot
        KeyedStateHandle snapshot1 = runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        // make some more modifications
        backend.setCurrentKey("1");
        state.put(1, "101");
        backend.setCurrentKey("2");
        state.put(102, "102");
        backend.setCurrentKey("3");
        state.put(103, "103");
        state.putAll(new HashMap<Integer, String>() {

            {
                put(1031, "1031");
                put(1032, "1032");
            }
        });
        // draw another snapshot
        KeyedStateHandle snapshot2 = runSnapshot(backend.snapshot(682375462379L, 4, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        // validate the original state
        backend.setCurrentKey("1");
        assertEquals("101", state.get(1));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(1, "101");
            }
        }, getSerializedMap(kvState, "1", keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        backend.setCurrentKey("2");
        assertEquals("102", state.get(102));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(2, "2");
                put(102, "102");
            }
        }, getSerializedMap(kvState, "2", keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        backend.setCurrentKey("3");
        assertTrue(state.contains(103));
        assertEquals("103", state.get(103));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(103, "103");
                put(1031, "1031");
                put(1032, "1032");
            }
        }, getSerializedMap(kvState, "3", keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        List<Integer> keys = new ArrayList<>();
        for (Integer key : state.keys()) {
            keys.add(key);
        }
        List<Integer> expectedKeys = Arrays.asList(103, 1031, 1032);
        assertEquals(keys.size(), expectedKeys.size());
        keys.removeAll(expectedKeys);
        List<String> values = new ArrayList<>();
        for (String value : state.values()) {
            values.add(value);
        }
        List<String> expectedValues = Arrays.asList("103", "1031", "1032");
        assertEquals(values.size(), expectedValues.size());
        values.removeAll(expectedValues);
        // make some more modifications
        backend.setCurrentKey("1");
        state.clear();
        backend.setCurrentKey("2");
        state.remove(102);
        backend.setCurrentKey("3");
        final String updateSuffix = "_updated";
        Iterator<Map.Entry<Integer, String>> iterator = state.iterator();
        while (iterator.hasNext()) {
            Map.Entry<Integer, String> entry = iterator.next();
            if (entry.getValue().length() != 4) {
                iterator.remove();
            } else {
                entry.setValue(entry.getValue() + updateSuffix);
            }
        }
        // validate the state
        backend.setCurrentKey("1");
        backend.setCurrentKey("2");
        assertFalse(state.contains(102));
        backend.setCurrentKey("3");
        for (Map.Entry<Integer, String> entry : state.entries()) {
            assertEquals(4 + updateSuffix.length(), entry.getValue().length());
            assertTrue(entry.getValue().endsWith(updateSuffix));
        }
        backend.dispose();
        // restore the first snapshot and validate it
        backend = restoreKeyedBackend(StringSerializer.INSTANCE, snapshot1);
        snapshot1.discardState();
        MapState<Integer, String> restored1 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        @SuppressWarnings("unchecked") InternalKvState<String, VoidNamespace, Map<Integer, String>> restoredKvState1 = (InternalKvState<String, VoidNamespace, Map<Integer, String>>) restored1;
        backend.setCurrentKey("1");
        assertEquals("1", restored1.get(1));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(1, "1");
            }
        }, getSerializedMap(restoredKvState1, "1", keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        backend.setCurrentKey("2");
        assertEquals("2", restored1.get(2));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(2, "2");
            }
        }, getSerializedMap(restoredKvState1, "2", keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        backend.dispose();
        // restore the second snapshot and validate it
        backend = restoreKeyedBackend(StringSerializer.INSTANCE, snapshot2);
        snapshot2.discardState();
        @SuppressWarnings("unchecked") MapState<Integer, String> restored2 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        @SuppressWarnings("unchecked") InternalKvState<String, VoidNamespace, Map<Integer, String>> restoredKvState2 = (InternalKvState<String, VoidNamespace, Map<Integer, String>>) restored2;
        backend.setCurrentKey("1");
        assertEquals("101", restored2.get(1));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(1, "101");
            }
        }, getSerializedMap(restoredKvState2, "1", keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        backend.setCurrentKey("2");
        assertEquals("102", restored2.get(102));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(2, "2");
                put(102, "102");
            }
        }, getSerializedMap(restoredKvState2, "2", keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        backend.setCurrentKey("3");
        assertEquals("103", restored2.get(103));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(103, "103");
                put(1031, "1031");
                put(1032, "1032");
            }
        }, getSerializedMap(restoredKvState2, "3", keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
    } finally {
        IOUtils.closeQuietly(backend);
        backend.dispose();
    }
}

Example 35 with MapStateDescriptor

use of org.apache.flink.api.common.state.MapStateDescriptor in project flink by apache.

the class StateBackendTestBase method testSnapshotNonAccessedState.

@Test
public void testSnapshotNonAccessedState() throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    final String stateName = "test-name";
    CheckpointableKeyedStateBackend<String> backend = createKeyedBackend(StringSerializer.INSTANCE);
    try {
        MapStateDescriptor<Integer, String> kvId = new MapStateDescriptor<>(stateName, Integer.class, String.class);
        MapState<Integer, String> mapState = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        // write some state to be snapshotted
        backend.setCurrentKey("1");
        mapState.put(11, "foo");
        backend.setCurrentKey("2");
        mapState.put(8, "bar");
        backend.setCurrentKey("3");
        mapState.put(91, "hello world");
        // take a snapshot, and then restore backend with snapshot
        KeyedStateHandle snapshot = runSnapshot(backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        backend.dispose();
        backend = restoreKeyedBackend(StringSerializer.INSTANCE, snapshot);
        // now take a snapshot again without accessing the state
        snapshot = runSnapshot(backend.snapshot(2L, 3L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        backend.dispose();
        // we restore again and try to access previous state
        backend = restoreKeyedBackend(StringSerializer.INSTANCE, snapshot);
        mapState = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        backend.setCurrentKey("1");
        assertEquals("foo", mapState.get(11));
        backend.setCurrentKey("2");
        assertEquals("bar", mapState.get(8));
        backend.setCurrentKey("3");
        assertEquals("hello world", mapState.get(91));
        snapshot.discardState();
    } finally {
        IOUtils.closeQuietly(backend);
        backend.dispose();
    }
}