Search in sources :

Example 6 with InternalKvState

use of org.apache.flink.runtime.state.internal.InternalKvState in project flink by apache.

the class StateBackendTestBase method testConcurrentMapIfQueryable.

/**
	 * Tests that {@link AbstractHeapState} instances respect the queryable
	 * flag and create concurrent variants for internal state structures.
	 */
@SuppressWarnings("unchecked")
protected void testConcurrentMapIfQueryable() throws Exception {
    final int numberOfKeyGroups = 1;
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, numberOfKeyGroups, new KeyGroupRange(0, 0), new DummyEnvironment("test_op", 1, 0));
    {
        // ValueState
        ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("value-state", Integer.class, -1);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.update(121818273);
        StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    {
        // ListState
        ListStateDescriptor<Integer> desc = new ListStateDescriptor<>("list-state", Integer.class);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        ListState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.add(121818273);
        StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    {
        // ReducingState
        ReducingStateDescriptor<Integer> desc = new ReducingStateDescriptor<>("reducing-state", new ReduceFunction<Integer>() {

            @Override
            public Integer reduce(Integer value1, Integer value2) throws Exception {
                return value1 + value2;
            }
        }, Integer.class);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        ReducingState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.add(121818273);
        StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    {
        // FoldingState
        FoldingStateDescriptor<Integer, Integer> desc = new FoldingStateDescriptor<>("folding-state", 0, new FoldFunction<Integer, Integer>() {

            @Override
            public Integer fold(Integer accumulator, Integer value) throws Exception {
                return accumulator + value;
            }
        }, Integer.class);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        FoldingState<Integer, Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.add(121818273);
        StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    {
        // MapState
        MapStateDescriptor<Integer, String> desc = new MapStateDescriptor<>("map-state", Integer.class, String.class);
        desc.setQueryable("my-query");
        desc.initializeSerializerUnlessSet(new ExecutionConfig());
        MapState<Integer, String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        assertTrue(kvState instanceof AbstractHeapState);
        kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
        backend.setCurrentKey(1);
        state.put(121818273, "121818273");
        int keyGroupIndex = KeyGroupRangeAssignment.assignToKeyGroup(1, numberOfKeyGroups);
        StateTable stateTable = ((AbstractHeapState) kvState).getStateTable();
        assertNotNull("State not set", stateTable.get(keyGroupIndex));
        checkConcurrentStateTable(stateTable, numberOfKeyGroups);
    }
    backend.dispose();
}
Also used : NestedMapsStateTable(org.apache.flink.runtime.state.heap.NestedMapsStateTable) StateTable(org.apache.flink.runtime.state.heap.StateTable) AbstractHeapState(org.apache.flink.runtime.state.heap.AbstractHeapState) MapStateDescriptor(org.apache.flink.api.common.state.MapStateDescriptor) FoldFunction(org.apache.flink.api.common.functions.FoldFunction) ListStateDescriptor(org.apache.flink.api.common.state.ListStateDescriptor) ReduceFunction(org.apache.flink.api.common.functions.ReduceFunction) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) FoldingStateDescriptor(org.apache.flink.api.common.state.FoldingStateDescriptor) ValueStateDescriptor(org.apache.flink.api.common.state.ValueStateDescriptor) ListState(org.apache.flink.api.common.state.ListState) ReducingStateDescriptor(org.apache.flink.api.common.state.ReducingStateDescriptor) MapState(org.apache.flink.api.common.state.MapState) DummyEnvironment(org.apache.flink.runtime.operators.testutils.DummyEnvironment) ReducingState(org.apache.flink.api.common.state.ReducingState) ValueState(org.apache.flink.api.common.state.ValueState) InternalValueState(org.apache.flink.runtime.state.internal.InternalValueState) InternalKvState(org.apache.flink.runtime.state.internal.InternalKvState) FoldingState(org.apache.flink.api.common.state.FoldingState)

Example 7 with InternalKvState

use of org.apache.flink.runtime.state.internal.InternalKvState in project flink by apache.

the class StateBackendTestBase method testValueState.

@Test
@SuppressWarnings("unchecked")
public void testValueState() throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class);
    kvId.initializeSerializerUnlessSet(new ExecutionConfig());
    TypeSerializer<Integer> keySerializer = IntSerializer.INSTANCE;
    TypeSerializer<VoidNamespace> namespaceSerializer = VoidNamespaceSerializer.INSTANCE;
    TypeSerializer<String> valueSerializer = kvId.getSerializer();
    ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    @SuppressWarnings("unchecked") InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
    // some modifications to the state
    backend.setCurrentKey(1);
    assertNull(state.value());
    assertNull(getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
    state.update("1");
    backend.setCurrentKey(2);
    assertNull(state.value());
    assertNull(getSerializedValue(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
    state.update("2");
    backend.setCurrentKey(1);
    assertEquals("1", state.value());
    assertEquals("1", getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
    // draw a snapshot
    KeyGroupsStateHandle snapshot1 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
    // make some more modifications
    backend.setCurrentKey(1);
    state.update("u1");
    backend.setCurrentKey(2);
    state.update("u2");
    backend.setCurrentKey(3);
    state.update("u3");
    // draw another snapshot
    KeyGroupsStateHandle snapshot2 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462379L, 4, streamFactory, CheckpointOptions.forFullCheckpoint()));
    // validate the original state
    backend.setCurrentKey(1);
    assertEquals("u1", state.value());
    assertEquals("u1", getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
    backend.setCurrentKey(2);
    assertEquals("u2", state.value());
    assertEquals("u2", getSerializedValue(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
    backend.setCurrentKey(3);
    assertEquals("u3", state.value());
    assertEquals("u3", getSerializedValue(kvState, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
    backend.dispose();
    backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
    snapshot1.discardState();
    ValueState<String> restored1 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    @SuppressWarnings("unchecked") InternalKvState<VoidNamespace> restoredKvState1 = (InternalKvState<VoidNamespace>) restored1;
    backend.setCurrentKey(1);
    assertEquals("1", restored1.value());
    assertEquals("1", getSerializedValue(restoredKvState1, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
    backend.setCurrentKey(2);
    assertEquals("2", restored1.value());
    assertEquals("2", getSerializedValue(restoredKvState1, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
    backend.dispose();
    backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot2);
    snapshot2.discardState();
    ValueState<String> restored2 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
    @SuppressWarnings("unchecked") InternalKvState<VoidNamespace> restoredKvState2 = (InternalKvState<VoidNamespace>) restored2;
    backend.setCurrentKey(1);
    assertEquals("u1", restored2.value());
    assertEquals("u1", getSerializedValue(restoredKvState2, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
    backend.setCurrentKey(2);
    assertEquals("u2", restored2.value());
    assertEquals("u2", getSerializedValue(restoredKvState2, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
    backend.setCurrentKey(3);
    assertEquals("u3", restored2.value());
    assertEquals("u3", getSerializedValue(restoredKvState2, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
    backend.dispose();
}
Also used : BlockerCheckpointStreamFactory(org.apache.flink.runtime.util.BlockerCheckpointStreamFactory) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) ValueStateDescriptor(org.apache.flink.api.common.state.ValueStateDescriptor) InternalKvState(org.apache.flink.runtime.state.internal.InternalKvState) Test(org.junit.Test)

Example 8 with InternalKvState

use of org.apache.flink.runtime.state.internal.InternalKvState in project flink by apache.

the class StateBackendTestBase method testMapState.

@Test
@SuppressWarnings("unchecked,rawtypes")
public void testMapState() {
    try {
        CheckpointStreamFactory streamFactory = createStreamFactory();
        AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
        MapStateDescriptor<Integer, String> kvId = new MapStateDescriptor<>("id", Integer.class, String.class);
        kvId.initializeSerializerUnlessSet(new ExecutionConfig());
        TypeSerializer<Integer> keySerializer = IntSerializer.INSTANCE;
        TypeSerializer<VoidNamespace> namespaceSerializer = VoidNamespaceSerializer.INSTANCE;
        TypeSerializer<Integer> userKeySerializer = kvId.getKeySerializer();
        TypeSerializer<String> userValueSerializer = kvId.getValueSerializer();
        MapState<Integer, String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        @SuppressWarnings("unchecked") InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        // some modifications to the state
        backend.setCurrentKey(1);
        assertEquals(null, state.get(1));
        assertEquals(null, getSerializedMap(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        state.put(1, "1");
        backend.setCurrentKey(2);
        assertEquals(null, state.get(2));
        assertEquals(null, getSerializedMap(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        state.put(2, "2");
        backend.setCurrentKey(1);
        assertTrue(state.contains(1));
        assertEquals("1", state.get(1));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(1, "1");
            }
        }, getSerializedMap(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        // draw a snapshot
        KeyGroupsStateHandle snapshot1 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
        // make some more modifications
        backend.setCurrentKey(1);
        state.put(1, "101");
        backend.setCurrentKey(2);
        state.put(102, "102");
        backend.setCurrentKey(3);
        state.put(103, "103");
        state.putAll(new HashMap<Integer, String>() {

            {
                put(1031, "1031");
                put(1032, "1032");
            }
        });
        // draw another snapshot
        KeyGroupsStateHandle snapshot2 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462379L, 4, streamFactory, CheckpointOptions.forFullCheckpoint()));
        // validate the original state
        backend.setCurrentKey(1);
        assertEquals("101", state.get(1));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(1, "101");
            }
        }, getSerializedMap(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        backend.setCurrentKey(2);
        assertEquals("102", state.get(102));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(2, "2");
                put(102, "102");
            }
        }, getSerializedMap(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        backend.setCurrentKey(3);
        assertTrue(state.contains(103));
        assertEquals("103", state.get(103));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(103, "103");
                put(1031, "1031");
                put(1032, "1032");
            }
        }, getSerializedMap(kvState, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        List<Integer> keys = new ArrayList<>();
        for (Integer key : state.keys()) {
            keys.add(key);
        }
        List<Integer> expectedKeys = new ArrayList<Integer>() {

            {
                add(103);
                add(1031);
                add(1032);
            }
        };
        assertEquals(keys.size(), expectedKeys.size());
        keys.removeAll(expectedKeys);
        assertTrue(keys.isEmpty());
        List<String> values = new ArrayList<>();
        for (String value : state.values()) {
            values.add(value);
        }
        List<String> expectedValues = new ArrayList<String>() {

            {
                add("103");
                add("1031");
                add("1032");
            }
        };
        assertEquals(values.size(), expectedValues.size());
        values.removeAll(expectedValues);
        assertTrue(values.isEmpty());
        // make some more modifications
        backend.setCurrentKey(1);
        state.clear();
        backend.setCurrentKey(2);
        state.remove(102);
        backend.setCurrentKey(3);
        final String updateSuffix = "_updated";
        Iterator<Map.Entry<Integer, String>> iterator = state.iterator();
        while (iterator.hasNext()) {
            Map.Entry<Integer, String> entry = iterator.next();
            if (entry.getValue().length() != 4) {
                iterator.remove();
            } else {
                entry.setValue(entry.getValue() + updateSuffix);
            }
        }
        // validate the state
        backend.setCurrentKey(1);
        backend.setCurrentKey(2);
        assertFalse(state.contains(102));
        backend.setCurrentKey(3);
        for (Map.Entry<Integer, String> entry : state.entries()) {
            assertEquals(4 + updateSuffix.length(), entry.getValue().length());
            assertTrue(entry.getValue().endsWith(updateSuffix));
        }
        backend.dispose();
        // restore the first snapshot and validate it
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
        snapshot1.discardState();
        MapState<Integer, String> restored1 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        @SuppressWarnings("unchecked") InternalKvState<VoidNamespace> restoredKvState1 = (InternalKvState<VoidNamespace>) restored1;
        backend.setCurrentKey(1);
        assertEquals("1", restored1.get(1));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(1, "1");
            }
        }, getSerializedMap(restoredKvState1, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        backend.setCurrentKey(2);
        assertEquals("2", restored1.get(2));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(2, "2");
            }
        }, getSerializedMap(restoredKvState1, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        backend.dispose();
        // restore the second snapshot and validate it
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot2);
        snapshot2.discardState();
        @SuppressWarnings("unchecked") MapState<Integer, String> restored2 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        @SuppressWarnings("unchecked") InternalKvState<VoidNamespace> restoredKvState2 = (InternalKvState<VoidNamespace>) restored2;
        backend.setCurrentKey(1);
        assertEquals("101", restored2.get(1));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(1, "101");
            }
        }, getSerializedMap(restoredKvState2, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        backend.setCurrentKey(2);
        assertEquals("102", restored2.get(102));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(2, "2");
                put(102, "102");
            }
        }, getSerializedMap(restoredKvState2, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        backend.setCurrentKey(3);
        assertEquals("103", restored2.get(103));
        assertEquals(new HashMap<Integer, String>() {

            {
                put(103, "103");
                put(1031, "1031");
                put(1032, "1032");
            }
        }, getSerializedMap(restoredKvState2, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
        backend.dispose();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}
Also used : MapStateDescriptor(org.apache.flink.api.common.state.MapStateDescriptor) ArrayList(java.util.ArrayList) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) BlockerCheckpointStreamFactory(org.apache.flink.runtime.util.BlockerCheckpointStreamFactory) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) InternalKvState(org.apache.flink.runtime.state.internal.InternalKvState) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) Test(org.junit.Test)

Example 9 with InternalKvState

use of org.apache.flink.runtime.state.internal.InternalKvState in project flink by apache.

the class StateBackendTestBase method testReducingState.

@Test
@SuppressWarnings("unchecked")
public void testReducingState() {
    try {
        CheckpointStreamFactory streamFactory = createStreamFactory();
        AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
        ReducingStateDescriptor<String> kvId = new ReducingStateDescriptor<>("id", new AppendingReduce(), String.class);
        kvId.initializeSerializerUnlessSet(new ExecutionConfig());
        TypeSerializer<Integer> keySerializer = IntSerializer.INSTANCE;
        TypeSerializer<VoidNamespace> namespaceSerializer = VoidNamespaceSerializer.INSTANCE;
        TypeSerializer<String> valueSerializer = kvId.getSerializer();
        ReducingState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        @SuppressWarnings("unchecked") InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
        // some modifications to the state
        backend.setCurrentKey(1);
        assertEquals(null, state.get());
        assertNull(getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        state.add("1");
        backend.setCurrentKey(2);
        assertEquals(null, state.get());
        assertNull(getSerializedValue(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        state.add("2");
        backend.setCurrentKey(1);
        assertEquals("1", state.get());
        assertEquals("1", getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        // draw a snapshot
        KeyGroupsStateHandle snapshot1 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
        // make some more modifications
        backend.setCurrentKey(1);
        state.add("u1");
        backend.setCurrentKey(2);
        state.add("u2");
        backend.setCurrentKey(3);
        state.add("u3");
        // draw another snapshot
        KeyGroupsStateHandle snapshot2 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462379L, 4, streamFactory, CheckpointOptions.forFullCheckpoint()));
        // validate the original state
        backend.setCurrentKey(1);
        assertEquals("1,u1", state.get());
        assertEquals("1,u1", getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        backend.setCurrentKey(2);
        assertEquals("2,u2", state.get());
        assertEquals("2,u2", getSerializedValue(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        backend.setCurrentKey(3);
        assertEquals("u3", state.get());
        assertEquals("u3", getSerializedValue(kvState, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        backend.dispose();
        // restore the first snapshot and validate it
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
        snapshot1.discardState();
        ReducingState<String> restored1 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        @SuppressWarnings("unchecked") InternalKvState<VoidNamespace> restoredKvState1 = (InternalKvState<VoidNamespace>) restored1;
        backend.setCurrentKey(1);
        assertEquals("1", restored1.get());
        assertEquals("1", getSerializedValue(restoredKvState1, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        backend.setCurrentKey(2);
        assertEquals("2", restored1.get());
        assertEquals("2", getSerializedValue(restoredKvState1, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        backend.dispose();
        // restore the second snapshot and validate it
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot2);
        snapshot2.discardState();
        ReducingState<String> restored2 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        @SuppressWarnings("unchecked") InternalKvState<VoidNamespace> restoredKvState2 = (InternalKvState<VoidNamespace>) restored2;
        backend.setCurrentKey(1);
        assertEquals("1,u1", restored2.get());
        assertEquals("1,u1", getSerializedValue(restoredKvState2, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        backend.setCurrentKey(2);
        assertEquals("2,u2", restored2.get());
        assertEquals("2,u2", getSerializedValue(restoredKvState2, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        backend.setCurrentKey(3);
        assertEquals("u3", restored2.get());
        assertEquals("u3", getSerializedValue(restoredKvState2, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
        backend.dispose();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}
Also used : ReducingStateDescriptor(org.apache.flink.api.common.state.ReducingStateDescriptor) BlockerCheckpointStreamFactory(org.apache.flink.runtime.util.BlockerCheckpointStreamFactory) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) InternalKvState(org.apache.flink.runtime.state.internal.InternalKvState) Test(org.junit.Test)

Aggregations

InternalKvState (org.apache.flink.runtime.state.internal.InternalKvState)9 ExecutionConfig (org.apache.flink.api.common.ExecutionConfig)7 Test (org.junit.Test)7 IOException (java.io.IOException)5 ExecutionException (java.util.concurrent.ExecutionException)5 BlockerCheckpointStreamFactory (org.apache.flink.runtime.util.BlockerCheckpointStreamFactory)5 ValueStateDescriptor (org.apache.flink.api.common.state.ValueStateDescriptor)4 ArrayList (java.util.ArrayList)2 FoldingState (org.apache.flink.api.common.state.FoldingState)2 FoldingStateDescriptor (org.apache.flink.api.common.state.FoldingStateDescriptor)2 ListState (org.apache.flink.api.common.state.ListState)2 ListStateDescriptor (org.apache.flink.api.common.state.ListStateDescriptor)2 MapState (org.apache.flink.api.common.state.MapState)2 MapStateDescriptor (org.apache.flink.api.common.state.MapStateDescriptor)2 ReducingState (org.apache.flink.api.common.state.ReducingState)2 ReducingStateDescriptor (org.apache.flink.api.common.state.ReducingStateDescriptor)2 ValueState (org.apache.flink.api.common.state.ValueState)2 DummyEnvironment (org.apache.flink.runtime.operators.testutils.DummyEnvironment)2 InternalValueState (org.apache.flink.runtime.state.internal.InternalValueState)2 Joiner (com.google.common.base.Joiner)1