use of org.apache.flink.runtime.state.internal.InternalKvState in project flink by apache.
the class StateBackendTestBase method testConcurrentMapIfQueryable.
/**
* Tests that {@link AbstractHeapState} instances respect the queryable
* flag and create concurrent variants for internal state structures.
*/
@SuppressWarnings("unchecked")
protected void testConcurrentMapIfQueryable() throws Exception {
final int numberOfKeyGroups = 1;
AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, numberOfKeyGroups, new KeyGroupRange(0, 0), new DummyEnvironment("test_op", 1, 0));
{
// ValueState
ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("value-state", Integer.class, -1);
desc.setQueryable("my-query");
desc.initializeSerializerUnlessSet(new ExecutionConfig());
ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
assertTrue(kvState instanceof AbstractHeapState);
kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
backend.setCurrentKey(1);
state.update(121818273);
StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
checkConcurrentStateTable(stateTable, numberOfKeyGroups);
}
{
// ListState
ListStateDescriptor<Integer> desc = new ListStateDescriptor<>("list-state", Integer.class);
desc.setQueryable("my-query");
desc.initializeSerializerUnlessSet(new ExecutionConfig());
ListState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
assertTrue(kvState instanceof AbstractHeapState);
kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
backend.setCurrentKey(1);
state.add(121818273);
StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
checkConcurrentStateTable(stateTable, numberOfKeyGroups);
}
{
// ReducingState
ReducingStateDescriptor<Integer> desc = new ReducingStateDescriptor<>("reducing-state", new ReduceFunction<Integer>() {
@Override
public Integer reduce(Integer value1, Integer value2) throws Exception {
return value1 + value2;
}
}, Integer.class);
desc.setQueryable("my-query");
desc.initializeSerializerUnlessSet(new ExecutionConfig());
ReducingState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
assertTrue(kvState instanceof AbstractHeapState);
kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
backend.setCurrentKey(1);
state.add(121818273);
StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
checkConcurrentStateTable(stateTable, numberOfKeyGroups);
}
{
// FoldingState
FoldingStateDescriptor<Integer, Integer> desc = new FoldingStateDescriptor<>("folding-state", 0, new FoldFunction<Integer, Integer>() {
@Override
public Integer fold(Integer accumulator, Integer value) throws Exception {
return accumulator + value;
}
}, Integer.class);
desc.setQueryable("my-query");
desc.initializeSerializerUnlessSet(new ExecutionConfig());
FoldingState<Integer, Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
assertTrue(kvState instanceof AbstractHeapState);
kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
backend.setCurrentKey(1);
state.add(121818273);
StateTable<?, ?, ?> stateTable = ((AbstractHeapState<?, ?, ?, ?, ?>) kvState).getStateTable();
checkConcurrentStateTable(stateTable, numberOfKeyGroups);
}
{
// MapState
MapStateDescriptor<Integer, String> desc = new MapStateDescriptor<>("map-state", Integer.class, String.class);
desc.setQueryable("my-query");
desc.initializeSerializerUnlessSet(new ExecutionConfig());
MapState<Integer, String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
assertTrue(kvState instanceof AbstractHeapState);
kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
backend.setCurrentKey(1);
state.put(121818273, "121818273");
int keyGroupIndex = KeyGroupRangeAssignment.assignToKeyGroup(1, numberOfKeyGroups);
StateTable stateTable = ((AbstractHeapState) kvState).getStateTable();
assertNotNull("State not set", stateTable.get(keyGroupIndex));
checkConcurrentStateTable(stateTable, numberOfKeyGroups);
}
backend.dispose();
}
use of org.apache.flink.runtime.state.internal.InternalKvState in project flink by apache.
the class StateBackendTestBase method testValueState.
@Test
@SuppressWarnings("unchecked")
public void testValueState() throws Exception {
CheckpointStreamFactory streamFactory = createStreamFactory();
AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
ValueStateDescriptor<String> kvId = new ValueStateDescriptor<>("id", String.class);
kvId.initializeSerializerUnlessSet(new ExecutionConfig());
TypeSerializer<Integer> keySerializer = IntSerializer.INSTANCE;
TypeSerializer<VoidNamespace> namespaceSerializer = VoidNamespaceSerializer.INSTANCE;
TypeSerializer<String> valueSerializer = kvId.getSerializer();
ValueState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
@SuppressWarnings("unchecked") InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
// some modifications to the state
backend.setCurrentKey(1);
assertNull(state.value());
assertNull(getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
state.update("1");
backend.setCurrentKey(2);
assertNull(state.value());
assertNull(getSerializedValue(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
state.update("2");
backend.setCurrentKey(1);
assertEquals("1", state.value());
assertEquals("1", getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
// draw a snapshot
KeyGroupsStateHandle snapshot1 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
// make some more modifications
backend.setCurrentKey(1);
state.update("u1");
backend.setCurrentKey(2);
state.update("u2");
backend.setCurrentKey(3);
state.update("u3");
// draw another snapshot
KeyGroupsStateHandle snapshot2 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462379L, 4, streamFactory, CheckpointOptions.forFullCheckpoint()));
// validate the original state
backend.setCurrentKey(1);
assertEquals("u1", state.value());
assertEquals("u1", getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(2);
assertEquals("u2", state.value());
assertEquals("u2", getSerializedValue(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(3);
assertEquals("u3", state.value());
assertEquals("u3", getSerializedValue(kvState, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.dispose();
backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
snapshot1.discardState();
ValueState<String> restored1 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
@SuppressWarnings("unchecked") InternalKvState<VoidNamespace> restoredKvState1 = (InternalKvState<VoidNamespace>) restored1;
backend.setCurrentKey(1);
assertEquals("1", restored1.value());
assertEquals("1", getSerializedValue(restoredKvState1, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(2);
assertEquals("2", restored1.value());
assertEquals("2", getSerializedValue(restoredKvState1, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.dispose();
backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot2);
snapshot2.discardState();
ValueState<String> restored2 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
@SuppressWarnings("unchecked") InternalKvState<VoidNamespace> restoredKvState2 = (InternalKvState<VoidNamespace>) restored2;
backend.setCurrentKey(1);
assertEquals("u1", restored2.value());
assertEquals("u1", getSerializedValue(restoredKvState2, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(2);
assertEquals("u2", restored2.value());
assertEquals("u2", getSerializedValue(restoredKvState2, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(3);
assertEquals("u3", restored2.value());
assertEquals("u3", getSerializedValue(restoredKvState2, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.dispose();
}
use of org.apache.flink.runtime.state.internal.InternalKvState in project flink by apache.
the class StateBackendTestBase method testMapState.
@Test
@SuppressWarnings("unchecked,rawtypes")
public void testMapState() {
try {
CheckpointStreamFactory streamFactory = createStreamFactory();
AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
MapStateDescriptor<Integer, String> kvId = new MapStateDescriptor<>("id", Integer.class, String.class);
kvId.initializeSerializerUnlessSet(new ExecutionConfig());
TypeSerializer<Integer> keySerializer = IntSerializer.INSTANCE;
TypeSerializer<VoidNamespace> namespaceSerializer = VoidNamespaceSerializer.INSTANCE;
TypeSerializer<Integer> userKeySerializer = kvId.getKeySerializer();
TypeSerializer<String> userValueSerializer = kvId.getValueSerializer();
MapState<Integer, String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
@SuppressWarnings("unchecked") InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
// some modifications to the state
backend.setCurrentKey(1);
assertEquals(null, state.get(1));
assertEquals(null, getSerializedMap(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
state.put(1, "1");
backend.setCurrentKey(2);
assertEquals(null, state.get(2));
assertEquals(null, getSerializedMap(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
state.put(2, "2");
backend.setCurrentKey(1);
assertTrue(state.contains(1));
assertEquals("1", state.get(1));
assertEquals(new HashMap<Integer, String>() {
{
put(1, "1");
}
}, getSerializedMap(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
// draw a snapshot
KeyGroupsStateHandle snapshot1 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
// make some more modifications
backend.setCurrentKey(1);
state.put(1, "101");
backend.setCurrentKey(2);
state.put(102, "102");
backend.setCurrentKey(3);
state.put(103, "103");
state.putAll(new HashMap<Integer, String>() {
{
put(1031, "1031");
put(1032, "1032");
}
});
// draw another snapshot
KeyGroupsStateHandle snapshot2 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462379L, 4, streamFactory, CheckpointOptions.forFullCheckpoint()));
// validate the original state
backend.setCurrentKey(1);
assertEquals("101", state.get(1));
assertEquals(new HashMap<Integer, String>() {
{
put(1, "101");
}
}, getSerializedMap(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
backend.setCurrentKey(2);
assertEquals("102", state.get(102));
assertEquals(new HashMap<Integer, String>() {
{
put(2, "2");
put(102, "102");
}
}, getSerializedMap(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
backend.setCurrentKey(3);
assertTrue(state.contains(103));
assertEquals("103", state.get(103));
assertEquals(new HashMap<Integer, String>() {
{
put(103, "103");
put(1031, "1031");
put(1032, "1032");
}
}, getSerializedMap(kvState, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
List<Integer> keys = new ArrayList<>();
for (Integer key : state.keys()) {
keys.add(key);
}
List<Integer> expectedKeys = new ArrayList<Integer>() {
{
add(103);
add(1031);
add(1032);
}
};
assertEquals(keys.size(), expectedKeys.size());
keys.removeAll(expectedKeys);
assertTrue(keys.isEmpty());
List<String> values = new ArrayList<>();
for (String value : state.values()) {
values.add(value);
}
List<String> expectedValues = new ArrayList<String>() {
{
add("103");
add("1031");
add("1032");
}
};
assertEquals(values.size(), expectedValues.size());
values.removeAll(expectedValues);
assertTrue(values.isEmpty());
// make some more modifications
backend.setCurrentKey(1);
state.clear();
backend.setCurrentKey(2);
state.remove(102);
backend.setCurrentKey(3);
final String updateSuffix = "_updated";
Iterator<Map.Entry<Integer, String>> iterator = state.iterator();
while (iterator.hasNext()) {
Map.Entry<Integer, String> entry = iterator.next();
if (entry.getValue().length() != 4) {
iterator.remove();
} else {
entry.setValue(entry.getValue() + updateSuffix);
}
}
// validate the state
backend.setCurrentKey(1);
backend.setCurrentKey(2);
assertFalse(state.contains(102));
backend.setCurrentKey(3);
for (Map.Entry<Integer, String> entry : state.entries()) {
assertEquals(4 + updateSuffix.length(), entry.getValue().length());
assertTrue(entry.getValue().endsWith(updateSuffix));
}
backend.dispose();
// restore the first snapshot and validate it
backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
snapshot1.discardState();
MapState<Integer, String> restored1 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
@SuppressWarnings("unchecked") InternalKvState<VoidNamespace> restoredKvState1 = (InternalKvState<VoidNamespace>) restored1;
backend.setCurrentKey(1);
assertEquals("1", restored1.get(1));
assertEquals(new HashMap<Integer, String>() {
{
put(1, "1");
}
}, getSerializedMap(restoredKvState1, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
backend.setCurrentKey(2);
assertEquals("2", restored1.get(2));
assertEquals(new HashMap<Integer, String>() {
{
put(2, "2");
}
}, getSerializedMap(restoredKvState1, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
backend.dispose();
// restore the second snapshot and validate it
backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot2);
snapshot2.discardState();
@SuppressWarnings("unchecked") MapState<Integer, String> restored2 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
@SuppressWarnings("unchecked") InternalKvState<VoidNamespace> restoredKvState2 = (InternalKvState<VoidNamespace>) restored2;
backend.setCurrentKey(1);
assertEquals("101", restored2.get(1));
assertEquals(new HashMap<Integer, String>() {
{
put(1, "101");
}
}, getSerializedMap(restoredKvState2, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
backend.setCurrentKey(2);
assertEquals("102", restored2.get(102));
assertEquals(new HashMap<Integer, String>() {
{
put(2, "2");
put(102, "102");
}
}, getSerializedMap(restoredKvState2, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
backend.setCurrentKey(3);
assertEquals("103", restored2.get(103));
assertEquals(new HashMap<Integer, String>() {
{
put(103, "103");
put(1031, "1031");
put(1032, "1032");
}
}, getSerializedMap(restoredKvState2, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, userKeySerializer, userValueSerializer));
backend.dispose();
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.runtime.state.internal.InternalKvState in project flink by apache.
the class StateBackendTestBase method testReducingState.
@Test
@SuppressWarnings("unchecked")
public void testReducingState() {
try {
CheckpointStreamFactory streamFactory = createStreamFactory();
AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
ReducingStateDescriptor<String> kvId = new ReducingStateDescriptor<>("id", new AppendingReduce(), String.class);
kvId.initializeSerializerUnlessSet(new ExecutionConfig());
TypeSerializer<Integer> keySerializer = IntSerializer.INSTANCE;
TypeSerializer<VoidNamespace> namespaceSerializer = VoidNamespaceSerializer.INSTANCE;
TypeSerializer<String> valueSerializer = kvId.getSerializer();
ReducingState<String> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
@SuppressWarnings("unchecked") InternalKvState<VoidNamespace> kvState = (InternalKvState<VoidNamespace>) state;
// some modifications to the state
backend.setCurrentKey(1);
assertEquals(null, state.get());
assertNull(getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
state.add("1");
backend.setCurrentKey(2);
assertEquals(null, state.get());
assertNull(getSerializedValue(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
state.add("2");
backend.setCurrentKey(1);
assertEquals("1", state.get());
assertEquals("1", getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
// draw a snapshot
KeyGroupsStateHandle snapshot1 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forFullCheckpoint()));
// make some more modifications
backend.setCurrentKey(1);
state.add("u1");
backend.setCurrentKey(2);
state.add("u2");
backend.setCurrentKey(3);
state.add("u3");
// draw another snapshot
KeyGroupsStateHandle snapshot2 = FutureUtil.runIfNotDoneAndGet(backend.snapshot(682375462379L, 4, streamFactory, CheckpointOptions.forFullCheckpoint()));
// validate the original state
backend.setCurrentKey(1);
assertEquals("1,u1", state.get());
assertEquals("1,u1", getSerializedValue(kvState, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(2);
assertEquals("2,u2", state.get());
assertEquals("2,u2", getSerializedValue(kvState, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(3);
assertEquals("u3", state.get());
assertEquals("u3", getSerializedValue(kvState, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.dispose();
// restore the first snapshot and validate it
backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot1);
snapshot1.discardState();
ReducingState<String> restored1 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
@SuppressWarnings("unchecked") InternalKvState<VoidNamespace> restoredKvState1 = (InternalKvState<VoidNamespace>) restored1;
backend.setCurrentKey(1);
assertEquals("1", restored1.get());
assertEquals("1", getSerializedValue(restoredKvState1, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(2);
assertEquals("2", restored1.get());
assertEquals("2", getSerializedValue(restoredKvState1, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.dispose();
// restore the second snapshot and validate it
backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot2);
snapshot2.discardState();
ReducingState<String> restored2 = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
@SuppressWarnings("unchecked") InternalKvState<VoidNamespace> restoredKvState2 = (InternalKvState<VoidNamespace>) restored2;
backend.setCurrentKey(1);
assertEquals("1,u1", restored2.get());
assertEquals("1,u1", getSerializedValue(restoredKvState2, 1, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(2);
assertEquals("2,u2", restored2.get());
assertEquals("2,u2", getSerializedValue(restoredKvState2, 2, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.setCurrentKey(3);
assertEquals("u3", restored2.get());
assertEquals("u3", getSerializedValue(restoredKvState2, 3, keySerializer, VoidNamespace.INSTANCE, namespaceSerializer, valueSerializer));
backend.dispose();
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
Aggregations