Examples with StateIncrementalVisitor org.apache.flink.runtime.state.internal.InternalKvState.StateIncrementalVisitor used on opensource projects

Example 1 with StateIncrementalVisitor

use of org.apache.flink.runtime.state.internal.InternalKvState.StateIncrementalVisitor in project flink by apache.

the class CopyOnWriteStateMapTest method testRandomModificationsAndCopyOnWriteIsolation.

/**
 * This test does some random modifications to a state map and a reference (hash map). Then
 * draws snapshots, performs more modifications and checks snapshot integrity.
 */
@Test
public void testRandomModificationsAndCopyOnWriteIsolation() throws Exception {
    final CopyOnWriteStateMap<Integer, Integer, ArrayList<Integer>> stateMap = new CopyOnWriteStateMap<>(new ArrayListSerializer<>(IntSerializer.INSTANCE));
    final HashMap<Tuple2<Integer, Integer>, ArrayList<Integer>> referenceMap = new HashMap<>();
    final Random random = new Random(42);
    // holds snapshots from the map under test
    CopyOnWriteStateMap.StateMapEntry<Integer, Integer, ArrayList<Integer>>[] snapshot = null;
    int snapshotSize = 0;
    // holds a reference snapshot from our reference map that we compare against
    Tuple3<Integer, Integer, ArrayList<Integer>>[] reference = null;
    int val = 0;
    int snapshotCounter = 0;
    int referencedSnapshotId = 0;
    final StateTransformationFunction<ArrayList<Integer>, Integer> transformationFunction = (previousState, value) -> {
        if (previousState == null) {
            previousState = new ArrayList<>();
        }
        previousState.add(value);
        // we give back the original, attempting to spot errors in to copy-on-write
        return previousState;
    };
    StateIncrementalVisitor<Integer, Integer, ArrayList<Integer>> updatingIterator = stateMap.getStateIncrementalVisitor(5);
    // the main loop for modifications
    for (int i = 0; i < 10_000_000; ++i) {
        int key = random.nextInt(20);
        int namespace = random.nextInt(4);
        Tuple2<Integer, Integer> compositeKey = new Tuple2<>(key, namespace);
        int op = random.nextInt(10);
        ArrayList<Integer> state = null;
        ArrayList<Integer> referenceState = null;
        switch(op) {
            case 0:
            case 1:
                {
                    state = stateMap.get(key, namespace);
                    referenceState = referenceMap.get(compositeKey);
                    if (null == state) {
                        state = new ArrayList<>();
                        stateMap.put(key, namespace, state);
                        referenceState = new ArrayList<>();
                        referenceMap.put(compositeKey, referenceState);
                    }
                    break;
                }
            case 2:
                {
                    stateMap.put(key, namespace, new ArrayList<>());
                    referenceMap.put(compositeKey, new ArrayList<>());
                    break;
                }
            case 3:
                {
                    state = stateMap.putAndGetOld(key, namespace, new ArrayList<>());
                    referenceState = referenceMap.put(compositeKey, new ArrayList<>());
                    break;
                }
            case 4:
                {
                    stateMap.remove(key, namespace);
                    referenceMap.remove(compositeKey);
                    break;
                }
            case 5:
                {
                    state = stateMap.removeAndGetOld(key, namespace);
                    referenceState = referenceMap.remove(compositeKey);
                    break;
                }
            case 6:
                {
                    final int updateValue = random.nextInt(1000);
                    stateMap.transform(key, namespace, updateValue, transformationFunction);
                    referenceMap.put(compositeKey, transformationFunction.apply(referenceMap.remove(compositeKey), updateValue));
                    break;
                }
            case 7:
            case 8:
            case 9:
                if (!updatingIterator.hasNext()) {
                    updatingIterator = stateMap.getStateIncrementalVisitor(5);
                    if (!updatingIterator.hasNext()) {
                        break;
                    }
                }
                testStateIteratorWithUpdate(updatingIterator, stateMap, referenceMap, op == 8, op == 9);
                break;
            default:
                {
                    Assert.fail("Unknown op-code " + op);
                }
        }
        Assert.assertEquals(referenceMap.size(), stateMap.size());
        if (state != null) {
            Assert.assertNotNull(referenceState);
            // mutate the states a bit...
            if (random.nextBoolean() && !state.isEmpty()) {
                state.remove(state.size() - 1);
                referenceState.remove(referenceState.size() - 1);
            } else {
                state.add(val);
                referenceState.add(val);
                ++val;
            }
        }
        Assert.assertEquals(referenceState, state);
        // snapshot triggering / comparison / release
        if (i > 0 && i % 500 == 0) {
            if (snapshot != null) {
                // check our referenced snapshot
                deepCheck(reference, convert(snapshot, snapshotSize));
                if (i % 1_000 == 0) {
                    // draw and release some other snapshot while holding on the old snapshot
                    ++snapshotCounter;
                    stateMap.snapshotMapArrays();
                    stateMap.releaseSnapshot(snapshotCounter);
                }
                // release the snapshot after some time
                if (i % 5_000 == 0) {
                    snapshot = null;
                    reference = null;
                    snapshotSize = 0;
                    stateMap.releaseSnapshot(referencedSnapshotId);
                }
            } else {
                // if there is no more referenced snapshot, we create one
                ++snapshotCounter;
                referencedSnapshotId = snapshotCounter;
                snapshot = stateMap.snapshotMapArrays();
                snapshotSize = stateMap.size();
                reference = manualDeepDump(referenceMap);
            }
        }
    }
}