Search in sources :

Example 1 with StateIncrementalVisitor

use of org.apache.flink.runtime.state.internal.InternalKvState.StateIncrementalVisitor in project flink by apache.

the class CopyOnWriteStateMapTest method testRandomModificationsAndCopyOnWriteIsolation.

/**
 * This test does some random modifications to a state map and a reference (hash map). Then
 * draws snapshots, performs more modifications and checks snapshot integrity.
 */
@Test
public void testRandomModificationsAndCopyOnWriteIsolation() throws Exception {
    final CopyOnWriteStateMap<Integer, Integer, ArrayList<Integer>> stateMap = new CopyOnWriteStateMap<>(new ArrayListSerializer<>(IntSerializer.INSTANCE));
    final HashMap<Tuple2<Integer, Integer>, ArrayList<Integer>> referenceMap = new HashMap<>();
    final Random random = new Random(42);
    // holds snapshots from the map under test
    CopyOnWriteStateMap.StateMapEntry<Integer, Integer, ArrayList<Integer>>[] snapshot = null;
    int snapshotSize = 0;
    // holds a reference snapshot from our reference map that we compare against
    Tuple3<Integer, Integer, ArrayList<Integer>>[] reference = null;
    int val = 0;
    int snapshotCounter = 0;
    int referencedSnapshotId = 0;
    final StateTransformationFunction<ArrayList<Integer>, Integer> transformationFunction = (previousState, value) -> {
        if (previousState == null) {
            previousState = new ArrayList<>();
        }
        previousState.add(value);
        // we give back the original, attempting to spot errors in to copy-on-write
        return previousState;
    };
    StateIncrementalVisitor<Integer, Integer, ArrayList<Integer>> updatingIterator = stateMap.getStateIncrementalVisitor(5);
    // the main loop for modifications
    for (int i = 0; i < 10_000_000; ++i) {
        int key = random.nextInt(20);
        int namespace = random.nextInt(4);
        Tuple2<Integer, Integer> compositeKey = new Tuple2<>(key, namespace);
        int op = random.nextInt(10);
        ArrayList<Integer> state = null;
        ArrayList<Integer> referenceState = null;
        switch(op) {
            case 0:
            case 1:
                {
                    state = stateMap.get(key, namespace);
                    referenceState = referenceMap.get(compositeKey);
                    if (null == state) {
                        state = new ArrayList<>();
                        stateMap.put(key, namespace, state);
                        referenceState = new ArrayList<>();
                        referenceMap.put(compositeKey, referenceState);
                    }
                    break;
                }
            case 2:
                {
                    stateMap.put(key, namespace, new ArrayList<>());
                    referenceMap.put(compositeKey, new ArrayList<>());
                    break;
                }
            case 3:
                {
                    state = stateMap.putAndGetOld(key, namespace, new ArrayList<>());
                    referenceState = referenceMap.put(compositeKey, new ArrayList<>());
                    break;
                }
            case 4:
                {
                    stateMap.remove(key, namespace);
                    referenceMap.remove(compositeKey);
                    break;
                }
            case 5:
                {
                    state = stateMap.removeAndGetOld(key, namespace);
                    referenceState = referenceMap.remove(compositeKey);
                    break;
                }
            case 6:
                {
                    final int updateValue = random.nextInt(1000);
                    stateMap.transform(key, namespace, updateValue, transformationFunction);
                    referenceMap.put(compositeKey, transformationFunction.apply(referenceMap.remove(compositeKey), updateValue));
                    break;
                }
            case 7:
            case 8:
            case 9:
                if (!updatingIterator.hasNext()) {
                    updatingIterator = stateMap.getStateIncrementalVisitor(5);
                    if (!updatingIterator.hasNext()) {
                        break;
                    }
                }
                testStateIteratorWithUpdate(updatingIterator, stateMap, referenceMap, op == 8, op == 9);
                break;
            default:
                {
                    Assert.fail("Unknown op-code " + op);
                }
        }
        Assert.assertEquals(referenceMap.size(), stateMap.size());
        if (state != null) {
            Assert.assertNotNull(referenceState);
            // mutate the states a bit...
            if (random.nextBoolean() && !state.isEmpty()) {
                state.remove(state.size() - 1);
                referenceState.remove(referenceState.size() - 1);
            } else {
                state.add(val);
                referenceState.add(val);
                ++val;
            }
        }
        Assert.assertEquals(referenceState, state);
        // snapshot triggering / comparison / release
        if (i > 0 && i % 500 == 0) {
            if (snapshot != null) {
                // check our referenced snapshot
                deepCheck(reference, convert(snapshot, snapshotSize));
                if (i % 1_000 == 0) {
                    // draw and release some other snapshot while holding on the old snapshot
                    ++snapshotCounter;
                    stateMap.snapshotMapArrays();
                    stateMap.releaseSnapshot(snapshotCounter);
                }
                // release the snapshot after some time
                if (i % 5_000 == 0) {
                    snapshot = null;
                    reference = null;
                    snapshotSize = 0;
                    stateMap.releaseSnapshot(referencedSnapshotId);
                }
            } else {
                // if there is no more referenced snapshot, we create one
                ++snapshotCounter;
                referencedSnapshotId = snapshotCounter;
                snapshot = stateMap.snapshotMapArrays();
                snapshotSize = stateMap.size();
                reference = manualDeepDump(referenceMap);
            }
        }
    }
}
Also used : Arrays(java.util.Arrays) StateIncrementalVisitor(org.apache.flink.runtime.state.internal.InternalKvState.StateIncrementalVisitor) Tuple3(org.apache.flink.api.java.tuple.Tuple3) Tuple2(org.apache.flink.api.java.tuple.Tuple2) HashMap(java.util.HashMap) Random(java.util.Random) ArrayList(java.util.ArrayList) Assert.assertThat(org.junit.Assert.assertThat) IntSerializer(org.apache.flink.api.common.typeutils.base.IntSerializer) StateTransformationFunction(org.apache.flink.runtime.state.StateTransformationFunction) Map(java.util.Map) StateSnapshotTransformer(org.apache.flink.runtime.state.StateSnapshotTransformer) TestLogger(org.apache.flink.util.TestLogger) LongSerializer(org.apache.flink.api.common.typeutils.base.LongSerializer) StateEntryMatcher.entry(org.apache.flink.runtime.state.testutils.StateEntryMatcher.entry) Nullable(javax.annotation.Nullable) Iterator(java.util.Iterator) Matchers(org.hamcrest.Matchers) Test(org.junit.Test) ArrayListSerializer(org.apache.flink.runtime.state.ArrayListSerializer) List(java.util.List) Matchers.contains(org.hamcrest.Matchers.contains) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) StateEntry(org.apache.flink.runtime.state.StateEntry) Assert(org.junit.Assert) Comparator(java.util.Comparator) ListSerializer(org.apache.flink.api.common.typeutils.base.ListSerializer) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Random(java.util.Random) Tuple2(org.apache.flink.api.java.tuple.Tuple2) Tuple3(org.apache.flink.api.java.tuple.Tuple3) Test(org.junit.Test)

Aggregations

ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Comparator (java.util.Comparator)1 HashMap (java.util.HashMap)1 Iterator (java.util.Iterator)1 List (java.util.List)1 Map (java.util.Map)1 Random (java.util.Random)1 Nullable (javax.annotation.Nullable)1 IntSerializer (org.apache.flink.api.common.typeutils.base.IntSerializer)1 ListSerializer (org.apache.flink.api.common.typeutils.base.ListSerializer)1 LongSerializer (org.apache.flink.api.common.typeutils.base.LongSerializer)1 Tuple2 (org.apache.flink.api.java.tuple.Tuple2)1 Tuple3 (org.apache.flink.api.java.tuple.Tuple3)1 ArrayListSerializer (org.apache.flink.runtime.state.ArrayListSerializer)1 StateEntry (org.apache.flink.runtime.state.StateEntry)1 StateSnapshotTransformer (org.apache.flink.runtime.state.StateSnapshotTransformer)1 StateTransformationFunction (org.apache.flink.runtime.state.StateTransformationFunction)1 StateIncrementalVisitor (org.apache.flink.runtime.state.internal.InternalKvState.StateIncrementalVisitor)1 StateEntryMatcher.entry (org.apache.flink.runtime.state.testutils.StateEntryMatcher.entry)1