Search in sources :

Example 1 with StateEntry

use of org.apache.flink.runtime.state.StateEntry in project flink by apache.

the class CopyOnWriteSkipListStateMapTestUtils method verifySnapshotWithoutTransform.

static <K, N, S> void verifySnapshotWithoutTransform(Map<N, Map<K, S>> referenceStates, @Nonnull CopyOnWriteSkipListStateMapSnapshot<K, N, S> snapshot, TypeSerializer<K> keySerializer, TypeSerializer<N> namespaceSerializer, TypeSerializer<S> stateSerializer, SnapshotVerificationMode verificationMode) throws IOException {
    ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos();
    DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);
    if (verificationMode == SnapshotVerificationMode.ITERATOR) {
        Iterator<StateEntry<K, N, S>> iterator = snapshot.getIterator(keySerializer, namespaceSerializer, stateSerializer, null);
        assertThat(() -> iterator, containsInAnyOrder(toMatchers(referenceStates)));
    } else {
        snapshot.writeState(keySerializer, namespaceSerializer, stateSerializer, outputView, null);
        Map<N, Map<K, S>> actualStates = readStateFromSnapshot(outputStream.toByteArray(), keySerializer, namespaceSerializer, stateSerializer);
        assertEquals(referenceStates, actualStates);
    }
}
Also used : StateEntry(org.apache.flink.runtime.state.StateEntry) DataOutputViewStreamWrapper(org.apache.flink.core.memory.DataOutputViewStreamWrapper) DataOutputView(org.apache.flink.core.memory.DataOutputView) ByteArrayOutputStreamWithPos(org.apache.flink.core.memory.ByteArrayOutputStreamWithPos) HashMap(java.util.HashMap) Map(java.util.Map)

Example 2 with StateEntry

use of org.apache.flink.runtime.state.StateEntry in project flink by apache.

the class CopyOnWriteSkipListStateMapTestUtils method verifySnapshotWithTransform.

static <K, N, S> void verifySnapshotWithTransform(@Nonnull Map<N, Map<K, S>> referenceStates, @Nonnull CopyOnWriteSkipListStateMapSnapshot<K, N, S> snapshot, StateSnapshotTransformer<S> transformer, TypeSerializer<K> keySerializer, TypeSerializer<N> namespaceSerializer, TypeSerializer<S> stateSerializer, SnapshotVerificationMode verificationMode) throws IOException {
    ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos();
    DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);
    Map<N, Map<K, S>> transformedStates = new HashMap<>();
    for (Map.Entry<N, Map<K, S>> namespaceEntry : referenceStates.entrySet()) {
        for (Map.Entry<K, S> keyEntry : namespaceEntry.getValue().entrySet()) {
            S state = transformer.filterOrTransform(keyEntry.getValue());
            if (state != null) {
                transformedStates.computeIfAbsent(namespaceEntry.getKey(), (none) -> new HashMap<>()).put(keyEntry.getKey(), state);
            }
        }
    }
    if (verificationMode == SnapshotVerificationMode.SERIALIZED) {
        snapshot.writeState(keySerializer, namespaceSerializer, stateSerializer, outputView, transformer);
        Map<N, Map<K, S>> actualStates = readStateFromSnapshot(outputStream.toByteArray(), keySerializer, namespaceSerializer, stateSerializer);
        assertEquals(transformedStates, actualStates);
    } else {
        Iterator<StateEntry<K, N, S>> iterator = snapshot.getIterator(keySerializer, namespaceSerializer, stateSerializer, null);
        assertThat(() -> iterator, containsInAnyOrder(toMatchers(referenceStates)));
    }
}
Also used : HashMap(java.util.HashMap) StringSerializer(org.apache.flink.api.common.typeutils.base.StringSerializer) DataOutputView(org.apache.flink.core.memory.DataOutputView) ByteArrayInputStreamWithPos(org.apache.flink.core.memory.ByteArrayInputStreamWithPos) ArrayList(java.util.ArrayList) DataOutputViewStreamWrapper(org.apache.flink.core.memory.DataOutputViewStreamWrapper) HashSet(java.util.HashSet) Assert.assertThat(org.junit.Assert.assertThat) IntSerializer(org.apache.flink.api.common.typeutils.base.IntSerializer) DEFAULT_MAX_KEYS_TO_DELETE_ONE_TIME(org.apache.flink.runtime.state.heap.CopyOnWriteSkipListStateMap.DEFAULT_MAX_KEYS_TO_DELETE_ONE_TIME) Map(java.util.Map) Matchers.nullValue(org.hamcrest.Matchers.nullValue) StateSnapshotTransformer(org.apache.flink.runtime.state.StateSnapshotTransformer) LongSerializer(org.apache.flink.api.common.typeutils.base.LongSerializer) DataInputView(org.apache.flink.core.memory.DataInputView) InternalKvState(org.apache.flink.runtime.state.internal.InternalKvState) StateEntryMatcher.entry(org.apache.flink.runtime.state.testutils.StateEntryMatcher.entry) Allocator(org.apache.flink.runtime.state.heap.space.Allocator) Nonnull(javax.annotation.Nonnull) NIL_NODE(org.apache.flink.runtime.state.heap.SkipListUtils.NIL_NODE) DEFAULT_LOGICAL_REMOVED_KEYS_RATIO(org.apache.flink.runtime.state.heap.CopyOnWriteSkipListStateMap.DEFAULT_LOGICAL_REMOVED_KEYS_RATIO) DataInputViewStreamWrapper(org.apache.flink.core.memory.DataInputViewStreamWrapper) TypeSerializer(org.apache.flink.api.common.typeutils.TypeSerializer) Iterator(java.util.Iterator) Collection(java.util.Collection) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) ByteArrayOutputStreamWithPos(org.apache.flink.core.memory.ByteArrayOutputStreamWithPos) Collectors(java.util.stream.Collectors) List(java.util.List) Assert.assertNull(org.junit.Assert.assertNull) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) Matcher(org.hamcrest.Matcher) StateEntry(org.apache.flink.runtime.state.StateEntry) Assert.assertEquals(org.junit.Assert.assertEquals) HashMap(java.util.HashMap) DataOutputView(org.apache.flink.core.memory.DataOutputView) StateEntry(org.apache.flink.runtime.state.StateEntry) DataOutputViewStreamWrapper(org.apache.flink.core.memory.DataOutputViewStreamWrapper) ByteArrayOutputStreamWithPos(org.apache.flink.core.memory.ByteArrayOutputStreamWithPos) HashMap(java.util.HashMap) Map(java.util.Map)

Aggregations

HashMap (java.util.HashMap)2 Map (java.util.Map)2 ByteArrayOutputStreamWithPos (org.apache.flink.core.memory.ByteArrayOutputStreamWithPos)2 DataOutputView (org.apache.flink.core.memory.DataOutputView)2 DataOutputViewStreamWrapper (org.apache.flink.core.memory.DataOutputViewStreamWrapper)2 StateEntry (org.apache.flink.runtime.state.StateEntry)2 IOException (java.io.IOException)1 ArrayList (java.util.ArrayList)1 Collection (java.util.Collection)1 HashSet (java.util.HashSet)1 Iterator (java.util.Iterator)1 List (java.util.List)1 Set (java.util.Set)1 Collectors (java.util.stream.Collectors)1 Nonnull (javax.annotation.Nonnull)1 TypeSerializer (org.apache.flink.api.common.typeutils.TypeSerializer)1 IntSerializer (org.apache.flink.api.common.typeutils.base.IntSerializer)1 LongSerializer (org.apache.flink.api.common.typeutils.base.LongSerializer)1 StringSerializer (org.apache.flink.api.common.typeutils.base.StringSerializer)1 ByteArrayInputStreamWithPos (org.apache.flink.core.memory.ByteArrayInputStreamWithPos)1