Examples with TestType org.apache.flink.runtime.testutils.statemigration.TestType used on opensource projects

Example 6 with TestType

use of org.apache.flink.runtime.testutils.statemigration.TestType in project flink by apache.

the class StateBackendMigrationTestBase method testKeySerializerUpgrade.

private void testKeySerializerUpgrade(TypeSerializer<TestType> initialKeySerializer, TypeSerializer<TestType> newKeySerializer) throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    AbstractKeyedStateBackend<TestType> backend = createKeyedBackend(initialKeySerializer);
    final String stateName = "test-name";
    try {
        ValueStateDescriptor<Integer> kvId = new ValueStateDescriptor<>(stateName, Integer.class);
        ValueState<Integer> valueState = backend.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId);
        backend.setCurrentKey(new TestType("foo", 123));
        valueState.update(1);
        backend.setCurrentKey(new TestType("bar", 456));
        valueState.update(5);
        KeyedStateHandle snapshot = runSnapshot(backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        backend.dispose();
        backend = restoreKeyedBackend(newKeySerializer, snapshot);
        valueState = backend.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, kvId);
        // access and check previous state
        backend.setCurrentKey(new TestType("foo", 123));
        assertEquals(1, valueState.value().intValue());
        backend.setCurrentKey(new TestType("bar", 456));
        assertEquals(5, valueState.value().intValue());
        // do another snapshot to verify the snapshot logic after migration
        snapshot = runSnapshot(backend.snapshot(2L, 3L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        snapshot.discardState();
    } finally {
        backend.dispose();
    }
}

Example 7 with TestType

use of org.apache.flink.runtime.testutils.statemigration.TestType in project flink by apache.

the class StateBackendMigrationTestBase method testKeyedValueStateUpgrade.

private void testKeyedValueStateUpgrade(ValueStateDescriptor<TestType> initialAccessDescriptor, ValueStateDescriptor<TestType> newAccessDescriptorAfterRestore) throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    try {
        ValueState<TestType> valueState = backend.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, initialAccessDescriptor);
        backend.setCurrentKey(1);
        valueState.update(new TestType("foo", 1456));
        backend.setCurrentKey(2);
        valueState.update(new TestType("bar", 478));
        backend.setCurrentKey(3);
        valueState.update(new TestType("hello", 189));
        KeyedStateHandle snapshot = runSnapshot(backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        backend.dispose();
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);
        valueState = backend.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, newAccessDescriptorAfterRestore);
        // make sure that reading and writing each key state works with the new serializer
        backend.setCurrentKey(1);
        assertEquals(new TestType("foo", 1456), valueState.value());
        valueState.update(new TestType("newValue1", 751));
        backend.setCurrentKey(2);
        assertEquals(new TestType("bar", 478), valueState.value());
        valueState.update(new TestType("newValue2", 167));
        backend.setCurrentKey(3);
        assertEquals(new TestType("hello", 189), valueState.value());
        valueState.update(new TestType("newValue3", 444));
        // do another snapshot to verify the snapshot logic after migration
        snapshot = runSnapshot(backend.snapshot(2L, 3L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        snapshot.discardState();
    } finally {
        backend.dispose();
    }
}

Example 8 with TestType

use of org.apache.flink.runtime.testutils.statemigration.TestType in project flink by apache.

the class StateBackendTestBase method testKeyGroupedInternalPriorityQueue.

public void testKeyGroupedInternalPriorityQueue(boolean addAll) throws Exception {
    String fieldName = "key-grouped-priority-queue";
    CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    try {
        KeyGroupedInternalPriorityQueue<TestType> priorityQueue = backend.create(fieldName, new TestType.V1TestTypeSerializer());
        TestType elementA42 = new TestType("a", 42);
        TestType elementA44 = new TestType("a", 44);
        TestType elementB1 = new TestType("b", 1);
        TestType elementB3 = new TestType("b", 3);
        TestType[] elements = { elementA44, elementB1, elementB1, elementB3, elementA42 };
        if (addAll) {
            priorityQueue.addAll(asList(elements));
        } else {
            assertTrue(priorityQueue.add(elements[0]));
            assertTrue(priorityQueue.add(elements[1]));
            assertFalse(priorityQueue.add(elements[2]));
            assertFalse(priorityQueue.add(elements[3]));
            assertFalse(priorityQueue.add(elements[4]));
        }
        assertFalse(priorityQueue.isEmpty());
        assertThat(priorityQueue.getSubsetForKeyGroup(1), containsInAnyOrder(elementA42, elementA44));
        assertThat(priorityQueue.getSubsetForKeyGroup(8), containsInAnyOrder(elementB1, elementB3));
        assertThat(priorityQueue.peek(), equalTo(elementB1));
        assertThat(priorityQueue.poll(), equalTo(elementB1));
        assertThat(priorityQueue.peek(), equalTo(elementB3));
        List<TestType> actualList = new ArrayList<>();
        try (CloseableIterator<TestType> iterator = priorityQueue.iterator()) {
            iterator.forEachRemaining(actualList::add);
        }
        assertThat(actualList, containsInAnyOrder(elementB3, elementA42, elementA44));
        assertEquals(3, priorityQueue.size());
        assertFalse(priorityQueue.remove(elementB1));
        assertTrue(priorityQueue.remove(elementB3));
        assertThat(priorityQueue.peek(), equalTo(elementA42));
    } finally {
        IOUtils.closeQuietly(backend);
        backend.dispose();
    }
}

Example 9 with TestType

use of org.apache.flink.runtime.testutils.statemigration.TestType in project flink by apache.

the class StateBackendMigrationTestBase method testKeyedListStateUpgrade.

private void testKeyedListStateUpgrade(ListStateDescriptor<TestType> initialAccessDescriptor, ListStateDescriptor<TestType> newAccessDescriptorAfterRestore) throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    try {
        ListState<TestType> listState = backend.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, initialAccessDescriptor);
        backend.setCurrentKey(1);
        listState.add(new TestType("key-1", 1));
        listState.add(new TestType("key-1", 2));
        listState.add(new TestType("key-1", 3));
        backend.setCurrentKey(2);
        listState.add(new TestType("key-2", 1));
        backend.setCurrentKey(3);
        listState.add(new TestType("key-3", 1));
        listState.add(new TestType("key-3", 2));
        KeyedStateHandle snapshot = runSnapshot(backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        backend.dispose();
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);
        listState = backend.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, newAccessDescriptorAfterRestore);
        // make sure that reading and writing each key state works with the new serializer
        backend.setCurrentKey(1);
        Iterator<TestType> iterable1 = listState.get().iterator();
        assertEquals(new TestType("key-1", 1), iterable1.next());
        assertEquals(new TestType("key-1", 2), iterable1.next());
        assertEquals(new TestType("key-1", 3), iterable1.next());
        Assert.assertFalse(iterable1.hasNext());
        listState.add(new TestType("new-key-1", 123));
        backend.setCurrentKey(2);
        Iterator<TestType> iterable2 = listState.get().iterator();
        assertEquals(new TestType("key-2", 1), iterable2.next());
        Assert.assertFalse(iterable2.hasNext());
        listState.add(new TestType("new-key-2", 456));
        backend.setCurrentKey(3);
        Iterator<TestType> iterable3 = listState.get().iterator();
        assertEquals(new TestType("key-3", 1), iterable3.next());
        assertEquals(new TestType("key-3", 2), iterable3.next());
        Assert.assertFalse(iterable3.hasNext());
        listState.add(new TestType("new-key-3", 777));
        // do another snapshot to verify the snapshot logic after migration
        snapshot = runSnapshot(backend.snapshot(2L, 3L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        snapshot.discardState();
    } finally {
        backend.dispose();
    }
}

Example 10 with TestType

use of org.apache.flink.runtime.testutils.statemigration.TestType in project flink by apache.

the class StateBackendMigrationTestBase method testPriorityQueueStateCreationFailsIfNewSerializerIsNotCompatible.

// -------------------------------------------------------------------------------
// Tests for keyed priority queue state
// -------------------------------------------------------------------------------
@Test
public void testPriorityQueueStateCreationFailsIfNewSerializerIsNotCompatible() throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    try {
        InternalPriorityQueue<TestType> internalPriorityQueue = backend.create("testPriorityQueue", new TestType.V1TestTypeSerializer());
        internalPriorityQueue.add(new TestType("key-1", 123));
        internalPriorityQueue.add(new TestType("key-2", 346));
        internalPriorityQueue.add(new TestType("key-1", 777));
        KeyedStateHandle snapshot = runSnapshot(backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        backend.dispose();
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);
        backend.create("testPriorityQueue", new TestType.IncompatibleTestTypeSerializer());
        fail("should have failed");
    } catch (Exception e) {
        Assert.assertTrue(ExceptionUtils.findThrowable(e, StateMigrationException.class).isPresent());
    } finally {
        backend.dispose();
    }
}