Examples with TestType org.apache.flink.runtime.testutils.statemigration.TestType used on opensource projects

Example 11 with TestType

use of org.apache.flink.runtime.testutils.statemigration.TestType in project flink by apache.

the class StateBackendMigrationTestBase method testOperatorPartitionableListStateUpgrade.

private void testOperatorPartitionableListStateUpgrade(ListStateDescriptor<TestType> initialAccessDescriptor, ListStateDescriptor<TestType> newAccessDescriptorAfterRestore) throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    OperatorStateBackend backend = createOperatorStateBackend();
    try {
        ListState<TestType> state = backend.getListState(initialAccessDescriptor);
        state.add(new TestType("foo", 13));
        state.add(new TestType("bar", 278));
        OperatorStateHandle snapshot = runSnapshot(backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
        backend.dispose();
        backend = restoreOperatorStateBackend(snapshot);
        state = backend.getListState(newAccessDescriptorAfterRestore);
        // make sure that reading and writing each state partition works with the new serializer
        Iterator<TestType> iterator = state.get().iterator();
        assertEquals(new TestType("foo", 13), iterator.next());
        assertEquals(new TestType("bar", 278), iterator.next());
        Assert.assertFalse(iterator.hasNext());
        state.add(new TestType("new-entry", 777));
    } finally {
        backend.dispose();
    }
}

Example 12 with TestType

use of org.apache.flink.runtime.testutils.statemigration.TestType in project flink by apache.

the class ChangelogStateBackendTestUtils method testMaterializedRestoreForPriorityQueue.

public static void testMaterializedRestoreForPriorityQueue(StateBackend stateBackend, Environment env, CheckpointStreamFactory streamFactory) throws Exception {
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    String fieldName = "key-grouped-priority-queue";
    ChangelogKeyedStateBackend<Integer> keyedBackend = (ChangelogKeyedStateBackend<Integer>) createKeyedBackend(stateBackend, env);
    CompletableFuture<Void> asyncComplete = new CompletableFuture<>();
    PeriodicMaterializationManager periodicMaterializationManager = periodicMaterializationManager(keyedBackend, asyncComplete);
    try {
        KeyGroupedInternalPriorityQueue<TestType> priorityQueue = keyedBackend.create(fieldName, new TestType.V1TestTypeSerializer());
        TestType elementA100 = new TestType("a", 100);
        TestType elementA10 = new TestType("a", 10);
        TestType elementA20 = new TestType("a", 20);
        assertTrue(priorityQueue.add(elementA100));
        assertTrue(priorityQueue.add(elementA10));
        assertFalse(priorityQueue.add(elementA20));
        assertFalse(priorityQueue.add(elementA10));
        List<TestType> actualList = new ArrayList<>();
        try (CloseableIterator<TestType> iterator = priorityQueue.iterator()) {
            iterator.forEachRemaining(actualList::add);
        }
        assertThat(actualList, containsInAnyOrder(elementA100, elementA10, elementA20));
        materialize(keyedBackend, periodicMaterializationManager);
        TestType elementB9 = new TestType("b", 9);
        assertTrue(priorityQueue.add(elementB9));
        materialize(keyedBackend, periodicMaterializationManager);
        TestType elementC9 = new TestType("c", 9);
        TestType elementC8 = new TestType("c", 8);
        assertFalse(priorityQueue.add(elementC9));
        assertTrue(priorityQueue.add(elementC8));
        KeyedStateHandle snapshot = runSnapshot(keyedBackend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        IOUtils.closeQuietly(keyedBackend);
        keyedBackend.dispose();
        // make sure the asycn phase completes successfully
        if (asyncComplete.isCompletedExceptionally()) {
            asyncComplete.get();
        }
        // ============================ restore snapshot ===============================
        keyedBackend = (ChangelogKeyedStateBackend<Integer>) restoreKeyedBackend(stateBackend, snapshot, env);
        snapshot.discardState();
        KeyGroupedInternalPriorityQueue<TestType> priorityQueueRestored = keyedBackend.create(fieldName, new TestType.V1TestTypeSerializer());
        List<TestType> actualListRestore = new ArrayList<>();
        try (CloseableIterator<TestType> iterator = priorityQueueRestored.iterator()) {
            iterator.forEachRemaining(actualListRestore::add);
        }
        assertThat(actualListRestore, containsInAnyOrder(elementA100, elementA10, elementA20, elementB9, elementC9, elementC8));
        assertFalse(priorityQueueRestored.add(new TestType("d", 11)));
    } finally {
        IOUtils.closeQuietly(keyedBackend);
        keyedBackend.dispose();
    }
}