Examples with TestType org.apache.flink.runtime.testutils.statemigration.TestType used on opensource projects

Example 1 with TestType

use of org.apache.flink.runtime.testutils.statemigration.TestType in project flink by apache.

the class StateBackendMigrationTestBase method testOperatorUnionListStateUpgrade.

private void testOperatorUnionListStateUpgrade(ListStateDescriptor<TestType> initialAccessDescriptor, ListStateDescriptor<TestType> newAccessDescriptorAfterRestore) throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    OperatorStateBackend backend = createOperatorStateBackend();
    try {
        ListState<TestType> state = backend.getUnionListState(initialAccessDescriptor);
        state.add(new TestType("foo", 13));
        state.add(new TestType("bar", 278));
        OperatorStateHandle snapshot = runSnapshot(backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
        backend.dispose();
        backend = restoreOperatorStateBackend(snapshot);
        state = backend.getUnionListState(newAccessDescriptorAfterRestore);
        // the state backend should have decided whether or not it needs to perform state
        // migration;
        // make sure that reading and writing each state partition works with the new serializer
        Iterator<TestType> iterator = state.get().iterator();
        assertEquals(new TestType("foo", 13), iterator.next());
        assertEquals(new TestType("bar", 278), iterator.next());
        Assert.assertFalse(iterator.hasNext());
        state.add(new TestType("new-entry", 777));
    } finally {
        backend.dispose();
    }
}

Example 2 with TestType

use of org.apache.flink.runtime.testutils.statemigration.TestType in project flink by apache.

the class StateBackendMigrationTestBase method testNamespaceSerializerUpgrade.

private void testNamespaceSerializerUpgrade(TypeSerializer<TestType> initialNamespaceSerializer, TypeSerializer<TestType> newNamespaceSerializerAfterRestore) throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    final String stateName = "test-name";
    try {
        ValueStateDescriptor<Integer> kvId = new ValueStateDescriptor<>(stateName, Integer.class);
        ValueState<Integer> valueState = backend.getPartitionedState(new TestType("namespace", 123), initialNamespaceSerializer, kvId);
        backend.setCurrentKey(1);
        valueState.update(10);
        backend.setCurrentKey(5);
        valueState.update(50);
        KeyedStateHandle snapshot = runSnapshot(backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        // test incompatible namespace serializer; start with a freshly restored backend
        backend.dispose();
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);
        valueState = backend.getPartitionedState(new TestType("namespace", 123), newNamespaceSerializerAfterRestore, kvId);
        // access and check previous state
        backend.setCurrentKey(1);
        assertEquals(10, valueState.value().intValue());
        valueState.update(10);
        backend.setCurrentKey(5);
        assertEquals(50, valueState.value().intValue());
        // do another snapshot to verify the snapshot logic after migration
        snapshot = runSnapshot(backend.snapshot(2L, 3L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        snapshot.discardState();
    } finally {
        backend.dispose();
    }
}

Example 3 with TestType

use of org.apache.flink.runtime.testutils.statemigration.TestType in project flink by apache.

the class StateBackendMigrationTestBase method testBroadcastStateKeyUpgrade.

private void testBroadcastStateKeyUpgrade(MapStateDescriptor<TestType, Integer> initialAccessDescriptor, MapStateDescriptor<TestType, Integer> newAccessDescriptorAfterRestore) throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    OperatorStateBackend backend = createOperatorStateBackend();
    try {
        BroadcastState<TestType, Integer> state = backend.getBroadcastState(initialAccessDescriptor);
        state.put(new TestType("foo", 13), 3);
        state.put(new TestType("bar", 278), 5);
        OperatorStateHandle snapshot = runSnapshot(backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
        backend.dispose();
        backend = restoreOperatorStateBackend(snapshot);
        state = backend.getBroadcastState(newAccessDescriptorAfterRestore);
        // the state backend should have decided whether or not it needs to perform state
        // migration;
        // make sure that reading and writing each broadcast entry works with the new serializer
        assertEquals((Integer) 3, state.get(new TestType("foo", 13)));
        assertEquals((Integer) 5, state.get(new TestType("bar", 278)));
        state.put(new TestType("new-entry", 777), 17);
    } finally {
        backend.dispose();
    }
}

Example 4 with TestType

use of org.apache.flink.runtime.testutils.statemigration.TestType in project flink by apache.

the class StateBackendMigrationTestBase method testKeyedMapStateUpgrade.

private void testKeyedMapStateUpgrade(MapStateDescriptor<Integer, TestType> initialAccessDescriptor, MapStateDescriptor<Integer, TestType> newAccessDescriptorAfterRestore) throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    try {
        MapState<Integer, TestType> mapState = backend.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, initialAccessDescriptor);
        backend.setCurrentKey(1);
        mapState.put(1, new TestType("key-1", 1));
        mapState.put(2, new TestType("key-1", 2));
        mapState.put(3, new TestType("key-1", 3));
        backend.setCurrentKey(2);
        mapState.put(1, new TestType("key-2", 1));
        backend.setCurrentKey(3);
        mapState.put(1, new TestType("key-3", 1));
        mapState.put(2, new TestType("key-3", 2));
        KeyedStateHandle snapshot = runSnapshot(backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        backend.dispose();
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot);
        mapState = backend.getPartitionedState(VoidNamespace.INSTANCE, CustomVoidNamespaceSerializer.INSTANCE, newAccessDescriptorAfterRestore);
        // make sure that reading and writing each key state works with the new serializer
        backend.setCurrentKey(1);
        Iterator<Map.Entry<Integer, TestType>> iterable1 = mapState.iterator();
        Map.Entry<Integer, TestType> actual = iterable1.next();
        assertEquals((Integer) 1, actual.getKey());
        assertEquals(new TestType("key-1", 1), actual.getValue());
        actual = iterable1.next();
        assertEquals((Integer) 2, actual.getKey());
        assertEquals(new TestType("key-1", 2), actual.getValue());
        actual = iterable1.next();
        assertEquals((Integer) 3, actual.getKey());
        assertEquals(new TestType("key-1", 3), actual.getValue());
        Assert.assertFalse(iterable1.hasNext());
        mapState.put(123, new TestType("new-key-1", 123));
        backend.setCurrentKey(2);
        Iterator<Map.Entry<Integer, TestType>> iterable2 = mapState.iterator();
        actual = iterable2.next();
        assertEquals((Integer) 1, actual.getKey());
        assertEquals(new TestType("key-2", 1), actual.getValue());
        Assert.assertFalse(iterable2.hasNext());
        mapState.put(456, new TestType("new-key-2", 456));
        backend.setCurrentKey(3);
        Iterator<Map.Entry<Integer, TestType>> iterable3 = mapState.iterator();
        actual = iterable3.next();
        assertEquals((Integer) 1, actual.getKey());
        assertEquals(new TestType("key-3", 1), actual.getValue());
        actual = iterable3.next();
        assertEquals((Integer) 2, actual.getKey());
        assertEquals(new TestType("key-3", 2), actual.getValue());
        Assert.assertFalse(iterable3.hasNext());
        mapState.put(777, new TestType("new-key-3", 777));
        // do another snapshot to verify the snapshot logic after migration
        snapshot = runSnapshot(backend.snapshot(2L, 3L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        snapshot.discardState();
    } finally {
        backend.dispose();
    }
}

Example 5 with TestType

use of org.apache.flink.runtime.testutils.statemigration.TestType in project flink by apache.

the class StateBackendMigrationTestBase method testBroadcastStateValueUpgrade.

private void testBroadcastStateValueUpgrade(MapStateDescriptor<Integer, TestType> initialAccessDescriptor, MapStateDescriptor<Integer, TestType> newAccessDescriptorAfterRestore) throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    OperatorStateBackend backend = createOperatorStateBackend();
    try {
        BroadcastState<Integer, TestType> state = backend.getBroadcastState(initialAccessDescriptor);
        state.put(3, new TestType("foo", 13));
        state.put(5, new TestType("bar", 278));
        OperatorStateHandle snapshot = runSnapshot(backend.snapshot(1L, 2L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()));
        backend.dispose();
        backend = restoreOperatorStateBackend(snapshot);
        state = backend.getBroadcastState(newAccessDescriptorAfterRestore);
        // the state backend should have decided whether or not it needs to perform state
        // migration;
        // make sure that reading and writing each broadcast entry works with the new serializer
        assertEquals(new TestType("foo", 13), state.get(3));
        assertEquals(new TestType("bar", 278), state.get(5));
        state.put(17, new TestType("new-entry", 777));
    } finally {
        backend.dispose();
    }
}