Examples with GenericTypeInfo org.apache.flink.api.java.typeutils.GenericTypeInfo used on opensource projects

Example 6 with GenericTypeInfo

use of org.apache.flink.api.java.typeutils.GenericTypeInfo in project flink by apache.

the class StateBackendTestBase method testKryoRegisteringRestoreResilienceWithDefaultSerializer.

/**
 * Verify state restore resilience when: - snapshot was taken without any Kryo registrations,
 * specific serializers or default serializers for the state type - restored with a default
 * serializer for the state type
 *
 * <p>The default serializer used on restore is {@link CustomKryoTestSerializer}, which
 * deliberately fails only on deserialization. We use the deliberate deserialization failure to
 * acknowledge test success.
 *
 * @throws Exception expects {@link ExpectedKryoTestException} to be thrown.
 */
@Test
@SuppressWarnings("unchecked")
public void testKryoRegisteringRestoreResilienceWithDefaultSerializer() throws Exception {
    assumeTrue(supportsMetaInfoVerification());
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);
    try {
        TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
        // make sure that we are in fact using the KryoSerializer
        assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
        ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id", pojoType);
        ValueState<TestPojo> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        // ============== create snapshot - no Kryo registration or specific / default
        // serializers ==============
        // make some more modifications
        backend.setCurrentKey(1);
        state.update(new TestPojo("u1", 1));
        backend.setCurrentKey(2);
        state.update(new TestPojo("u2", 2));
        KeyedStateHandle snapshot = runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        backend.dispose();
        // ========== restore snapshot - should use default serializer (ONLY SERIALIZATION)
        // ==========
        // cast because our test serializer is not typed to TestPojo
        env.getExecutionConfig().addDefaultKryoSerializer(TestPojo.class, (Class) CustomKryoTestSerializer.class);
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot, env);
        // re-initialize to ensure that we create the KryoSerializer from scratch, otherwise
        // initializeSerializerUnlessSet would not pick up our new config
        kvId = new ValueStateDescriptor<>("id", pojoType);
        state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        backend.setCurrentKey(1);
        // update to test state backends that eagerly serialize, such as RocksDB
        state.update(new TestPojo("u1", 11));
        KeyedStateHandle snapshot2 = runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        snapshot.discardState();
        backend.dispose();
        // ========= restore snapshot - should use default serializer (FAIL ON DESERIALIZATION)
        // =========
        // cast because our test serializer is not typed to TestPojo
        env.getExecutionConfig().addDefaultKryoSerializer(TestPojo.class, (Class) CustomKryoTestSerializer.class);
        // on the second restore, since the custom serializer will be used for
        // deserialization, we expect the deliberate failure to be thrown
        expectedException.expect(anyOf(isA(ExpectedKryoTestException.class), Matchers.<Throwable>hasProperty("cause", isA(ExpectedKryoTestException.class))));
        // state backends that eagerly deserializes (such as the memory state backend) will fail
        // here
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot2, env);
        state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        backend.setCurrentKey(1);
        // state backends that lazily deserializes (such as RocksDB) will fail here
        state.value();
        snapshot2.discardState();
    } finally {
        // ensure to release native resources even when we exit through exception
        IOUtils.closeQuietly(backend);
        backend.dispose();
    }
}

Example 7 with GenericTypeInfo

use of org.apache.flink.api.java.typeutils.GenericTypeInfo in project flink by apache.

the class StateBackendTestBase method testBackendUsesRegisteredKryoDefaultSerializer.

@Test
@SuppressWarnings("unchecked")
public void testBackendUsesRegisteredKryoDefaultSerializer() throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);
    try {
        // cast because our test serializer is not typed to TestPojo
        env.getExecutionConfig().addDefaultKryoSerializer(TestPojo.class, (Class) ExceptionThrowingTestSerializer.class);
        TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
        // make sure that we are in fact using the KryoSerializer
        assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
        ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id", pojoType);
        ValueState<TestPojo> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        // we will be expecting ExpectedKryoTestException to be thrown,
        // because the ExceptionThrowingTestSerializer should be used
        int numExceptions = 0;
        backend.setCurrentKey(1);
        try {
            // backends that eagerly serializes (such as RocksDB) will fail here
            state.update(new TestPojo("u1", 1));
        } catch (ExpectedKryoTestException e) {
            numExceptions++;
        } catch (Exception e) {
            if (e.getCause() instanceof ExpectedKryoTestException) {
                numExceptions++;
            } else {
                throw e;
            }
        }
        try {
            // backends that lazily serializes (such as memory state backend) will fail here
            runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        } catch (ExpectedKryoTestException e) {
            numExceptions++;
        } catch (Exception e) {
            if (e.getCause() instanceof ExpectedKryoTestException) {
                numExceptions++;
            } else {
                throw e;
            }
        }
        assertTrue("Didn't see the expected Kryo exception.", numExceptions > 0);
    } finally {
        IOUtils.closeQuietly(backend);
        backend.dispose();
    }
}

Example 8 with GenericTypeInfo

use of org.apache.flink.api.java.typeutils.GenericTypeInfo in project flink by apache.

the class StateBackendTestBase method testBackendUsesRegisteredKryoDefaultSerializerUsingGetOrCreate.

@Test
@SuppressWarnings("unchecked")
public void testBackendUsesRegisteredKryoDefaultSerializerUsingGetOrCreate() throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);
    try {
        // cast because our test serializer is not typed to TestPojo
        env.getExecutionConfig().addDefaultKryoSerializer(TestPojo.class, (Class) ExceptionThrowingTestSerializer.class);
        TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
        // make sure that we are in fact using the KryoSerializer
        assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
        pojoType.createSerializer(env.getExecutionConfig());
        ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id", pojoType);
        ValueState<TestPojo> state = backend.getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE, kvId);
        assertTrue(state instanceof InternalValueState);
        ((InternalValueState) state).setCurrentNamespace(VoidNamespace.INSTANCE);
        // we will be expecting ExpectedKryoTestException to be thrown,
        // because the ExceptionThrowingTestSerializer should be used
        int numExceptions = 0;
        backend.setCurrentKey(1);
        try {
            // backends that eagerly serializes (such as RocksDB) will fail here
            state.update(new TestPojo("u1", 1));
        } catch (ExpectedKryoTestException e) {
            numExceptions++;
        } catch (Exception e) {
            if (e.getCause() instanceof ExpectedKryoTestException) {
                numExceptions++;
            } else {
                throw e;
            }
        }
        try {
            // backends that lazily serializes (such as memory state backend) will fail here
            runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        } catch (ExpectedKryoTestException e) {
            numExceptions++;
        } catch (Exception e) {
            if (e.getCause() instanceof ExpectedKryoTestException) {
                numExceptions++;
            } else {
                throw e;
            }
        }
        assertTrue("Didn't see the expected Kryo exception.", numExceptions > 0);
    } finally {
        IOUtils.closeQuietly(backend);
        backend.dispose();
    }
}

Example 9 with GenericTypeInfo

use of org.apache.flink.api.java.typeutils.GenericTypeInfo in project flink by apache.

the class StateBackendTestBase method testKryoRegisteringRestoreResilienceWithRegisteredType.

/**
 * Verify state restore resilience when: - snapshot was taken without any Kryo registrations,
 * specific serializers or default serializers for the state type - restored with the state type
 * registered (no specific serializer)
 *
 * <p>This test should not fail, because de- / serialization of the state should not be
 * performed with Kryo's default {@link com.esotericsoftware.kryo.serializers.FieldSerializer}.
 */
@Test
public void testKryoRegisteringRestoreResilienceWithRegisteredType() throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
    // make sure that we are in fact using the KryoSerializer
    assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
    ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id", pojoType);
    CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);
    try {
        ValueState<TestPojo> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        // ============== create snapshot - no Kryo registration or specific / default
        // serializers
        // ==============
        // make some more modifications
        backend.setCurrentKey(1);
        state.update(new TestPojo("u1", 1));
        backend.setCurrentKey(2);
        state.update(new TestPojo("u2", 2));
        KeyedStateHandle snapshot = runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        IOUtils.closeQuietly(backend);
        backend.dispose();
        // ====================================== restore snapshot
        // ======================================
        env.getExecutionConfig().registerKryoType(TestPojo.class);
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot, env);
        snapshot.discardState();
        state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        backend.setCurrentKey(1);
        assertEquals(state.value(), new TestPojo("u1", 1));
        backend.setCurrentKey(2);
        assertEquals(state.value(), new TestPojo("u2", 2));
    } finally {
        IOUtils.closeQuietly(backend);
        backend.dispose();
    }
}

Example 10 with GenericTypeInfo

use of org.apache.flink.api.java.typeutils.GenericTypeInfo in project flink by apache.

the class BinaryRowDataTest method testGenericObject.

@Test
public void testGenericObject() throws Exception {
    GenericTypeInfo<MyObj> info = new GenericTypeInfo<>(MyObj.class);
    TypeSerializer<MyObj> genericSerializer = info.createSerializer(new ExecutionConfig());
    RawValueDataSerializer<MyObj> binarySerializer = new RawValueDataSerializer<>(genericSerializer);
    BinaryRowData row = new BinaryRowData(4);
    BinaryRowWriter writer = new BinaryRowWriter(row);
    writer.writeInt(0, 0);
    RawValueData<MyObj> myObj1 = RawValueData.fromObject(new MyObj(0, 1));
    writer.writeRawValue(1, myObj1, binarySerializer);
    RawValueData<MyObj> myObj2 = RawValueData.fromObject(new MyObj(123, 5.0));
    writer.writeRawValue(2, myObj2, binarySerializer);
    RawValueData<MyObj> myObj3 = RawValueData.fromObject(new MyObj(1, 1));
    writer.writeRawValue(3, myObj3, binarySerializer);
    writer.complete();
    assertTestGenericObjectRow(row, genericSerializer);
    // getBytes from var-length memorySegments.
    BinaryRowDataSerializer serializer = new BinaryRowDataSerializer(4);
    MemorySegment[] memorySegments = new MemorySegment[3];
    ArrayList<MemorySegment> memorySegmentList = new ArrayList<>();
    for (int i = 0; i < 3; i++) {
        memorySegments[i] = MemorySegmentFactory.wrap(new byte[64]);
        memorySegmentList.add(memorySegments[i]);
    }
    RandomAccessOutputView out = new RandomAccessOutputView(memorySegments, 64);
    serializer.serializeToPages(row, out);
    BinaryRowData mapRow = serializer.createInstance();
    mapRow = serializer.mapFromPages(mapRow, new RandomAccessInputView(memorySegmentList, 64));
    assertTestGenericObjectRow(mapRow, genericSerializer);
}