Examples with GenericTypeInfo org.apache.flink.api.java.typeutils.GenericTypeInfo used on opensource projects

Example 16 with GenericTypeInfo

use of org.apache.flink.api.java.typeutils.GenericTypeInfo in project flink by apache.

the class AvroTypeInfo method generateFieldsFromAvroSchema.

@Internal
private static <T extends SpecificRecordBase> List<PojoField> generateFieldsFromAvroSchema(Class<T> typeClass) {
    PojoTypeExtractor pte = new PojoTypeExtractor();
    List<Type> typeHierarchy = new ArrayList<>();
    typeHierarchy.add(typeClass);
    TypeInformation<T> ti = pte.analyzePojo(typeClass, typeHierarchy, null, null);
    if (!(ti instanceof PojoTypeInfo)) {
        throw new IllegalStateException("Expecting type to be a PojoTypeInfo");
    }
    PojoTypeInfo<T> pti = (PojoTypeInfo<T>) ti;
    List<PojoField> newFields = new ArrayList<>(pti.getTotalFields());
    for (int i = 0; i < pti.getArity(); i++) {
        PojoField f = pti.getPojoFieldAt(i);
        TypeInformation<?> newType = f.getTypeInformation();
        // check if type is a CharSequence
        if (newType instanceof GenericTypeInfo) {
            if ((newType).getTypeClass().equals(CharSequence.class)) {
                // replace the type by a org.apache.avro.util.Utf8
                newType = new GenericTypeInfo<>(org.apache.avro.util.Utf8.class);
            }
        }
        PojoField newField = new PojoField(f.getField(), newType);
        newFields.add(newField);
    }
    return newFields;
}

Example 17 with GenericTypeInfo

use of org.apache.flink.api.java.typeutils.GenericTypeInfo in project flink by apache.

the class StreamExecutionEnvironmentTest method testFromElementsPostConstructionType.

@Test
public void testFromElementsPostConstructionType() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStreamSource<String> source = env.fromElements("a", "b");
    TypeInformation<String> customType = new GenericTypeInfo<>(String.class);
    source.returns(customType);
    FromElementsFunction<String> elementsFunction = (FromElementsFunction<String>) getFunctionFromDataSource(source);
    assertNotEquals(BasicTypeInfo.STRING_TYPE_INFO.createSerializer(env.getConfig()), elementsFunction.getSerializer());
    assertEquals(customType.createSerializer(env.getConfig()), elementsFunction.getSerializer());
}

Example 18 with GenericTypeInfo

use of org.apache.flink.api.java.typeutils.GenericTypeInfo in project flink by apache.

the class StateBackendTestBase method testBackendUsesRegisteredKryoSerializer.

@Test
public void testBackendUsesRegisteredKryoSerializer() throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    env.getExecutionConfig().registerTypeWithKryoSerializer(TestPojo.class, ExceptionThrowingTestSerializer.class);
    TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
    // make sure that we are in fact using the KryoSerializer
    assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
    ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id", pojoType);
    CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);
    try {
        ValueState<TestPojo> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        // we will be expecting ExpectedKryoTestException to be thrown,
        // because the ExceptionThrowingTestSerializer should be used
        int numExceptions = 0;
        backend.setCurrentKey(1);
        try {
            // backends that eagerly serializes (such as RocksDB) will fail here
            state.update(new TestPojo("u1", 1));
        } catch (ExpectedKryoTestException e) {
            numExceptions++;
        } catch (Exception e) {
            if (e.getCause() instanceof ExpectedKryoTestException) {
                numExceptions++;
            } else {
                throw e;
            }
        }
        try {
            // backends that lazily serializes (such as memory state backend) will fail here
            runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        } catch (ExpectedKryoTestException e) {
            numExceptions++;
        } catch (Exception e) {
            if (e.getCause() instanceof ExpectedKryoTestException) {
                numExceptions++;
            } else {
                throw e;
            }
        }
        assertTrue("Didn't see the expected Kryo exception.", numExceptions > 0);
    } finally {
        IOUtils.closeQuietly(backend);
        backend.dispose();
    }
}

Example 19 with GenericTypeInfo

use of org.apache.flink.api.java.typeutils.GenericTypeInfo in project flink by apache.

the class StateBackendTestBase method testBackendUsesRegisteredKryoSerializerUsingGetOrCreate.

@Test
@SuppressWarnings("unchecked")
public void testBackendUsesRegisteredKryoSerializerUsingGetOrCreate() throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    env.getExecutionConfig().registerTypeWithKryoSerializer(TestPojo.class, ExceptionThrowingTestSerializer.class);
    TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
    // make sure that we are in fact using the KryoSerializer
    assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
    ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id", pojoType);
    CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);
    try {
        ValueState<TestPojo> state = backend.getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE, kvId);
        assertTrue(state instanceof InternalValueState);
        ((InternalValueState) state).setCurrentNamespace(VoidNamespace.INSTANCE);
        // we will be expecting ExpectedKryoTestException to be thrown,
        // because the ExceptionThrowingTestSerializer should be used
        int numExceptions = 0;
        backend.setCurrentKey(1);
        try {
            // backends that eagerly serializes (such as RocksDB) will fail here
            state.update(new TestPojo("u1", 1));
        } catch (ExpectedKryoTestException e) {
            numExceptions++;
        } catch (Exception e) {
            if (e.getCause() instanceof ExpectedKryoTestException) {
                numExceptions++;
            } else {
                throw e;
            }
        }
        try {
            // backends that lazily serializes (such as memory state backend) will fail here
            runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        } catch (ExpectedKryoTestException e) {
            numExceptions++;
        } catch (Exception e) {
            if (e.getCause() instanceof ExpectedKryoTestException) {
                numExceptions++;
            } else {
                throw e;
            }
        }
        assertTrue("Didn't see the expected Kryo exception.", numExceptions > 0);
    } finally {
        IOUtils.closeQuietly(backend);
        backend.dispose();
    }
}

Example 20 with GenericTypeInfo

use of org.apache.flink.api.java.typeutils.GenericTypeInfo in project flink by apache.

the class StateBackendTestBase method testKryoRegisteringRestoreResilienceWithRegisteredSerializer.

/**
 * Verify state restore resilience when: - snapshot was taken without any Kryo registrations,
 * specific serializers or default serializers for the state type - restored with a specific
 * serializer for the state type
 *
 * <p>The specific serializer used on restore is {@link CustomKryoTestSerializer}, which
 * deliberately fails only on deserialization. We use the deliberate deserialization failure to
 * acknowledge test success.
 *
 * @throws Exception expects {@link ExpectedKryoTestException} to be thrown.
 */
@Test
public void testKryoRegisteringRestoreResilienceWithRegisteredSerializer() throws Exception {
    assumeTrue(supportsMetaInfoVerification());
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CheckpointableKeyedStateBackend<Integer> backend = null;
    try {
        backend = createKeyedBackend(IntSerializer.INSTANCE, env);
        TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
        // make sure that we are in fact using the KryoSerializer
        assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
        ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id", pojoType);
        ValueState<TestPojo> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        // ============== create snapshot - no Kryo registration or specific / default
        // serializers ==============
        // make some more modifications
        backend.setCurrentKey(1);
        state.update(new TestPojo("u1", 1));
        backend.setCurrentKey(2);
        state.update(new TestPojo("u2", 2));
        KeyedStateHandle snapshot = runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        backend.dispose();
        // ========== restore snapshot - should use specific serializer (ONLY SERIALIZATION)
        // ==========
        env.getExecutionConfig().registerTypeWithKryoSerializer(TestPojo.class, CustomKryoTestSerializer.class);
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot, env);
        // re-initialize to ensure that we create the KryoSerializer from scratch, otherwise
        // initializeSerializerUnlessSet would not pick up our new config
        kvId = new ValueStateDescriptor<>("id", pojoType);
        state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        backend.setCurrentKey(1);
        // update to test state backends that eagerly serialize, such as RocksDB
        state.update(new TestPojo("u1", 11));
        KeyedStateHandle snapshot2 = runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        snapshot.discardState();
        backend.dispose();
        // ========= restore snapshot - should use specific serializer (FAIL ON DESERIALIZATION)
        // =========
        env.getExecutionConfig().registerTypeWithKryoSerializer(TestPojo.class, CustomKryoTestSerializer.class);
        // on the second restore, since the custom serializer will be used for
        // deserialization, we expect the deliberate failure to be thrown
        expectedException.expect(anyOf(isA(ExpectedKryoTestException.class), Matchers.<Throwable>hasProperty("cause", isA(ExpectedKryoTestException.class))));
        // state backends that eagerly deserializes (such as the memory state backend) will fail
        // here
        backend = restoreKeyedBackend(IntSerializer.INSTANCE, snapshot2, env);
        state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);
        backend.setCurrentKey(1);
        // state backends that lazily deserializes (such as RocksDB) will fail here
        state.value();
    } finally {
        // ensure that native resources are also released in case of exception
        if (backend != null) {
            backend.dispose();
        }
    }
}