Examples with TypeSerializerSnapshot org.apache.flink.api.common.typeutils.TypeSerializerSnapshot used on opensource projects

Example 6 with TypeSerializerSnapshot

use of org.apache.flink.api.common.typeutils.TypeSerializerSnapshot in project flink by apache.

the class PojoSerializerSnapshotData method createFrom.

// ---------------------------------------------------------------------------------------------
// Factory methods
// ---------------------------------------------------------------------------------------------
/**
 * Creates a {@link PojoSerializerSnapshotData} from configuration of a {@link PojoSerializer}.
 *
 * <p>This factory method is meant to be used in regular write paths, i.e. when taking a
 * snapshot of the {@link PojoSerializer}. All registered subclass classes, and non-registered
 * subclass classes are all present. Some POJO fields may be absent, if the originating {@link
 * PojoSerializer} was a restored one with already missing fields, and was never replaced by a
 * new {@link PojoSerializer} (i.e. because the serialized old data was never accessed).
 */
static <T> PojoSerializerSnapshotData<T> createFrom(Class<T> pojoClass, Field[] fields, TypeSerializer<?>[] fieldSerializers, LinkedHashMap<Class<?>, TypeSerializer<?>> registeredSubclassSerializers, Map<Class<?>, TypeSerializer<?>> nonRegisteredSubclassSerializers) {
    final LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots = new LinkedOptionalMap<>(fields.length);
    for (int i = 0; i < fields.length; i++) {
        Field field = fields[i];
        String fieldName = (field == null) ? getDummyNameForMissingField(i) : field.getName();
        fieldSerializerSnapshots.put(fieldName, field, TypeSerializerUtils.snapshotBackwardsCompatible(fieldSerializers[i]));
    }
    LinkedHashMap<Class<?>, TypeSerializerSnapshot<?>> registeredSubclassSerializerSnapshots = new LinkedHashMap<>(registeredSubclassSerializers.size());
    registeredSubclassSerializers.forEach((k, v) -> registeredSubclassSerializerSnapshots.put(k, TypeSerializerUtils.snapshotBackwardsCompatible(v)));
    Map<Class<?>, TypeSerializerSnapshot<?>> nonRegisteredSubclassSerializerSnapshots = new HashMap<>(nonRegisteredSubclassSerializers.size());
    nonRegisteredSubclassSerializers.forEach((k, v) -> nonRegisteredSubclassSerializerSnapshots.put(k, TypeSerializerUtils.snapshotBackwardsCompatible(v)));
    return new PojoSerializerSnapshotData<>(pojoClass, fieldSerializerSnapshots, optionalMapOf(registeredSubclassSerializerSnapshots, Class::getName), optionalMapOf(nonRegisteredSubclassSerializerSnapshots, Class::getName));
}

Example 7 with TypeSerializerSnapshot

use of org.apache.flink.api.common.typeutils.TypeSerializerSnapshot in project flink by apache.

the class PojoSerializerTest method testReconfigureRepopulateNonregisteredSubclassSerializerCache.

/**
 * Tests that reconfiguration repopulates previously cached subclass serializers.
 */
@Test
public void testReconfigureRepopulateNonregisteredSubclassSerializerCache() throws Exception {
    // don't register any subclasses
    PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig());
    // create cached serializers for SubTestUserClassA and SubTestUserClassB
    pojoSerializer.getSubclassSerializer(SubTestUserClassA.class);
    pojoSerializer.getSubclassSerializer(SubTestUserClassB.class);
    assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
    assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
    assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
    // snapshot configuration and serialize to bytes
    TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration();
    byte[] serializedConfig;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer);
        serializedConfig = out.toByteArray();
    }
    // instantiate new PojoSerializer
    pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig());
    // read configuration from bytes
    try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
        pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer);
    }
    // reconfigure - check reconfiguration result and that subclass serializer cache is
    // repopulated
    @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestUserClass> compatResult = pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer);
    assertTrue(compatResult.isCompatibleWithReconfiguredSerializer());
    assertThat(compatResult.getReconfiguredSerializer(), instanceOf(PojoSerializer.class));
    PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer();
    assertEquals(2, reconfiguredPojoSerializer.getSubclassSerializerCache().size());
    assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
    assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
}

Example 8 with TypeSerializerSnapshot

use of org.apache.flink.api.common.typeutils.TypeSerializerSnapshot in project flink by apache.

the class PojoSerializerTest method testReconfigureWithDifferentPojoType.

// --------------------------------------------------------------------------------------------
// Configuration snapshotting & reconfiguring tests
// --------------------------------------------------------------------------------------------
/**
 * Verifies that reconfiguring with a config snapshot of a preceding POJO serializer with
 * different POJO type will result in INCOMPATIBLE.
 */
@Test
public void testReconfigureWithDifferentPojoType() throws Exception {
    PojoSerializer<SubTestUserClassB> pojoSerializer1 = (PojoSerializer<SubTestUserClassB>) TypeExtractor.getForClass(SubTestUserClassB.class).createSerializer(new ExecutionConfig());
    // snapshot configuration and serialize to bytes
    TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer1.snapshotConfiguration();
    byte[] serializedConfig;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer1);
        serializedConfig = out.toByteArray();
    }
    PojoSerializer<SubTestUserClassA> pojoSerializer2 = (PojoSerializer<SubTestUserClassA>) TypeExtractor.getForClass(SubTestUserClassA.class).createSerializer(new ExecutionConfig());
    // read configuration again from bytes
    try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
        pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer2);
    }
    @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<SubTestUserClassA> compatResult = pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer2);
    assertTrue(compatResult.isIncompatible());
}

Example 9 with TypeSerializerSnapshot

use of org.apache.flink.api.common.typeutils.TypeSerializerSnapshot in project flink by apache.

the class PojoSerializerTest method testReconfigureDifferentSubclassRegistrationOrder.

/**
 * Tests that reconfiguration correctly reorders subclass registrations to their previous order.
 */
@Test
public void testReconfigureDifferentSubclassRegistrationOrder() throws Exception {
    ExecutionConfig executionConfig = new ExecutionConfig();
    executionConfig.registerPojoType(SubTestUserClassA.class);
    executionConfig.registerPojoType(SubTestUserClassB.class);
    PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(executionConfig);
    // get original registration ids
    int subClassATag = pojoSerializer.getRegisteredClasses().get(SubTestUserClassA.class);
    int subClassBTag = pojoSerializer.getRegisteredClasses().get(SubTestUserClassB.class);
    // snapshot configuration and serialize to bytes
    TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration();
    byte[] serializedConfig;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer);
        serializedConfig = out.toByteArray();
    }
    // use new config and instantiate new PojoSerializer
    executionConfig = new ExecutionConfig();
    executionConfig.registerPojoType(// test with B registered before A
    SubTestUserClassB.class);
    executionConfig.registerPojoType(SubTestUserClassA.class);
    pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(executionConfig);
    // read configuration from bytes
    try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
        pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer);
    }
    @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestUserClass> compatResult = pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer);
    assertTrue(compatResult.isCompatibleWithReconfiguredSerializer());
    assertThat(compatResult.getReconfiguredSerializer(), instanceOf(PojoSerializer.class));
    // reconfigure - check reconfiguration result and that registration ids remains the same
    // assertEquals(ReconfigureResult.COMPATIBLE,
    // pojoSerializer.reconfigure(pojoSerializerConfigSnapshot));
    PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer();
    assertEquals(subClassATag, reconfiguredPojoSerializer.getRegisteredClasses().get(SubTestUserClassA.class).intValue());
    assertEquals(subClassBTag, reconfiguredPojoSerializer.getRegisteredClasses().get(SubTestUserClassB.class).intValue());
}

Example 10 with TypeSerializerSnapshot

use of org.apache.flink.api.common.typeutils.TypeSerializerSnapshot in project flink by apache.

the class KryoSerializerCompatibilityTest method testMigrationStrategyWithDifferentKryoType.

/**
 * Verifies that reconfiguration result is INCOMPATIBLE if data type has changed.
 */
@Test
public void testMigrationStrategyWithDifferentKryoType() throws Exception {
    KryoSerializer<TestClassA> kryoSerializerForA = new KryoSerializer<>(TestClassA.class, new ExecutionConfig());
    // snapshot configuration and serialize to bytes
    TypeSerializerSnapshot kryoSerializerConfigSnapshot = kryoSerializerForA.snapshotConfiguration();
    byte[] serializedConfig;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(new DataOutputViewStreamWrapper(out), kryoSerializerConfigSnapshot, kryoSerializerForA);
        serializedConfig = out.toByteArray();
    }
    KryoSerializer<TestClassB> kryoSerializerForB = new KryoSerializer<>(TestClassB.class, new ExecutionConfig());
    // read configuration again from bytes
    try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
        kryoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), kryoSerializerForB);
    }
    @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestClassB> compatResult = kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializerForB);
    assertTrue(compatResult.isIncompatible());
}