Examples with TypeSerializerSnapshot org.apache.flink.api.common.typeutils.TypeSerializerSnapshot used on opensource projects

Example 1 with TypeSerializerSnapshot

use of org.apache.flink.api.common.typeutils.TypeSerializerSnapshot in project flink by apache.

the class KryoSerializerCompatibilityTest method testMigrationStrategyForDifferentRegistrationOrder.

/**
 * Tests that after reconfiguration, registration ids are reconfigured to remain the same as the
 * preceding KryoSerializer.
 */
@Test
public void testMigrationStrategyForDifferentRegistrationOrder() throws Exception {
    ExecutionConfig executionConfig = new ExecutionConfig();
    executionConfig.registerKryoType(TestClassA.class);
    executionConfig.registerKryoType(TestClassB.class);
    KryoSerializer<TestClass> kryoSerializer = new KryoSerializer<>(TestClass.class, executionConfig);
    // get original registration ids
    int testClassId = kryoSerializer.getKryo().getRegistration(TestClass.class).getId();
    int testClassAId = kryoSerializer.getKryo().getRegistration(TestClassA.class).getId();
    int testClassBId = kryoSerializer.getKryo().getRegistration(TestClassB.class).getId();
    // snapshot configuration and serialize to bytes
    TypeSerializerSnapshot kryoSerializerConfigSnapshot = kryoSerializer.snapshotConfiguration();
    byte[] serializedConfig;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(new DataOutputViewStreamWrapper(out), kryoSerializerConfigSnapshot, kryoSerializer);
        serializedConfig = out.toByteArray();
    }
    // use new config and instantiate new KryoSerializer
    executionConfig = new ExecutionConfig();
    // test with B registered before A
    executionConfig.registerKryoType(TestClassB.class);
    executionConfig.registerKryoType(TestClassA.class);
    kryoSerializer = new KryoSerializer<>(TestClass.class, executionConfig);
    // read configuration from bytes
    try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
        kryoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), kryoSerializer);
    }
    // reconfigure - check reconfiguration result and that registration id remains the same
    @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestClass> compatResult = kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializer);
    assertTrue(compatResult.isCompatibleWithReconfiguredSerializer());
    kryoSerializer = (KryoSerializer<TestClass>) compatResult.getReconfiguredSerializer();
    assertEquals(testClassId, kryoSerializer.getKryo().getRegistration(TestClass.class).getId());
    assertEquals(testClassAId, kryoSerializer.getKryo().getRegistration(TestClassA.class).getId());
    assertEquals(testClassBId, kryoSerializer.getKryo().getRegistration(TestClassB.class).getId());
}

Example 2 with TypeSerializerSnapshot

use of org.apache.flink.api.common.typeutils.TypeSerializerSnapshot in project flink by apache.

the class EnumSerializerCompatibilityTest method checkCompatibility.

@SuppressWarnings("unchecked")
private static TypeSerializerSchemaCompatibility checkCompatibility(String enumSourceA, String enumSourceB) throws IOException, ClassNotFoundException {
    ClassLoader classLoader = ClassLoaderUtils.compileAndLoadJava(temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceA);
    EnumSerializer enumSerializer = new EnumSerializer(classLoader.loadClass(ENUM_NAME));
    TypeSerializerSnapshot snapshot = enumSerializer.snapshotConfiguration();
    byte[] snapshotBytes;
    try (ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
        DataOutputViewStreamWrapper outputViewStreamWrapper = new DataOutputViewStreamWrapper(outBuffer)) {
        TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(outputViewStreamWrapper, snapshot, enumSerializer);
        snapshotBytes = outBuffer.toByteArray();
    }
    ClassLoader classLoader2 = ClassLoaderUtils.compileAndLoadJava(temporaryFolder.newFolder(), ENUM_NAME + ".java", enumSourceB);
    TypeSerializerSnapshot restoredSnapshot;
    try (ByteArrayInputStream inBuffer = new ByteArrayInputStream(snapshotBytes);
        DataInputViewStreamWrapper inputViewStreamWrapper = new DataInputViewStreamWrapper(inBuffer)) {
        restoredSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(inputViewStreamWrapper, classLoader2, enumSerializer);
    }
    EnumSerializer enumSerializer2 = new EnumSerializer(classLoader2.loadClass(ENUM_NAME));
    return restoredSnapshot.resolveSchemaCompatibility(enumSerializer2);
}

Example 3 with TypeSerializerSnapshot

use of org.apache.flink.api.common.typeutils.TypeSerializerSnapshot in project flink by apache.

the class PojoSerializerTest method testReconfigureWithPreviouslyNonregisteredSubclasses.

/**
 * Tests that: - Previous Pojo serializer did not have registrations, and created cached
 * serializers for subclasses - On restore, it had those subclasses registered
 *
 * <p>In this case, after reconfiguration, the cache should be repopulated, and registrations
 * should also exist for the subclasses.
 *
 * <p>Note: the cache still needs to be repopulated because previous data of those subclasses
 * were written with the cached serializers. In this case, the repopulated cache has
 * reconfigured serializers for the subclasses so that previous written data can be read, but
 * the registered serializers for the subclasses do not necessarily need to be reconfigured
 * since they will only be used to write new data.
 */
@Test
public void testReconfigureWithPreviouslyNonregisteredSubclasses() throws Exception {
    // don't register any subclasses at first
    PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig());
    // create cached serializers for SubTestUserClassA and SubTestUserClassB
    pojoSerializer.getSubclassSerializer(SubTestUserClassA.class);
    pojoSerializer.getSubclassSerializer(SubTestUserClassB.class);
    // make sure serializers are in cache
    assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
    assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
    assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
    // make sure that registrations are empty
    assertTrue(pojoSerializer.getRegisteredClasses().isEmpty());
    assertEquals(0, pojoSerializer.getRegisteredSerializers().length);
    // snapshot configuration and serialize to bytes
    TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration();
    byte[] serializedConfig;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer);
        serializedConfig = out.toByteArray();
    }
    // instantiate new PojoSerializer, with new execution config that has the subclass
    // registrations
    ExecutionConfig newExecutionConfig = new ExecutionConfig();
    newExecutionConfig.registerPojoType(SubTestUserClassA.class);
    newExecutionConfig.registerPojoType(SubTestUserClassB.class);
    pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(newExecutionConfig);
    // read configuration from bytes
    try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
        pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer);
    }
    // reconfigure - check reconfiguration result and that
    // 1) subclass serializer cache is repopulated
    // 2) registrations also contain the now registered subclasses
    @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestUserClass> compatResult = pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer);
    assertTrue(compatResult.isCompatibleWithReconfiguredSerializer());
    assertThat(compatResult.getReconfiguredSerializer(), instanceOf(PojoSerializer.class));
    PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer();
    assertEquals(2, reconfiguredPojoSerializer.getSubclassSerializerCache().size());
    assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
    assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
    assertEquals(2, reconfiguredPojoSerializer.getRegisteredClasses().size());
    assertTrue(reconfiguredPojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassA.class));
    assertTrue(reconfiguredPojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassB.class));
}

Example 4 with TypeSerializerSnapshot

use of org.apache.flink.api.common.typeutils.TypeSerializerSnapshot in project beam by apache.

the class CoderTypeSerializerTest method testWriteAndReadConfigSnapshot.

private void testWriteAndReadConfigSnapshot(Coder<String> coder) throws IOException {
    CoderTypeSerializer<String> serializer = new CoderTypeSerializer<>(coder, new SerializablePipelineOptions(PipelineOptionsFactory.create()));
    TypeSerializerSnapshot writtenSnapshot = serializer.snapshotConfiguration();
    ComparatorTestBase.TestOutputView outView = new ComparatorTestBase.TestOutputView();
    writtenSnapshot.writeSnapshot(outView);
    TypeSerializerSnapshot readSnapshot = new CoderTypeSerializer.LegacySnapshot();
    readSnapshot.readSnapshot(writtenSnapshot.getCurrentVersion(), outView.getInputView(), getClass().getClassLoader());
    assertThat(readSnapshot.restoreSerializer(), is(serializer));
}

Example 5 with TypeSerializerSnapshot

use of org.apache.flink.api.common.typeutils.TypeSerializerSnapshot in project flink by apache.

the class PojoSerializerSnapshotData method createFrom.

/**
 * Creates a {@link PojoSerializerSnapshotData} from existing snapshotted configuration of a
 * {@link PojoSerializer}.
 */
static <T> PojoSerializerSnapshotData<T> createFrom(Class<T> pojoClass, Field[] fields, TypeSerializerSnapshot<?>[] existingFieldSerializerSnapshots, LinkedHashMap<Class<?>, TypeSerializerSnapshot<?>> existingRegisteredSubclassSerializerSnapshots, Map<Class<?>, TypeSerializerSnapshot<?>> existingNonRegisteredSubclassSerializerSnapshots) {
    final LinkedOptionalMap<Field, TypeSerializerSnapshot<?>> fieldSerializerSnapshots = new LinkedOptionalMap<>(fields.length);
    for (int i = 0; i < fields.length; i++) {
        Field field = fields[i];
        String fieldName = (field == null) ? getDummyNameForMissingField(i) : field.getName();
        fieldSerializerSnapshots.put(fieldName, field, existingFieldSerializerSnapshots[i]);
    }
    return new PojoSerializerSnapshotData<>(pojoClass, fieldSerializerSnapshots, optionalMapOf(existingRegisteredSubclassSerializerSnapshots, Class::getName), optionalMapOf(existingNonRegisteredSubclassSerializerSnapshots, Class::getName));
}