Examples with DataOutputSerializer org.apache.flink.core.memory.DataOutputSerializer used on opensource projects

Example 6 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class AvroSerializerSnapshotTest method writeCurrentVersionSnapshot.

/**
 * Creates a new serializer snapshot for the current version. Use this before bumping the
 * snapshot version and also add the version (before bumping) to {@link #PAST_VERSIONS}.
 */
@Ignore
@Test
public void writeCurrentVersionSnapshot() throws IOException {
    AvroSerializer<GenericRecord> serializer = new AvroSerializer<>(GenericRecord.class, Address.getClassSchema());
    DataOutputSerializer out = new DataOutputSerializer(1024);
    TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(out, serializer.snapshotConfiguration(), serializer);
    Path snapshotPath = getSerializerSnapshotFilePath(new AvroSerializerSnapshot<>().getCurrentVersion());
    Files.write(snapshotPath, out.getCopyOfBuffer());
}

Example 7 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class AvroSerializerConcurrencyTest method testConcurrentUseOfSerializer.

@Test
public void testConcurrentUseOfSerializer() throws Exception {
    final AvroSerializer<String> serializer = new AvroSerializer<>(String.class);
    final BlockerSync sync = new BlockerSync();
    final DataOutputView regularOut = new DataOutputSerializer(32);
    final DataOutputView lockingOut = new LockingView(sync);
    // this thread serializes and gets stuck there
    final CheckedThread thread = new CheckedThread("serializer") {

        @Override
        public void go() throws Exception {
            serializer.serialize("a value", lockingOut);
        }
    };
    thread.start();
    sync.awaitBlocker();
    // this should fail with an exception
    try {
        serializer.serialize("value", regularOut);
        fail("should have failed with an exception");
    } catch (IllegalStateException e) {
    // expected
    } finally {
        // release the thread that serializes
        sync.releaseBlocker();
    }
    // this propagates exceptions from the spawned thread
    thread.sync();
}

Example 8 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class TypeSerializerUpgradeTestBase method generateTestSetupFiles.

/**
 * Execute this test to generate test files. Remember to be using the correct branch when
 * generating the test files, e.g. to generate test files for {@link FlinkVersion#v1_8}, you
 * should be under the release-1.8 branch.
 */
@Test
@Ignore
public void generateTestSetupFiles() throws Exception {
    Files.createDirectories(getSerializerSnapshotFilePath().getParent());
    try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(testSpecification.setup.setupClassloader)) {
        TypeSerializer<PreviousElementT> priorSerializer = testSpecification.setup.createPriorSerializer();
        // first, use the serializer to write test data
        // NOTE: it is important that we write test data first, because some serializers'
        // configuration
        // mutates only after being used for serialization (e.g. dynamic type
        // registrations for Pojo / Kryo)
        DataOutputSerializer testDataOut = new DataOutputSerializer(INITIAL_OUTPUT_BUFFER_SIZE);
        priorSerializer.serialize(testSpecification.setup.createTestData(), testDataOut);
        writeContentsTo(getGenerateDataFilePath(), testDataOut.getCopyOfBuffer());
        // ... then write the serializer snapshot
        DataOutputSerializer serializerSnapshotOut = new DataOutputSerializer(INITIAL_OUTPUT_BUFFER_SIZE);
        writeSerializerSnapshot(serializerSnapshotOut, priorSerializer, CURRENT_VERSION);
        writeContentsTo(getGenerateSerializerSnapshotFilePath(), serializerSnapshotOut.getCopyOfBuffer());
    }
}

Example 9 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class TypeSerializerSnapshotTest method testSerializerDeserializationFailure.

@Test
public void testSerializerDeserializationFailure() throws Exception {
    TestSerializer ser = new TestSerializer();
    TypeSerializerConfigSnapshot<Object> snap = (TypeSerializerConfigSnapshot<Object>) ser.snapshotConfiguration();
    snap.setPriorSerializer(ser);
    DataOutputSerializer out = new DataOutputSerializer(64);
    TypeSerializerSnapshot.writeVersionedSnapshot(out, snap);
    TypeSerializerSnapshot<Object> readBack = TypeSerializerSnapshot.readVersionedSnapshot(new DataInputDeserializer(out.getCopyOfBuffer()), getClass().getClassLoader());
    assertNotNull(readBack);
    try {
        readBack.restoreSerializer();
        fail("expected exception");
    } catch (IllegalStateException e) {
    // expected
    }
    ((TypeSerializerConfigSnapshot<Object>) readBack).setPriorSerializer(new UnloadableDummyTypeSerializer<>(new byte[0]));
    try {
        readBack.restoreSerializer();
        fail("expected exception");
    } catch (IllegalStateException e) {
    // expected
    }
}

Example 10 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class CompositeTypeSerializerSnapshotTest method snapshotCompositeSerializerAndGetSchemaCompatibilityAfterRestore.

private TypeSerializerSchemaCompatibility<String> snapshotCompositeSerializerAndGetSchemaCompatibilityAfterRestore(TypeSerializer<?>[] initialNestedSerializers, TypeSerializer<?>[] newNestedSerializer, OuterSchemaCompatibility mockOuterSchemaCompatibilityResult) throws IOException {
    TestCompositeTypeSerializer testSerializer = new TestCompositeTypeSerializer(initialNestedSerializers);
    TypeSerializerSnapshot<String> testSerializerSnapshot = testSerializer.snapshotConfiguration();
    DataOutputSerializer out = new DataOutputSerializer(128);
    TypeSerializerSnapshot.writeVersionedSnapshot(out, testSerializerSnapshot);
    DataInputDeserializer in = new DataInputDeserializer(out.getCopyOfBuffer());
    testSerializerSnapshot = TypeSerializerSnapshot.readVersionedSnapshot(in, Thread.currentThread().getContextClassLoader());
    TestCompositeTypeSerializer newTestSerializer = new TestCompositeTypeSerializer(mockOuterSchemaCompatibilityResult, newNestedSerializer);
    return testSerializerSnapshot.resolveSchemaCompatibility(newTestSerializer);
}