Examples with DataOutputSerializer org.apache.flink.core.memory.DataOutputSerializer used on opensource projects

Example 11 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class CompositeTypeSerializerSnapshotTest method testRestoreCompositeTypeSerializer.

// ------------------------------------------------------------------------------------------------
// Scope: tests CompositeTypeSerializerSnapshot#restoreSerializer
// ------------------------------------------------------------------------------------------------
@Test
public void testRestoreCompositeTypeSerializer() throws IOException {
    // the target compatibilities of the nested serializers doesn't matter,
    // because we're only testing the restore serializer
    TypeSerializer<?>[] testNestedSerializers = { new NestedSerializer(TargetCompatibility.COMPATIBLE_AS_IS), new NestedSerializer(TargetCompatibility.INCOMPATIBLE), new NestedSerializer(TargetCompatibility.COMPATIBLE_AFTER_MIGRATION) };
    TestCompositeTypeSerializer testSerializer = new TestCompositeTypeSerializer(testNestedSerializers);
    TypeSerializerSnapshot<String> testSerializerSnapshot = testSerializer.snapshotConfiguration();
    DataOutputSerializer out = new DataOutputSerializer(128);
    TypeSerializerSnapshot.writeVersionedSnapshot(out, testSerializerSnapshot);
    DataInputDeserializer in = new DataInputDeserializer(out.getCopyOfBuffer());
    testSerializerSnapshot = TypeSerializerSnapshot.readVersionedSnapshot(in, Thread.currentThread().getContextClassLoader());
    // now, restore the composite type serializer;
    // the restored nested serializer should be a RestoredNestedSerializer
    testSerializer = (TestCompositeTypeSerializer) testSerializerSnapshot.restoreSerializer();
    Assert.assertTrue(testSerializer.getNestedSerializers()[0].getClass() == RestoredNestedSerializer.class);
    Assert.assertTrue(testSerializer.getNestedSerializers()[1].getClass() == RestoredNestedSerializer.class);
    Assert.assertTrue(testSerializer.getNestedSerializers()[2].getClass() == RestoredNestedSerializer.class);
}

Example 12 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class SerializableHadoopConfigWrapper method writeObject.

// ------------------------------------------------------------------------
private void writeObject(ObjectOutputStream out) throws IOException {
    out.defaultWriteObject();
    // we write the Hadoop config through a separate serializer to avoid cryptic exceptions when
    // it
    // corrupts the serialization stream
    final DataOutputSerializer ser = new DataOutputSerializer(256);
    hadoopConfig.write(ser);
    out.writeInt(ser.length());
    out.write(ser.getSharedBuffer(), 0, ser.length());
}

Example 13 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class TypeInformationSerializationSchema method serialize.

@Override
public byte[] serialize(T element) {
    if (dos == null) {
        dos = new DataOutputSerializer(16);
    }
    try {
        serializer.serialize(element, dos);
    } catch (IOException e) {
        throw new RuntimeException("Unable to serialize record", e);
    }
    byte[] ret = dos.getCopyOfBuffer();
    dos.clear();
    return ret;
}

Example 14 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class PassThroughPythonStreamGroupWindowAggregateOperator method cleanWindowIfNeeded.

private void cleanWindowIfNeeded(RowData key, TimeWindow window, long currentTime) throws IOException {
    if (currentTime == cleanupTime(window)) {
        // 1. delete state
        // only output first value in group window.
        DataOutputSerializer output = new DataOutputSerializer(1);
        RowData windowProperty = windowExtractor.apply(window);
        windowAggResult.replace(GenericRowData.of(StringData.fromString("state_cleanup_triggered: " + key.getString(0).toString() + " : " + window)), GenericRowData.of(0L));
        reuseJoinedRow.replace(windowAggResult, windowProperty);
        reusePythonRowData.setField(1, reuseJoinedRow);
        udfOutputTypeSerializer.serialize(reusePythonRowData, output);
        resultBuffer.add(output.getCopyOfBuffer());
        // 2. delete window timer
        if (windowAssigner.isEventTime()) {
            deleteEventTimeTimer(key, window);
        } else {
            deleteProcessingTimeTimer(key, window);
        }
    }
}

Example 15 with DataOutputSerializer

use of org.apache.flink.core.memory.DataOutputSerializer in project flink by apache.

the class PassThroughPythonStreamGroupWindowAggregateOperator method triggerWindowProcess.

private void triggerWindowProcess(RowData key, TimeWindow window) throws Exception {
    DataOutputSerializer output = new DataOutputSerializer(1);
    Iterable<RowData> currentWindowAccumulateData = windowAccumulateData.get(key.getString(0).toString()).get(window);
    Iterable<RowData> currentWindowRetractData = windowRetractData.get(key.getString(0).toString()).get(window);
    if (currentWindowAccumulateData != null) {
        for (RowData accumulateData : currentWindowAccumulateData) {
            if (!hasRetractData(accumulateData, currentWindowRetractData)) {
                // only output first value in group window.
                RowData aggResult = aggExtracter.apply(accumulateData);
                RowData windowProperty = windowExtractor.apply(window);
                windowAggResult.replace(key, aggResult);
                reuseJoinedRow.replace(windowAggResult, windowProperty);
                reusePythonRowData.setField(1, reuseJoinedRow);
                udfOutputTypeSerializer.serialize(reusePythonRowData, output);
                resultBuffer.add(output.getCopyOfBuffer());
                break;
            }
        }
    }
}