Examples with DataInputDeserializer org.apache.flink.core.memory.DataInputDeserializer used on opensource projects

Example 21 with DataInputDeserializer

use of org.apache.flink.core.memory.DataInputDeserializer in project flink by apache.

the class NullableSerializer method checkIfNullSupported.

/**
 * This method checks if {@code serializer} supports {@code null} value.
 *
 * @param serializer serializer to check
 */
public static <T> boolean checkIfNullSupported(@Nonnull TypeSerializer<T> serializer) {
    int length = serializer.getLength() > 0 ? serializer.getLength() : 1;
    DataOutputSerializer dos = new DataOutputSerializer(length);
    try {
        serializer.serialize(null, dos);
    } catch (IOException | RuntimeException e) {
        return false;
    }
    checkArgument(serializer.getLength() < 0 || serializer.getLength() == dos.getCopyOfBuffer().length, "The serialized form of the null value should have the same length " + "as any other if the length is fixed in the serializer");
    DataInputDeserializer dis = new DataInputDeserializer(dos.getSharedBuffer());
    try {
        checkArgument(serializer.deserialize(dis) == null);
    } catch (IOException e) {
        throw new RuntimeException(String.format("Unexpected failure to deserialize just serialized null value with %s", serializer.getClass().getName()), e);
    }
    checkArgument(serializer.copy(null) == null, "Serializer %s has to be able properly copy null value if it can serialize it", serializer.getClass().getName());
    return true;
}

Example 22 with DataInputDeserializer

use of org.apache.flink.core.memory.DataInputDeserializer in project flink by apache.

the class PassThroughPythonStreamGroupWindowAggregateOperator method processPythonElement.

public void processPythonElement(byte[] inputBytes) {
    try {
        RowData input = udfInputTypeSerializer.deserialize(new DataInputDeserializer(inputBytes));
        if (input.getByte(0) == NORMAL_RECORD) {
            // normal data
            RowData inputRow = input.getRow(1, inputType.getFieldCount());
            BinaryRowData key = groupKeyProjection.apply(inputRow).copy();
            Map<TimeWindow, List<RowData>> curKeyWindowAccumulateData = windowAccumulateData.computeIfAbsent(key.getString(0).toString(), k -> new HashMap<>());
            Map<TimeWindow, List<RowData>> curKeyWindowRetractData = windowRetractData.computeIfAbsent(key.getString(0).toString(), k -> new HashMap<>());
            long watermark = input.getLong(3);
            // advance watermark
            mockPythonInternalService.advanceWatermark(watermark);
            // get timestamp
            long timestamp = inputRow.getLong(inputTimeFieldIndex);
            timestamp = TimeWindowUtil.toUtcTimestampMills(timestamp, shiftTimeZone);
            Collection<TimeWindow> elementWindows = windowAssigner.assignWindows(inputRow, timestamp);
            for (TimeWindow window : elementWindows) {
                if (RowDataUtil.isAccumulateMsg(inputRow)) {
                    List<RowData> currentWindowDatas = curKeyWindowAccumulateData.computeIfAbsent(window, k -> new LinkedList<>());
                    currentWindowDatas.add(inputRow);
                } else {
                    List<RowData> currentWindowDatas = curKeyWindowRetractData.computeIfAbsent(window, k -> new LinkedList<>());
                    currentWindowDatas.add(inputRow);
                }
            }
            List<TimeWindow> actualWindows = new ArrayList<>(elementWindows.size());
            for (TimeWindow window : elementWindows) {
                if (!isWindowLate(window)) {
                    actualWindows.add(window);
                }
            }
            for (TimeWindow window : actualWindows) {
                boolean triggerResult = onElement(key, window);
                if (triggerResult) {
                    triggerWindowProcess(key, window);
                }
                // register a clean up timer for the window
                registerCleanupTimer(key, window);
            }
        } else {
            RowData timerData = input.getRow(4, 3);
            long timestamp = input.getLong(2);
            RowData key = timerData.getRow(1, getKeyType().getFieldCount());
            byte[] encodedNamespace = timerData.getBinary(2);
            bais.setBuffer(encodedNamespace, 0, encodedNamespace.length);
            TimeWindow window = windowSerializer.deserialize(baisWrapper);
            if (timestamp == window.maxTimestamp()) {
                triggerWindowProcess(key, window);
            }
            cleanWindowIfNeeded(key, window, timestamp);
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 23 with DataInputDeserializer

use of org.apache.flink.core.memory.DataInputDeserializer in project flink by apache.

the class SimpleVersionedSerializationTest method testSerializeEmpty.

@Test
public void testSerializeEmpty() throws IOException {
    final String testString = "beeeep!";
    SimpleVersionedSerializer<String> emptySerializer = new SimpleVersionedSerializer<String>() {

        @Override
        public int getVersion() {
            return 42;
        }

        @Override
        public byte[] serialize(String obj) throws IOException {
            return new byte[0];
        }

        @Override
        public String deserialize(int version, byte[] serialized) throws IOException {
            assertEquals(42, version);
            assertEquals(0, serialized.length);
            return testString;
        }
    };
    final DataOutputSerializer out = new DataOutputSerializer(32);
    SimpleVersionedSerialization.writeVersionAndSerialize(emptySerializer, "abc", out);
    final byte[] outBytes = out.getCopyOfBuffer();
    final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(emptySerializer, "abc");
    assertArrayEquals(bytes, outBytes);
    final DataInputDeserializer in = new DataInputDeserializer(bytes);
    final String deserialized = SimpleVersionedSerialization.readVersionAndDeSerialize(emptySerializer, in);
    final String deserializedFromBytes = SimpleVersionedSerialization.readVersionAndDeSerialize(emptySerializer, outBytes);
    assertEquals(testString, deserialized);
    assertEquals(testString, deserializedFromBytes);
}

Example 24 with DataInputDeserializer

use of org.apache.flink.core.memory.DataInputDeserializer in project flink by apache.

the class AvroSerializerSnapshotTest method roundTrip.

// ---------------------------------------------------------------------------------------------------------------
// Utils
// ---------------------------------------------------------------------------------------------------------------
/**
 * Serialize an (avro)TypeSerializerSnapshot and deserialize it.
 */
private static <T> AvroSerializerSnapshot<T> roundTrip(TypeSerializerSnapshot<T> original) throws IOException {
    // writeSnapshot();
    DataOutputSerializer out = new DataOutputSerializer(1024);
    original.writeSnapshot(out);
    // init
    AvroSerializerSnapshot<T> restored = new AvroSerializerSnapshot<>();
    // readSnapshot();
    DataInputView in = new DataInputDeserializer(out.wrapAsByteBuffer());
    restored.readSnapshot(restored.getCurrentVersion(), in, original.getClass().getClassLoader());
    return restored;
}

Example 25 with DataInputDeserializer

use of org.apache.flink.core.memory.DataInputDeserializer in project flink by apache.

the class SerializedCompositeKeyBuilderTest method testSetKeyNamespaceUserKeyInternal.

private <K, N, U> void testSetKeyNamespaceUserKeyInternal(TypeSerializer<K> keySerializer, TypeSerializer<N> namespaceSerializer, TypeSerializer<U> userKeySerializer, Collection<K> testKeys, Collection<N> testNamespaces, Collection<U> testUserKeys, int maxParallelism, BuildKeyAndNamespaceType buildKeyAndNamespaceType) throws IOException {
    final int prefixBytes = maxParallelism > Byte.MAX_VALUE ? 2 : 1;
    SerializedCompositeKeyBuilder<K> keyBuilder = createRocksDBSerializedCompositeKeyBuilder(keySerializer, prefixBytes);
    final DataInputDeserializer deserializer = new DataInputDeserializer();
    final boolean ambiguousPossible = keyBuilder.isAmbiguousCompositeKeyPossible(namespaceSerializer);
    for (K testKey : testKeys) {
        int keyGroup = setKeyAndReturnKeyGroup(keyBuilder, testKey, maxParallelism);
        for (N testNamespace : testNamespaces) {
            if (buildKeyAndNamespaceType == BuildKeyAndNamespaceType.SET_AND_BUILD) {
                keyBuilder.setNamespace(testNamespace, namespaceSerializer);
            }
            for (U testUserKey : testUserKeys) {
                final byte[] compositeBytes;
                if (buildKeyAndNamespaceType == BuildKeyAndNamespaceType.BUILD) {
                    compositeBytes = keyBuilder.buildCompositeKeyNamesSpaceUserKey(testNamespace, namespaceSerializer, testUserKey, userKeySerializer);
                } else {
                    compositeBytes = keyBuilder.buildCompositeKeyUserKey(testUserKey, userKeySerializer);
                }
                deserializer.setBuffer(compositeBytes);
                assertKeyGroupKeyNamespaceUserKeyBytes(testKey, keyGroup, prefixBytes, keySerializer, testNamespace, namespaceSerializer, testUserKey, userKeySerializer, deserializer, ambiguousPossible);
                Assert.assertEquals(0, deserializer.available());
            }
        }
    }
}