Examples with DataInputDeserializer org.apache.flink.core.memory.DataInputDeserializer used on opensource projects

Example 31 with DataInputDeserializer

use of org.apache.flink.core.memory.DataInputDeserializer in project flink by apache.

the class CompositeKeySerializationUtilsTest method testKeySerializationAndDeserialization.

@Test
public void testKeySerializationAndDeserialization() throws Exception {
    final DataOutputSerializer outputView = new DataOutputSerializer(8);
    final DataInputDeserializer inputView = new DataInputDeserializer();
    // test for key
    for (int orgKey = 0; orgKey < 100; ++orgKey) {
        outputView.clear();
        CompositeKeySerializationUtils.writeKey(orgKey, IntSerializer.INSTANCE, outputView, false);
        inputView.setBuffer(outputView.getCopyOfBuffer());
        int deserializedKey = CompositeKeySerializationUtils.readKey(IntSerializer.INSTANCE, inputView, false);
        Assert.assertEquals(orgKey, deserializedKey);
        CompositeKeySerializationUtils.writeKey(orgKey, IntSerializer.INSTANCE, outputView, true);
        inputView.setBuffer(outputView.getCopyOfBuffer());
        deserializedKey = CompositeKeySerializationUtils.readKey(IntSerializer.INSTANCE, inputView, true);
        Assert.assertEquals(orgKey, deserializedKey);
    }
}

Example 32 with DataInputDeserializer

use of org.apache.flink.core.memory.DataInputDeserializer in project flink by apache.

the class CompositeKeySerializationUtilsTest method testNamespaceSerializationAndDeserialization.

@Test
public void testNamespaceSerializationAndDeserialization() throws Exception {
    final DataOutputSerializer outputView = new DataOutputSerializer(8);
    final DataInputDeserializer inputView = new DataInputDeserializer();
    for (int orgNamespace = 0; orgNamespace < 100; ++orgNamespace) {
        outputView.clear();
        CompositeKeySerializationUtils.writeNameSpace(orgNamespace, IntSerializer.INSTANCE, outputView, false);
        inputView.setBuffer(outputView.getCopyOfBuffer());
        int deserializedNamepsace = CompositeKeySerializationUtils.readNamespace(IntSerializer.INSTANCE, inputView, false);
        Assert.assertEquals(orgNamespace, deserializedNamepsace);
        CompositeKeySerializationUtils.writeNameSpace(orgNamespace, IntSerializer.INSTANCE, outputView, true);
        inputView.setBuffer(outputView.getCopyOfBuffer());
        deserializedNamepsace = CompositeKeySerializationUtils.readNamespace(IntSerializer.INSTANCE, inputView, true);
        Assert.assertEquals(orgNamespace, deserializedNamepsace);
    }
}

Example 33 with DataInputDeserializer

use of org.apache.flink.core.memory.DataInputDeserializer in project flink by apache.

the class EventSerializer method fromSerializedEvent.

public static AbstractEvent fromSerializedEvent(ByteBuffer buffer, ClassLoader classLoader) throws IOException {
    if (buffer.remaining() < 4) {
        throw new IOException("Incomplete event");
    }
    final ByteOrder bufferOrder = buffer.order();
    buffer.order(ByteOrder.BIG_ENDIAN);
    try {
        final int type = buffer.getInt();
        if (type == END_OF_PARTITION_EVENT) {
            return EndOfPartitionEvent.INSTANCE;
        } else if (type == CHECKPOINT_BARRIER_EVENT) {
            return deserializeCheckpointBarrier(buffer);
        } else if (type == END_OF_SUPERSTEP_EVENT) {
            return EndOfSuperstepEvent.INSTANCE;
        } else if (type == END_OF_CHANNEL_STATE_EVENT) {
            return EndOfChannelStateEvent.INSTANCE;
        } else if (type == END_OF_USER_RECORDS_EVENT) {
            return new EndOfData(StopMode.values()[buffer.get()]);
        } else if (type == CANCEL_CHECKPOINT_MARKER_EVENT) {
            long id = buffer.getLong();
            return new CancelCheckpointMarker(id);
        } else if (type == ANNOUNCEMENT_EVENT) {
            int sequenceNumber = buffer.getInt();
            AbstractEvent announcedEvent = fromSerializedEvent(buffer, classLoader);
            return new EventAnnouncement(announcedEvent, sequenceNumber);
        } else if (type == VIRTUAL_CHANNEL_SELECTOR_EVENT) {
            return new SubtaskConnectionDescriptor(buffer.getInt(), buffer.getInt());
        } else if (type == OTHER_EVENT) {
            try {
                final DataInputDeserializer deserializer = new DataInputDeserializer(buffer);
                final String className = deserializer.readUTF();
                final Class<? extends AbstractEvent> clazz;
                try {
                    clazz = classLoader.loadClass(className).asSubclass(AbstractEvent.class);
                } catch (ClassNotFoundException e) {
                    throw new IOException("Could not load event class '" + className + "'.", e);
                } catch (ClassCastException e) {
                    throw new IOException("The class '" + className + "' is not a valid subclass of '" + AbstractEvent.class.getName() + "'.", e);
                }
                final AbstractEvent event = InstantiationUtil.instantiate(clazz, AbstractEvent.class);
                event.read(deserializer);
                return event;
            } catch (Exception e) {
                throw new IOException("Error while deserializing or instantiating event.", e);
            }
        } else {
            throw new IOException("Corrupt byte stream for event");
        }
    } finally {
        buffer.order(bufferOrder);
    }
}

Example 34 with DataInputDeserializer

use of org.apache.flink.core.memory.DataInputDeserializer in project flink by apache.

the class FileSourceSplitSerializer method deserializeV1.

private static FileSourceSplit deserializeV1(byte[] serialized) throws IOException {
    final DataInputDeserializer in = new DataInputDeserializer(serialized);
    final String id = in.readUTF();
    final Path path = new Path();
    path.read(in);
    final long offset = in.readLong();
    final long len = in.readLong();
    final long modificationTime = in.readLong();
    final long fileSize = in.readLong();
    final String[] hosts = readStringArray(in);
    final CheckpointedPosition readerPosition = in.readBoolean() ? new CheckpointedPosition(in.readLong(), in.readLong()) : null;
    // instantiate a new split and cache the serialized form
    return new FileSourceSplit(id, path, offset, len, modificationTime, fileSize, hosts, readerPosition, serialized);
}

Example 35 with DataInputDeserializer

use of org.apache.flink.core.memory.DataInputDeserializer in project flink by apache.

the class TestManagedSinkCommittableSerializer method deserialize.

@Override
public TestManagedCommittable deserialize(int version, byte[] serialized) throws IOException {
    if (version == VERSION) {
        final DataInputDeserializer in = new DataInputDeserializer(serialized);
        int newFileSize = in.readInt();
        Map<CatalogPartitionSpec, List<RowData>> toCommit = new HashMap<>(newFileSize);
        for (int i = 0; i < newFileSize; i++) {
            CatalogPartitionSpec partitionSpec = deserializePartitionSpec(in);
            List<RowData> elements = deserializeRowDataElements(in);
            toCommit.put(partitionSpec, elements);
        }
        int cleanupFileSize = in.readInt();
        Map<CatalogPartitionSpec, Set<Path>> toCleanup = new HashMap<>(cleanupFileSize);
        for (int i = 0; i < cleanupFileSize; i++) {
            CatalogPartitionSpec partitionSpec = deserializePartitionSpec(in);
            Set<Path> paths = deserializePaths(in);
            toCleanup.put(partitionSpec, paths);
        }
        return new TestManagedCommittable(toCommit, toCleanup);
    }
    throw new IOException(String.format("Unknown version %d", version));
}