Examples with DataInputDeserializer org.apache.flink.runtime.util.DataInputDeserializer used on opensource projects

Example 1 with DataInputDeserializer

use of org.apache.flink.runtime.util.DataInputDeserializer in project flink by apache.

the class SerializedCheckpointData method toDeque.

// ------------------------------------------------------------------------
//  De-Serialize from Checkpoint
// ------------------------------------------------------------------------
/**
	 * De-serializes an array of SerializedCheckpointData back into an ArrayDeque of element checkpoints.
	 * 
	 * @param data The data to be deserialized.
	 * @param serializer The serializer used to deserialize the data.
	 * @param <T> The type of the elements.
	 * @return An ArrayDeque of element checkpoints.
	 * 
	 * @throws IOException Thrown, if the serialization fails.
	 */
public static <T> ArrayDeque<Tuple2<Long, List<T>>> toDeque(SerializedCheckpointData[] data, TypeSerializer<T> serializer) throws IOException {
    ArrayDeque<Tuple2<Long, List<T>>> deque = new ArrayDeque<>(data.length);
    DataInputDeserializer deser = null;
    for (SerializedCheckpointData checkpoint : data) {
        byte[] serializedData = checkpoint.getSerializedData();
        if (deser == null) {
            deser = new DataInputDeserializer(serializedData, 0, serializedData.length);
        } else {
            deser.setBuffer(serializedData, 0, serializedData.length);
        }
        final List<T> ids = new ArrayList<>(checkpoint.getNumIds());
        final int numIds = checkpoint.getNumIds();
        for (int i = 0; i < numIds; i++) {
            ids.add(serializer.deserialize(deser));
        }
        deque.addLast(new Tuple2<Long, List<T>>(checkpoint.checkpointId, ids));
    }
    return deque;
}

Example 2 with DataInputDeserializer

use of org.apache.flink.runtime.util.DataInputDeserializer in project flink by apache.

the class AbstractMemStateSnapshot method deserialize.

@Override
@SuppressWarnings("unchecked")
public StateTable<K, N, SV> deserialize(String stateName, HeapKeyedStateBackend<K> stateBackend) throws IOException {
    final DataInputDeserializer inView = new DataInputDeserializer(data, 0, data.length);
    AbstractMigrationRestoreStrategy<K, N, SV> restoreStrategy = new AbstractMigrationRestoreStrategy<K, N, SV>(keySerializer, namespaceSerializer, stateSerializer) {

        @Override
        protected DataInputView openDataInputView() throws IOException {
            return inView;
        }
    };
    return restoreStrategy.deserialize(stateName, stateBackend);
}

Example 3 with DataInputDeserializer

use of org.apache.flink.runtime.util.DataInputDeserializer in project flink by apache.

the class EventSerializer method isEvent.

/**
	 * Identifies whether the given buffer encodes the given event.
	 *
	 * <p><strong>Pre-condition</strong>: This buffer must encode some event!</p>
	 *
	 * @param buffer the buffer to peak into
	 * @param eventClass the expected class of the event type
	 * @param classLoader the class loader to use for custom event classes
	 * @return whether the event class of the <tt>buffer</tt> matches the given <tt>eventClass</tt>
	 * @throws IOException
	 */
private static boolean isEvent(ByteBuffer buffer, Class<?> eventClass, ClassLoader classLoader) throws IOException {
    if (buffer.remaining() < 4) {
        throw new IOException("Incomplete event");
    }
    final int bufferPos = buffer.position();
    final ByteOrder bufferOrder = buffer.order();
    buffer.order(ByteOrder.BIG_ENDIAN);
    try {
        int type = buffer.getInt();
        switch(type) {
            case END_OF_PARTITION_EVENT:
                return eventClass.equals(EndOfPartitionEvent.class);
            case CHECKPOINT_BARRIER_EVENT:
                return eventClass.equals(CheckpointBarrier.class);
            case END_OF_SUPERSTEP_EVENT:
                return eventClass.equals(EndOfSuperstepEvent.class);
            case CANCEL_CHECKPOINT_MARKER_EVENT:
                return eventClass.equals(CancelCheckpointMarker.class);
            case OTHER_EVENT:
                try {
                    final DataInputDeserializer deserializer = new DataInputDeserializer(buffer);
                    final String className = deserializer.readUTF();
                    final Class<? extends AbstractEvent> clazz;
                    try {
                        clazz = classLoader.loadClass(className).asSubclass(AbstractEvent.class);
                    } catch (ClassNotFoundException e) {
                        throw new IOException("Could not load event class '" + className + "'.", e);
                    } catch (ClassCastException e) {
                        throw new IOException("The class '" + className + "' is not a valid subclass of '" + AbstractEvent.class.getName() + "'.", e);
                    }
                    return eventClass.equals(clazz);
                } catch (Exception e) {
                    throw new IOException("Error while deserializing or instantiating event.", e);
                }
            default:
                throw new IOException("Corrupt byte stream for event");
        }
    } finally {
        buffer.order(bufferOrder);
        // restore the original position in the buffer (recall: we only peak into it!)
        buffer.position(bufferPos);
    }
}

Example 4 with DataInputDeserializer

use of org.apache.flink.runtime.util.DataInputDeserializer in project flink by apache.

the class EventSerializer method fromSerializedEvent.

public static AbstractEvent fromSerializedEvent(ByteBuffer buffer, ClassLoader classLoader) throws IOException {
    if (buffer.remaining() < 4) {
        throw new IOException("Incomplete event");
    }
    final ByteOrder bufferOrder = buffer.order();
    buffer.order(ByteOrder.BIG_ENDIAN);
    try {
        int type = buffer.getInt();
        if (type == END_OF_PARTITION_EVENT) {
            return EndOfPartitionEvent.INSTANCE;
        } else if (type == CHECKPOINT_BARRIER_EVENT) {
            long id = buffer.getLong();
            long timestamp = buffer.getLong();
            CheckpointOptions checkpointOptions;
            int checkpointTypeOrdinal = buffer.getInt();
            Preconditions.checkElementIndex(type, CheckpointType.values().length, "Illegal CheckpointType ordinal");
            CheckpointType checkpointType = CheckpointType.values()[checkpointTypeOrdinal];
            if (checkpointType == CheckpointType.FULL_CHECKPOINT) {
                checkpointOptions = CheckpointOptions.forFullCheckpoint();
            } else if (checkpointType == CheckpointType.SAVEPOINT) {
                int len = buffer.getInt();
                byte[] bytes = new byte[len];
                buffer.get(bytes);
                String targetLocation = new String(bytes, STRING_CODING_CHARSET);
                checkpointOptions = CheckpointOptions.forSavepoint(targetLocation);
            } else {
                throw new IOException("Unknown checkpoint type: " + checkpointType);
            }
            return new CheckpointBarrier(id, timestamp, checkpointOptions);
        } else if (type == END_OF_SUPERSTEP_EVENT) {
            return EndOfSuperstepEvent.INSTANCE;
        } else if (type == CANCEL_CHECKPOINT_MARKER_EVENT) {
            long id = buffer.getLong();
            return new CancelCheckpointMarker(id);
        } else if (type == OTHER_EVENT) {
            try {
                final DataInputDeserializer deserializer = new DataInputDeserializer(buffer);
                final String className = deserializer.readUTF();
                final Class<? extends AbstractEvent> clazz;
                try {
                    clazz = classLoader.loadClass(className).asSubclass(AbstractEvent.class);
                } catch (ClassNotFoundException e) {
                    throw new IOException("Could not load event class '" + className + "'.", e);
                } catch (ClassCastException e) {
                    throw new IOException("The class '" + className + "' is not a valid subclass of '" + AbstractEvent.class.getName() + "'.", e);
                }
                final AbstractEvent event = InstantiationUtil.instantiate(clazz, AbstractEvent.class);
                event.read(deserializer);
                return event;
            } catch (Exception e) {
                throw new IOException("Error while deserializing or instantiating event.", e);
            }
        } else {
            throw new IOException("Corrupt byte stream for event");
        }
    } finally {
        buffer.order(bufferOrder);
    }
}

Example 5 with DataInputDeserializer

use of org.apache.flink.runtime.util.DataInputDeserializer in project flink by apache.

the class KvStateRequestSerializer method deserializeMap.

/**
	 * Deserializes all kv pairs with the given serializer.
	 *
	 * @param serializedValue Serialized value of type Map<UK, UV>
	 * @param keySerializer   Serializer for UK
	 * @param valueSerializer Serializer for UV
	 * @param <UK>            Type of the key
	 * @param <UV>            Type of the value.
	 * @return Deserialized map or <code>null</code> if the serialized value
	 * is <code>null</code>
	 * @throws IOException On failure during deserialization
	 */
public static <UK, UV> Map<UK, UV> deserializeMap(byte[] serializedValue, TypeSerializer<UK> keySerializer, TypeSerializer<UV> valueSerializer) throws IOException {
    if (serializedValue != null) {
        DataInputDeserializer in = new DataInputDeserializer(serializedValue, 0, serializedValue.length);
        Map<UK, UV> result = new HashMap<>();
        while (in.available() > 0) {
            UK key = keySerializer.deserialize(in);
            boolean isNull = in.readBoolean();
            UV value = isNull ? null : valueSerializer.deserialize(in);
            result.put(key, value);
        }
        return result;
    } else {
        return null;
    }
}