Examples with DataInputDeserializer org.apache.flink.core.memory.DataInputDeserializer used on opensource projects

Example 26 with DataInputDeserializer

use of org.apache.flink.core.memory.DataInputDeserializer in project flink by apache.

the class SerializedCheckpointData method toDeque.

// ------------------------------------------------------------------------
// De-Serialize from Checkpoint
// ------------------------------------------------------------------------
/**
 * De-serializes an array of SerializedCheckpointData back into an ArrayDeque of element
 * checkpoints.
 *
 * @param data The data to be deserialized.
 * @param serializer The serializer used to deserialize the data.
 * @param <T> The type of the elements.
 * @return An ArrayDeque of element checkpoints.
 * @throws IOException Thrown, if the serialization fails.
 */
public static <T> ArrayDeque<Tuple2<Long, Set<T>>> toDeque(SerializedCheckpointData[] data, TypeSerializer<T> serializer) throws IOException {
    ArrayDeque<Tuple2<Long, Set<T>>> deque = new ArrayDeque<>(data.length);
    DataInputDeserializer deser = null;
    for (SerializedCheckpointData checkpoint : data) {
        byte[] serializedData = checkpoint.getSerializedData();
        if (deser == null) {
            deser = new DataInputDeserializer(serializedData, 0, serializedData.length);
        } else {
            deser.setBuffer(serializedData);
        }
        final Set<T> ids = new HashSet<>(checkpoint.getNumIds());
        final int numIds = checkpoint.getNumIds();
        for (int i = 0; i < numIds; i++) {
            ids.add(serializer.deserialize(deser));
        }
        deque.addLast(new Tuple2<Long, Set<T>>(checkpoint.checkpointId, ids));
    }
    return deque;
}

Example 27 with DataInputDeserializer

use of org.apache.flink.core.memory.DataInputDeserializer in project flink by apache.

the class RocksDBKeyedStateBackend method migrateStateValues.

/**
 * Migrate only the state value, that is the "value" that is stored in RocksDB. We don't migrate
 * the key here, which is made up of key group, key, namespace and map key (in case of
 * MapState).
 */
@SuppressWarnings("unchecked")
private <N, S extends State, SV> void migrateStateValues(StateDescriptor<S, SV> stateDesc, Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> stateMetaInfo) throws Exception {
    if (stateDesc.getType() == StateDescriptor.Type.MAP) {
        TypeSerializerSnapshot<SV> previousSerializerSnapshot = stateMetaInfo.f1.getPreviousStateSerializerSnapshot();
        checkState(previousSerializerSnapshot != null, "the previous serializer snapshot should exist.");
        checkState(previousSerializerSnapshot instanceof MapSerializerSnapshot, "previous serializer snapshot should be a MapSerializerSnapshot.");
        TypeSerializer<SV> newSerializer = stateMetaInfo.f1.getStateSerializer();
        checkState(newSerializer instanceof MapSerializer, "new serializer should be a MapSerializer.");
        MapSerializer<?, ?> mapSerializer = (MapSerializer<?, ?>) newSerializer;
        MapSerializerSnapshot<?, ?> mapSerializerSnapshot = (MapSerializerSnapshot<?, ?>) previousSerializerSnapshot;
        if (!checkMapStateKeySchemaCompatibility(mapSerializerSnapshot, mapSerializer)) {
            throw new StateMigrationException("The new serializer for a MapState requires state migration in order for the job to proceed, since the key schema has changed. However, migration for MapState currently only allows value schema evolutions.");
        }
    }
    LOG.info("Performing state migration for state {} because the state serializer's schema, i.e. serialization format, has changed.", stateDesc);
    // we need to get an actual state instance because migration is different
    // for different state types. For example, ListState needs to deal with
    // individual elements
    StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getType());
    if (stateFactory == null) {
        String message = String.format("State %s is not supported by %s", stateDesc.getClass(), this.getClass());
        throw new FlinkRuntimeException(message);
    }
    State state = stateFactory.createState(stateDesc, stateMetaInfo, RocksDBKeyedStateBackend.this);
    if (!(state instanceof AbstractRocksDBState)) {
        throw new FlinkRuntimeException("State should be an AbstractRocksDBState but is " + state);
    }
    @SuppressWarnings("unchecked") AbstractRocksDBState<?, ?, SV> rocksDBState = (AbstractRocksDBState<?, ?, SV>) state;
    Snapshot rocksDBSnapshot = db.getSnapshot();
    try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(db, stateMetaInfo.f0, readOptions);
        RocksDBWriteBatchWrapper batchWriter = new RocksDBWriteBatchWrapper(db, getWriteOptions(), getWriteBatchSize())) {
        iterator.seekToFirst();
        DataInputDeserializer serializedValueInput = new DataInputDeserializer();
        DataOutputSerializer migratedSerializedValueOutput = new DataOutputSerializer(512);
        while (iterator.isValid()) {
            serializedValueInput.setBuffer(iterator.value());
            rocksDBState.migrateSerializedValue(serializedValueInput, migratedSerializedValueOutput, stateMetaInfo.f1.getPreviousStateSerializer(), stateMetaInfo.f1.getStateSerializer());
            batchWriter.put(stateMetaInfo.f0, iterator.key(), migratedSerializedValueOutput.getCopyOfBuffer());
            migratedSerializedValueOutput.clear();
            iterator.next();
        }
    } finally {
        db.releaseSnapshot(rocksDBSnapshot);
        rocksDBSnapshot.close();
    }
}

Example 28 with DataInputDeserializer

use of org.apache.flink.core.memory.DataInputDeserializer in project flink by apache.

the class RocksDBMapState method getSerializedValue.

@Override
public byte[] getSerializedValue(final byte[] serializedKeyAndNamespace, final TypeSerializer<K> safeKeySerializer, final TypeSerializer<N> safeNamespaceSerializer, final TypeSerializer<Map<UK, UV>> safeValueSerializer) throws Exception {
    Preconditions.checkNotNull(serializedKeyAndNamespace);
    Preconditions.checkNotNull(safeKeySerializer);
    Preconditions.checkNotNull(safeNamespaceSerializer);
    Preconditions.checkNotNull(safeValueSerializer);
    // TODO make KvStateSerializer key-group aware to save this round trip and key-group
    // computation
    Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace(serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer);
    int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup(keyAndNamespace.f0, backend.getNumberOfKeyGroups());
    SerializedCompositeKeyBuilder<K> keyBuilder = new SerializedCompositeKeyBuilder<>(safeKeySerializer, backend.getKeyGroupPrefixBytes(), 32);
    keyBuilder.setKeyAndKeyGroup(keyAndNamespace.f0, keyGroup);
    final byte[] keyPrefixBytes = keyBuilder.buildCompositeKeyNamespace(keyAndNamespace.f1, namespaceSerializer);
    final MapSerializer<UK, UV> serializer = (MapSerializer<UK, UV>) safeValueSerializer;
    final TypeSerializer<UK> dupUserKeySerializer = serializer.getKeySerializer();
    final TypeSerializer<UV> dupUserValueSerializer = serializer.getValueSerializer();
    final DataInputDeserializer inputView = new DataInputDeserializer();
    final Iterator<Map.Entry<UK, UV>> iterator = new RocksDBMapIterator<Map.Entry<UK, UV>>(backend.db, keyPrefixBytes, dupUserKeySerializer, dupUserValueSerializer, inputView) {

        @Override
        public Map.Entry<UK, UV> next() {
            return nextEntry();
        }
    };
    // Return null to make the behavior consistent with other backends
    if (!iterator.hasNext()) {
        return null;
    }
    return KvStateSerializer.serializeMap(() -> iterator, dupUserKeySerializer, dupUserValueSerializer);
}

Example 29 with DataInputDeserializer

use of org.apache.flink.core.memory.DataInputDeserializer in project flink by apache.

the class StreamElementSerializerTest method serializeAndDeserialize.

@SuppressWarnings("unchecked")
private static <T, X extends StreamElement> X serializeAndDeserialize(X record, StreamElementSerializer<T> serializer) throws IOException {
    DataOutputSerializer output = new DataOutputSerializer(32);
    serializer.serialize(record, output);
    // additional binary copy step
    DataInputDeserializer copyInput = new DataInputDeserializer(output.getByteArray(), 0, output.length());
    DataOutputSerializer copyOutput = new DataOutputSerializer(32);
    serializer.copy(copyInput, copyOutput);
    DataInputDeserializer input = new DataInputDeserializer(copyOutput.getByteArray(), 0, copyOutput.length());
    return (X) serializer.deserialize(input);
}

Example 30 with DataInputDeserializer

use of org.apache.flink.core.memory.DataInputDeserializer in project flink by apache.

the class CheckpointBarrierTest method testSerialization.

/**
 * Test serialization of the checkpoint barrier. The checkpoint barrier does not support its own
 * serialization, in order to be immutable.
 */
@Test
public void testSerialization() throws Exception {
    long id = Integer.MAX_VALUE + 123123L;
    long timestamp = Integer.MAX_VALUE + 1228L;
    CheckpointOptions options = CheckpointOptions.forCheckpointWithDefaultLocation();
    CheckpointBarrier barrier = new CheckpointBarrier(id, timestamp, options);
    try {
        barrier.write(new DataOutputSerializer(1024));
        fail("should throw an exception");
    } catch (UnsupportedOperationException e) {
    // expected
    }
    try {
        barrier.read(new DataInputDeserializer(new byte[32]));
        fail("should throw an exception");
    } catch (UnsupportedOperationException e) {
    // expected
    }
}