Examples with DataInputView org.apache.flink.core.memory.DataInputView used on opensource projects

Example 11 with DataInputView

use of org.apache.flink.core.memory.DataInputView in project flink by apache.

the class FromElementsFunction method run.

@Override
public void run(SourceContext<T> ctx) throws Exception {
    Preconditions.checkState(serializer != null, "serializer not configured");
    ByteArrayInputStream bais = new ByteArrayInputStream(elementsSerialized);
    final DataInputView input = new DataInputViewStreamWrapper(bais);
    // if we are restored from a checkpoint and need to skip elements, skip them now.
    int toSkip = numElementsToSkip;
    if (toSkip > 0) {
        try {
            while (toSkip > 0) {
                serializer.deserialize(input);
                toSkip--;
            }
        } catch (Exception e) {
            throw new IOException("Failed to deserialize an element from the source. " + "If you are using user-defined serialization (Value and Writable types), check the " + "serialization functions.\nSerializer is " + serializer, e);
        }
        this.numElementsEmitted = this.numElementsToSkip;
    }
    final Object lock = ctx.getCheckpointLock();
    while (isRunning && numElementsEmitted < numElements) {
        T next;
        try {
            next = serializer.deserialize(input);
        } catch (Exception e) {
            throw new IOException("Failed to deserialize an element from the source. " + "If you are using user-defined serialization (Value and Writable types), check the " + "serialization functions.\nSerializer is " + serializer, e);
        }
        synchronized (lock) {
            ctx.collect(next);
            numElementsEmitted++;
        }
    }
}

Example 12 with DataInputView

use of org.apache.flink.core.memory.DataInputView in project flink by apache.

the class DefaultOperatorStateBackend method deserializeStateValues.

private static <S> void deserializeStateValues(PartitionableListState<S> stateListForName, FSDataInputStream in, OperatorStateHandle.StateMetaInfo metaInfo) throws IOException {
    if (null != metaInfo) {
        long[] offsets = metaInfo.getOffsets();
        if (null != offsets) {
            DataInputView div = new DataInputViewStreamWrapper(in);
            TypeSerializer<S> serializer = stateListForName.getPartitionStateSerializer();
            for (long offset : offsets) {
                in.seek(offset);
                stateListForName.add(serializer.deserialize(div));
            }
        }
    }
}

Example 13 with DataInputView

use of org.apache.flink.core.memory.DataInputView in project flink by apache.

the class SpanningRecordSerializerTest method testEmptyRecords.

@Test
public void testEmptyRecords() {
    final int SEGMENT_SIZE = 11;
    final SpanningRecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<SerializationTestType>();
    final Buffer buffer = new Buffer(MemorySegmentFactory.allocateUnpooledSegment(SEGMENT_SIZE), mock(BufferRecycler.class));
    try {
        Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, serializer.setNextBuffer(buffer));
    } catch (IOException e) {
        e.printStackTrace();
    }
    try {
        SerializationTestType emptyRecord = new SerializationTestType() {

            @Override
            public SerializationTestType getRandom(Random rnd) {
                throw new UnsupportedOperationException();
            }

            @Override
            public int length() {
                throw new UnsupportedOperationException();
            }

            @Override
            public void write(DataOutputView out) {
            }

            @Override
            public void read(DataInputView in) {
            }

            @Override
            public int hashCode() {
                throw new UnsupportedOperationException();
            }

            @Override
            public boolean equals(Object obj) {
                throw new UnsupportedOperationException();
            }
        };
        RecordSerializer.SerializationResult result = serializer.addRecord(emptyRecord);
        Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
        result = serializer.addRecord(emptyRecord);
        Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
        result = serializer.addRecord(emptyRecord);
        Assert.assertEquals(RecordSerializer.SerializationResult.PARTIAL_RECORD_MEMORY_SEGMENT_FULL, result);
        result = serializer.setNextBuffer(buffer);
        Assert.assertEquals(RecordSerializer.SerializationResult.FULL_RECORD, result);
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 14 with DataInputView

use of org.apache.flink.core.memory.DataInputView in project flink by apache.

the class PostVersionedIOReadableWritable method read.

/**
 * This read attempts to first identify if the input view contains the special {@link
 * #VERSIONED_IDENTIFIER} by reading and buffering the first few bytes. If identified to be
 * versioned, the usual version resolution read path in {@link
 * VersionedIOReadableWritable#read(DataInputView)} is invoked. Otherwise, we "reset" the input
 * stream by pushing back the read buffered bytes into the stream.
 */
public final void read(InputStream inputStream) throws IOException {
    byte[] tmp = new byte[VERSIONED_IDENTIFIER.length];
    int totalRead = IOUtils.tryReadFully(inputStream, tmp);
    if (Arrays.equals(tmp, VERSIONED_IDENTIFIER)) {
        DataInputView inputView = new DataInputViewStreamWrapper(inputStream);
        super.read(inputView);
        read(inputView, true);
    } else {
        InputStream streamToRead = inputStream;
        if (totalRead > 0) {
            PushbackInputStream resetStream = new PushbackInputStream(inputStream, totalRead);
            resetStream.unread(tmp, 0, totalRead);
            streamToRead = resetStream;
        }
        read(new DataInputViewStreamWrapper(streamToRead), false);
    }
}

Example 15 with DataInputView

use of org.apache.flink.core.memory.DataInputView in project flink by apache.

the class TypeSerializerUpgradeTestBase method assertSerializerIsValid.

/**
 * Asserts that a given {@link TypeSerializer} is valid, given a {@link DataInputView} of
 * serialized data.
 *
 * <p>A serializer is valid, iff:
 *
 * <ul>
 *   <li>1. The serializer can read and then write again the given serialized data.
 *   <li>2. The serializer can produce a serializer snapshot which can be written and then read
 *       back again.
 *   <li>3. The serializer's produced snapshot is capable of creating a restore serializer.
 *   <li>4. The restore serializer created from the serializer snapshot can read and then write
 *       again data written by step 1. Given that the serializer is not a restore serializer
 *       already.
 * </ul>
 */
private static <T> void assertSerializerIsValid(TypeSerializer<T> serializer, DataInputView dataInput, Matcher<T> testDataMatcher) throws Exception {
    DataInputView serializedData = readAndThenWriteData(dataInput, serializer, serializer, testDataMatcher);
    TypeSerializerSnapshot<T> snapshot = writeAndThenReadSerializerSnapshot(serializer);
    TypeSerializer<T> restoreSerializer = snapshot.restoreSerializer();
    serializedData = readAndThenWriteData(serializedData, restoreSerializer, restoreSerializer, testDataMatcher);
    TypeSerializer<T> duplicateSerializer = snapshot.restoreSerializer().duplicate();
    readAndThenWriteData(serializedData, duplicateSerializer, duplicateSerializer, testDataMatcher);
}