Examples with DeserializationResult org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult used on opensource projects

Example 1 with DeserializationResult

use of org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult in project flink by apache.

the class StreamInputProcessor method processInput.

public boolean processInput() throws Exception {
    if (isFinished) {
        return false;
    }
    if (numRecordsIn == null) {
        numRecordsIn = ((OperatorMetricGroup) streamOperator.getMetricGroup()).getIOMetricGroup().getNumRecordsInCounter();
    }
    while (true) {
        if (currentRecordDeserializer != null) {
            DeserializationResult result = currentRecordDeserializer.getNextRecord(deserializationDelegate);
            if (result.isBufferConsumed()) {
                currentRecordDeserializer.getCurrentBuffer().recycle();
                currentRecordDeserializer = null;
            }
            if (result.isFullRecord()) {
                StreamElement recordOrMark = deserializationDelegate.getInstance();
                if (recordOrMark.isWatermark()) {
                    // handle watermark
                    statusWatermarkValve.inputWatermark(recordOrMark.asWatermark(), currentChannel);
                    continue;
                } else if (recordOrMark.isStreamStatus()) {
                    // handle stream status
                    statusWatermarkValve.inputStreamStatus(recordOrMark.asStreamStatus(), currentChannel);
                    continue;
                } else if (recordOrMark.isLatencyMarker()) {
                    // handle latency marker
                    synchronized (lock) {
                        streamOperator.processLatencyMarker(recordOrMark.asLatencyMarker());
                    }
                    continue;
                } else {
                    // now we can do the actual processing
                    StreamRecord<IN> record = recordOrMark.asRecord();
                    synchronized (lock) {
                        numRecordsIn.inc();
                        streamOperator.setKeyContextElement1(record);
                        streamOperator.processElement(record);
                    }
                    return true;
                }
            }
        }
        final BufferOrEvent bufferOrEvent = barrierHandler.getNextNonBlocked();
        if (bufferOrEvent != null) {
            if (bufferOrEvent.isBuffer()) {
                currentChannel = bufferOrEvent.getChannelIndex();
                currentRecordDeserializer = recordDeserializers[currentChannel];
                currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
            } else {
                // Event received
                final AbstractEvent event = bufferOrEvent.getEvent();
                if (event.getClass() != EndOfPartitionEvent.class) {
                    throw new IOException("Unexpected event: " + event);
                }
            }
        } else {
            isFinished = true;
            if (!barrierHandler.isEmpty()) {
                throw new IllegalStateException("Trailing data in checkpoint barrier handler.");
            }
            return false;
        }
    }
}

Example 2 with DeserializationResult

use of org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult in project flink by apache.

the class StreamTwoInputProcessor method processInput.

public boolean processInput() throws Exception {
    if (isFinished) {
        return false;
    }
    while (true) {
        if (currentRecordDeserializer != null) {
            DeserializationResult result;
            if (currentChannel < numInputChannels1) {
                result = currentRecordDeserializer.getNextRecord(deserializationDelegate1);
            } else {
                result = currentRecordDeserializer.getNextRecord(deserializationDelegate2);
            }
            if (result.isBufferConsumed()) {
                currentRecordDeserializer.getCurrentBuffer().recycle();
                currentRecordDeserializer = null;
            }
            if (result.isFullRecord()) {
                if (currentChannel < numInputChannels1) {
                    StreamElement recordOrWatermark = deserializationDelegate1.getInstance();
                    if (recordOrWatermark.isWatermark()) {
                        statusWatermarkValve1.inputWatermark(recordOrWatermark.asWatermark(), currentChannel);
                        continue;
                    } else if (recordOrWatermark.isStreamStatus()) {
                        statusWatermarkValve1.inputStreamStatus(recordOrWatermark.asStreamStatus(), currentChannel);
                        continue;
                    } else if (recordOrWatermark.isLatencyMarker()) {
                        synchronized (lock) {
                            streamOperator.processLatencyMarker1(recordOrWatermark.asLatencyMarker());
                        }
                        continue;
                    } else {
                        StreamRecord<IN1> record = recordOrWatermark.asRecord();
                        synchronized (lock) {
                            streamOperator.setKeyContextElement1(record);
                            streamOperator.processElement1(record);
                        }
                        return true;
                    }
                } else {
                    StreamElement recordOrWatermark = deserializationDelegate2.getInstance();
                    if (recordOrWatermark.isWatermark()) {
                        statusWatermarkValve2.inputWatermark(recordOrWatermark.asWatermark(), currentChannel - numInputChannels1);
                        continue;
                    } else if (recordOrWatermark.isStreamStatus()) {
                        statusWatermarkValve2.inputStreamStatus(recordOrWatermark.asStreamStatus(), currentChannel - numInputChannels1);
                        continue;
                    } else if (recordOrWatermark.isLatencyMarker()) {
                        synchronized (lock) {
                            streamOperator.processLatencyMarker2(recordOrWatermark.asLatencyMarker());
                        }
                        continue;
                    } else {
                        StreamRecord<IN2> record = recordOrWatermark.asRecord();
                        synchronized (lock) {
                            streamOperator.setKeyContextElement2(record);
                            streamOperator.processElement2(record);
                        }
                        return true;
                    }
                }
            }
        }
        final BufferOrEvent bufferOrEvent = barrierHandler.getNextNonBlocked();
        if (bufferOrEvent != null) {
            if (bufferOrEvent.isBuffer()) {
                currentChannel = bufferOrEvent.getChannelIndex();
                currentRecordDeserializer = recordDeserializers[currentChannel];
                currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
            } else {
                // Event received
                final AbstractEvent event = bufferOrEvent.getEvent();
                if (event.getClass() != EndOfPartitionEvent.class) {
                    throw new IOException("Unexpected event: " + event);
                }
            }
        } else {
            isFinished = true;
            if (!barrierHandler.isEmpty()) {
                throw new IllegalStateException("Trailing data in checkpoint barrier handler.");
            }
            return false;
        }
    }
}

Example 3 with DeserializationResult

use of org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult in project flink by apache.

the class AbstractRecordReader method getNextRecord.

protected boolean getNextRecord(T target) throws IOException, InterruptedException {
    if (isFinished) {
        return false;
    }
    while (true) {
        if (currentRecordDeserializer != null) {
            DeserializationResult result = currentRecordDeserializer.getNextRecord(target);
            if (result.isBufferConsumed()) {
                final Buffer currentBuffer = currentRecordDeserializer.getCurrentBuffer();
                currentBuffer.recycle();
                currentRecordDeserializer = null;
            }
            if (result.isFullRecord()) {
                return true;
            }
        }
        final BufferOrEvent bufferOrEvent = inputGate.getNextBufferOrEvent();
        if (bufferOrEvent.isBuffer()) {
            currentRecordDeserializer = recordDeserializers[bufferOrEvent.getChannelIndex()];
            currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
        } else {
            // records, not in the middle of a fragment
            if (recordDeserializers[bufferOrEvent.getChannelIndex()].hasUnfinishedData()) {
                throw new IOException("Received an event in channel " + bufferOrEvent.getChannelIndex() + " while still having " + "data from a record. This indicates broken serialization logic. " + "If you are using custom serialization code (Writable or Value types), check their " + "serialization routines. In the case of Kryo, check the respective Kryo serializer.");
            }
            if (handleEvent(bufferOrEvent.getEvent())) {
                if (inputGate.isFinished()) {
                    isFinished = true;
                    return false;
                } else if (hasReachedEndOfSuperstep()) {
                    return false;
                }
            // else: More data is coming...
            }
        }
    }
}