Examples with AbstractEvent org.apache.flink.runtime.event.AbstractEvent used on opensource projects

Example 1 with AbstractEvent

use of org.apache.flink.runtime.event.AbstractEvent in project flink by apache.

the class SingleInputGate method getNextBufferOrEvent.

// ------------------------------------------------------------------------
// Consume
// ------------------------------------------------------------------------
@Override
public BufferOrEvent getNextBufferOrEvent() throws IOException, InterruptedException {
    if (hasReceivedAllEndOfPartitionEvents) {
        return null;
    }
    if (isReleased) {
        throw new IllegalStateException("Released");
    }
    requestPartitions();
    InputChannel currentChannel;
    boolean moreAvailable;
    synchronized (inputChannelsWithData) {
        while (inputChannelsWithData.size() == 0) {
            if (isReleased) {
                throw new IllegalStateException("Released");
            }
            inputChannelsWithData.wait();
        }
        currentChannel = inputChannelsWithData.remove();
        moreAvailable = inputChannelsWithData.size() > 0;
    }
    final BufferAndAvailability result = currentChannel.getNextBuffer();
    // Sanity check that notifications only happen when data is available
    if (result == null) {
        throw new IllegalStateException("Bug in input gate/channel logic: input gate got " + "notified by channel about available data, but none was available.");
    }
    // will come for that channel
    if (result.moreAvailable()) {
        queueChannel(currentChannel);
    }
    final Buffer buffer = result.buffer();
    if (buffer.isBuffer()) {
        return new BufferOrEvent(buffer, currentChannel.getChannelIndex(), moreAvailable);
    } else {
        final AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
        if (event.getClass() == EndOfPartitionEvent.class) {
            channelsWithEndOfPartitionEvents.set(currentChannel.getChannelIndex());
            if (channelsWithEndOfPartitionEvents.cardinality() == numberOfInputChannels) {
                hasReceivedAllEndOfPartitionEvents = true;
            }
            currentChannel.notifySubpartitionConsumed();
            currentChannel.releaseAllResources();
        }
        return new BufferOrEvent(event, currentChannel.getChannelIndex(), moreAvailable);
    }
}

Example 2 with AbstractEvent

use of org.apache.flink.runtime.event.AbstractEvent in project flink by apache.

the class StreamInputProcessor method processInput.

public boolean processInput() throws Exception {
    if (isFinished) {
        return false;
    }
    if (numRecordsIn == null) {
        numRecordsIn = ((OperatorMetricGroup) streamOperator.getMetricGroup()).getIOMetricGroup().getNumRecordsInCounter();
    }
    while (true) {
        if (currentRecordDeserializer != null) {
            DeserializationResult result = currentRecordDeserializer.getNextRecord(deserializationDelegate);
            if (result.isBufferConsumed()) {
                currentRecordDeserializer.getCurrentBuffer().recycle();
                currentRecordDeserializer = null;
            }
            if (result.isFullRecord()) {
                StreamElement recordOrMark = deserializationDelegate.getInstance();
                if (recordOrMark.isWatermark()) {
                    // handle watermark
                    statusWatermarkValve.inputWatermark(recordOrMark.asWatermark(), currentChannel);
                    continue;
                } else if (recordOrMark.isStreamStatus()) {
                    // handle stream status
                    statusWatermarkValve.inputStreamStatus(recordOrMark.asStreamStatus(), currentChannel);
                    continue;
                } else if (recordOrMark.isLatencyMarker()) {
                    // handle latency marker
                    synchronized (lock) {
                        streamOperator.processLatencyMarker(recordOrMark.asLatencyMarker());
                    }
                    continue;
                } else {
                    // now we can do the actual processing
                    StreamRecord<IN> record = recordOrMark.asRecord();
                    synchronized (lock) {
                        numRecordsIn.inc();
                        streamOperator.setKeyContextElement1(record);
                        streamOperator.processElement(record);
                    }
                    return true;
                }
            }
        }
        final BufferOrEvent bufferOrEvent = barrierHandler.getNextNonBlocked();
        if (bufferOrEvent != null) {
            if (bufferOrEvent.isBuffer()) {
                currentChannel = bufferOrEvent.getChannelIndex();
                currentRecordDeserializer = recordDeserializers[currentChannel];
                currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
            } else {
                // Event received
                final AbstractEvent event = bufferOrEvent.getEvent();
                if (event.getClass() != EndOfPartitionEvent.class) {
                    throw new IOException("Unexpected event: " + event);
                }
            }
        } else {
            isFinished = true;
            if (!barrierHandler.isEmpty()) {
                throw new IllegalStateException("Trailing data in checkpoint barrier handler.");
            }
            return false;
        }
    }
}

Example 3 with AbstractEvent

use of org.apache.flink.runtime.event.AbstractEvent in project flink by apache.

the class StreamTwoInputProcessor method processInput.

public boolean processInput() throws Exception {
    if (isFinished) {
        return false;
    }
    while (true) {
        if (currentRecordDeserializer != null) {
            DeserializationResult result;
            if (currentChannel < numInputChannels1) {
                result = currentRecordDeserializer.getNextRecord(deserializationDelegate1);
            } else {
                result = currentRecordDeserializer.getNextRecord(deserializationDelegate2);
            }
            if (result.isBufferConsumed()) {
                currentRecordDeserializer.getCurrentBuffer().recycle();
                currentRecordDeserializer = null;
            }
            if (result.isFullRecord()) {
                if (currentChannel < numInputChannels1) {
                    StreamElement recordOrWatermark = deserializationDelegate1.getInstance();
                    if (recordOrWatermark.isWatermark()) {
                        statusWatermarkValve1.inputWatermark(recordOrWatermark.asWatermark(), currentChannel);
                        continue;
                    } else if (recordOrWatermark.isStreamStatus()) {
                        statusWatermarkValve1.inputStreamStatus(recordOrWatermark.asStreamStatus(), currentChannel);
                        continue;
                    } else if (recordOrWatermark.isLatencyMarker()) {
                        synchronized (lock) {
                            streamOperator.processLatencyMarker1(recordOrWatermark.asLatencyMarker());
                        }
                        continue;
                    } else {
                        StreamRecord<IN1> record = recordOrWatermark.asRecord();
                        synchronized (lock) {
                            streamOperator.setKeyContextElement1(record);
                            streamOperator.processElement1(record);
                        }
                        return true;
                    }
                } else {
                    StreamElement recordOrWatermark = deserializationDelegate2.getInstance();
                    if (recordOrWatermark.isWatermark()) {
                        statusWatermarkValve2.inputWatermark(recordOrWatermark.asWatermark(), currentChannel - numInputChannels1);
                        continue;
                    } else if (recordOrWatermark.isStreamStatus()) {
                        statusWatermarkValve2.inputStreamStatus(recordOrWatermark.asStreamStatus(), currentChannel - numInputChannels1);
                        continue;
                    } else if (recordOrWatermark.isLatencyMarker()) {
                        synchronized (lock) {
                            streamOperator.processLatencyMarker2(recordOrWatermark.asLatencyMarker());
                        }
                        continue;
                    } else {
                        StreamRecord<IN2> record = recordOrWatermark.asRecord();
                        synchronized (lock) {
                            streamOperator.setKeyContextElement2(record);
                            streamOperator.processElement2(record);
                        }
                        return true;
                    }
                }
            }
        }
        final BufferOrEvent bufferOrEvent = barrierHandler.getNextNonBlocked();
        if (bufferOrEvent != null) {
            if (bufferOrEvent.isBuffer()) {
                currentChannel = bufferOrEvent.getChannelIndex();
                currentRecordDeserializer = recordDeserializers[currentChannel];
                currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
            } else {
                // Event received
                final AbstractEvent event = bufferOrEvent.getEvent();
                if (event.getClass() != EndOfPartitionEvent.class) {
                    throw new IOException("Unexpected event: " + event);
                }
            }
        } else {
            isFinished = true;
            if (!barrierHandler.isEmpty()) {
                throw new IllegalStateException("Trailing data in checkpoint barrier handler.");
            }
            return false;
        }
    }
}

Example 4 with AbstractEvent

use of org.apache.flink.runtime.event.AbstractEvent in project flink by apache.

the class PipelinedSubpartition method parseCheckpointBarrier.

@Nullable
private CheckpointBarrier parseCheckpointBarrier(BufferConsumer bufferConsumer) {
    CheckpointBarrier barrier;
    try (BufferConsumer bc = bufferConsumer.copy()) {
        Buffer buffer = bc.build();
        try {
            final AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
            barrier = event instanceof CheckpointBarrier ? (CheckpointBarrier) event : null;
        } catch (IOException e) {
            throw new IllegalStateException("Should always be able to deserialize in-memory event", e);
        } finally {
            buffer.recycleBuffer();
        }
    }
    return barrier;
}

Example 5 with AbstractEvent

use of org.apache.flink.runtime.event.AbstractEvent in project flink by apache.

the class CheckpointedInputGate method handleEvent.

private Optional<BufferOrEvent> handleEvent(BufferOrEvent bufferOrEvent) throws IOException {
    Class<? extends AbstractEvent> eventClass = bufferOrEvent.getEvent().getClass();
    if (eventClass == CheckpointBarrier.class) {
        CheckpointBarrier checkpointBarrier = (CheckpointBarrier) bufferOrEvent.getEvent();
        barrierHandler.processBarrier(checkpointBarrier, bufferOrEvent.getChannelInfo(), false);
    } else if (eventClass == CancelCheckpointMarker.class) {
        barrierHandler.processCancellationBarrier((CancelCheckpointMarker) bufferOrEvent.getEvent(), bufferOrEvent.getChannelInfo());
    } else if (eventClass == EndOfData.class) {
        inputGate.acknowledgeAllRecordsProcessed(bufferOrEvent.getChannelInfo());
    } else if (eventClass == EndOfPartitionEvent.class) {
        barrierHandler.processEndOfPartition(bufferOrEvent.getChannelInfo());
    } else if (eventClass == EventAnnouncement.class) {
        EventAnnouncement eventAnnouncement = (EventAnnouncement) bufferOrEvent.getEvent();
        AbstractEvent announcedEvent = eventAnnouncement.getAnnouncedEvent();
        checkState(announcedEvent instanceof CheckpointBarrier, "Only CheckpointBarrier announcement are currently supported, but found [%s]", announcedEvent);
        CheckpointBarrier announcedBarrier = (CheckpointBarrier) announcedEvent;
        barrierHandler.processBarrierAnnouncement(announcedBarrier, eventAnnouncement.getSequenceNumber(), bufferOrEvent.getChannelInfo());
    } else if (bufferOrEvent.getEvent().getClass() == EndOfChannelStateEvent.class) {
        upstreamRecoveryTracker.handleEndOfRecovery(bufferOrEvent.getChannelInfo());
    }
    return Optional.of(bufferOrEvent);
}