Examples with Watermark org.apache.flink.streaming.api.watermark.Watermark used on opensource projects

Example 26 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class AbstractStreamOperator method processWatermark1.

public void processWatermark1(Watermark mark) throws Exception {
    input1Watermark = mark.getTimestamp();
    long newMin = Math.min(input1Watermark, input2Watermark);
    if (newMin > combinedWatermark) {
        combinedWatermark = newMin;
        processWatermark(new Watermark(combinedWatermark));
    }
}

Example 27 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class StreamElementSerializer method deserialize.

@Override
public StreamElement deserialize(StreamElement reuse, DataInputView source) throws IOException {
    int tag = source.readByte();
    if (tag == TAG_REC_WITH_TIMESTAMP) {
        long timestamp = source.readLong();
        T value = typeSerializer.deserialize(source);
        StreamRecord<T> reuseRecord = reuse.asRecord();
        reuseRecord.replace(value, timestamp);
        return reuseRecord;
    } else if (tag == TAG_REC_WITHOUT_TIMESTAMP) {
        T value = typeSerializer.deserialize(source);
        StreamRecord<T> reuseRecord = reuse.asRecord();
        reuseRecord.replace(value);
        return reuseRecord;
    } else if (tag == TAG_WATERMARK) {
        return new Watermark(source.readLong());
    } else if (tag == TAG_LATENCY_MARKER) {
        return new LatencyMarker(source.readLong(), source.readInt(), source.readInt());
    } else {
        throw new IOException("Corrupt stream, found tag: " + tag);
    }
}

Example 28 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class StatusWatermarkValve method findAndOutputNewMinWatermarkAcrossAlignedChannels.

private void findAndOutputNewMinWatermarkAcrossAlignedChannels() {
    long newMinWatermark = Long.MAX_VALUE;
    // determine new overall watermark by considering only watermark-aligned channels across all channels
    for (InputChannelStatus channelStatus : channelStatuses) {
        if (channelStatus.isWatermarkAligned) {
            newMinWatermark = Math.min(channelStatus.watermark, newMinWatermark);
        }
    }
    // we acknowledge and output the new overall watermark if it is larger than the last output watermark
    if (newMinWatermark > lastOutputWatermark) {
        lastOutputWatermark = newMinWatermark;
        outputHandler.handleWatermark(new Watermark(lastOutputWatermark));
    }
}

Example 29 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class StreamIterationHead method run.

// ------------------------------------------------------------------------
@Override
protected void run() throws Exception {
    final String iterationId = getConfiguration().getIterationId();
    if (iterationId == null || iterationId.length() == 0) {
        throw new Exception("Missing iteration ID in the task configuration");
    }
    final String brokerID = createBrokerIdString(getEnvironment().getJobID(), iterationId, getEnvironment().getTaskInfo().getIndexOfThisSubtask());
    final long iterationWaitTime = getConfiguration().getIterationWaitTime();
    final boolean shouldWait = iterationWaitTime > 0;
    final BlockingQueue<StreamRecord<OUT>> dataChannel = new ArrayBlockingQueue<StreamRecord<OUT>>(1);
    // offer the queue for the tail
    BlockingQueueBroker.INSTANCE.handIn(brokerID, dataChannel);
    LOG.info("Iteration head {} added feedback queue under {}", getName(), brokerID);
    // do the work 
    try {
        @SuppressWarnings("unchecked") RecordWriterOutput<OUT>[] outputs = (RecordWriterOutput<OUT>[]) getStreamOutputs();
        // If timestamps are enabled we make sure to remove cyclic watermark dependencies
        if (isSerializingTimestamps()) {
            for (RecordWriterOutput<OUT> output : outputs) {
                output.emitWatermark(new Watermark(Long.MAX_VALUE));
            }
        }
        while (running) {
            StreamRecord<OUT> nextRecord = shouldWait ? dataChannel.poll(iterationWaitTime, TimeUnit.MILLISECONDS) : dataChannel.take();
            if (nextRecord != null) {
                for (RecordWriterOutput<OUT> output : outputs) {
                    output.collect(nextRecord);
                }
            } else {
                // done
                break;
            }
        }
    } finally {
        // make sure that we remove the queue from the broker, to prevent a resource leak
        BlockingQueueBroker.INSTANCE.remove(brokerID);
        LOG.info("Iteration head {} removed feedback queue under {}", getName(), brokerID);
    }
}

Example 30 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class BoundedOutOfOrdernessTimestampExtractorTest method runValidTests.

// ------------------------------------------------------------------------
private void runValidTests(BoundedOutOfOrdernessTimestampExtractor<Long> extractor) {
    assertEquals(new Watermark(Long.MIN_VALUE), extractor.getCurrentWatermark());
    assertEquals(13L, extractor.extractTimestamp(13L, 0L));
    assertEquals(13L, extractor.extractTimestamp(13L, 0L));
    assertEquals(14L, extractor.extractTimestamp(14L, 0L));
    assertEquals(20L, extractor.extractTimestamp(20L, 0L));
    assertEquals(new Watermark(10L), extractor.getCurrentWatermark());
    assertEquals(20L, extractor.extractTimestamp(20L, 0L));
    assertEquals(20L, extractor.extractTimestamp(20L, 0L));
    assertEquals(500L, extractor.extractTimestamp(500L, 0L));
    assertEquals(new Watermark(490L), extractor.getCurrentWatermark());
    assertEquals(Long.MAX_VALUE - 1, extractor.extractTimestamp(Long.MAX_VALUE - 1, 0L));
    assertEquals(new Watermark(Long.MAX_VALUE - 11), extractor.getCurrentWatermark());
}