use of org.apache.flink.runtime.operators.lifecycle.event.WatermarkReceivedEvent in project flink by apache.
the class OneInputTestStreamOperator method processWatermark.
@Override
public void processWatermark(Watermark mark) throws Exception {
eventQueue.add(new WatermarkReceivedEvent(operatorID, getRuntimeContext().getIndexOfThisSubtask(), getRuntimeContext().getAttemptNumber(), mark.getTimestamp(), 1));
super.processWatermark(mark);
}
use of org.apache.flink.runtime.operators.lifecycle.event.WatermarkReceivedEvent in project flink by apache.
the class DrainingValidator method validateSubtaskAttempt.
private void validateSubtaskAttempt(TestJobWithDescription job, String operatorId, int subtaskIndex, List<TestEvent> operatorEvents) {
BitSet endedInputs = new BitSet();
BitSet inputsWithMaxWatermark = new BitSet();
for (TestEvent ev : operatorEvents) {
if (ev instanceof WatermarkReceivedEvent) {
WatermarkReceivedEvent w = (WatermarkReceivedEvent) ev;
if (w.ts == Watermark.MAX_WATERMARK.getTimestamp()) {
assertFalse(String.format("Max Watermark received twice by %s/%d/%d", w.operatorId, w.subtaskIndex, w.inputId), inputsWithMaxWatermark.get(w.inputId));
inputsWithMaxWatermark.set(w.inputId);
}
} else if (ev instanceof InputEndedEvent) {
InputEndedEvent w = (InputEndedEvent) ev;
assertTrue(format("Input %d ended before receiving max watermark by %s[%d]#%d", w.inputId, operatorId, subtaskIndex, w.attemptNumber), inputsWithMaxWatermark.get(w.inputId));
assertFalse(endedInputs.get(w.inputId));
endedInputs.set(w.inputId);
}
}
assertEquals(format("Incorrect number of ended inputs for %s[%d]", operatorId, subtaskIndex), getNumInputs(job, operatorId), endedInputs.cardinality());
}
use of org.apache.flink.runtime.operators.lifecycle.event.WatermarkReceivedEvent in project flink by apache.
the class TwoInputTestStreamOperator method processWatermark2.
@Override
public void processWatermark2(Watermark mark) throws Exception {
collectEvent(new WatermarkReceivedEvent(operatorID, getRuntimeContext().getIndexOfThisSubtask(), getRuntimeContext().getAttemptNumber(), mark.getTimestamp(), 2));
super.processWatermark2(mark);
}
use of org.apache.flink.runtime.operators.lifecycle.event.WatermarkReceivedEvent in project flink by apache.
the class FinishingValidator method validateOperatorLifecycle.
@Override
public void validateOperatorLifecycle(TestJobWithDescription job, String operatorId, int subtaskIndex, List<TestEvent> operatorEvents) {
boolean opFinished = false;
Set<Long> finalCheckpointCandidates = new HashSet<>();
for (TestEvent ev : operatorEvents) {
if (ev instanceof OperatorFinishedEvent) {
opFinished = true;
} else if (ev instanceof CheckpointStartedEvent) {
if (opFinished) {
finalCheckpointCandidates.add(((CheckpointStartedEvent) ev).checkpointID);
}
} else if (ev instanceof CheckpointCompletedEvent) {
if (finalCheckpointCandidates.contains(((CheckpointCompletedEvent) ev).checkpointID)) {
return;
}
} else if (opFinished) {
fail(format("Unexpected event after operator %s[%d] finished: %s", operatorId, subtaskIndex, ev));
}
}
assertTrue(format("Operator %s[%d] wasn't finished (events: %s)", operatorId, subtaskIndex, operatorEvents), opFinished);
fail(format("Operator %s[%d] was finished but didn't finish the checkpoint after that;" + "checkpoints started after finish: %s (events (excluding watermarks): %s)", operatorId, subtaskIndex, finalCheckpointCandidates, operatorEvents.stream().filter(ev -> !(ev instanceof WatermarkReceivedEvent)).collect(toList())));
}
use of org.apache.flink.runtime.operators.lifecycle.event.WatermarkReceivedEvent in project flink by apache.
the class TwoInputTestStreamOperator method processWatermark1.
@Override
public void processWatermark1(Watermark mark) throws Exception {
collectEvent(new WatermarkReceivedEvent(operatorID, getRuntimeContext().getIndexOfThisSubtask(), getRuntimeContext().getAttemptNumber(), mark.getTimestamp(), 1));
super.processWatermark1(mark);
}
Aggregations