Search in sources :

Example 1 with TestEvent

use of org.apache.flink.runtime.operators.lifecycle.event.TestEvent in project flink by apache.

the class DrainingValidator method validateSubtaskAttempt.

private void validateSubtaskAttempt(TestJobWithDescription job, String operatorId, int subtaskIndex, List<TestEvent> operatorEvents) {
    BitSet endedInputs = new BitSet();
    BitSet inputsWithMaxWatermark = new BitSet();
    for (TestEvent ev : operatorEvents) {
        if (ev instanceof WatermarkReceivedEvent) {
            WatermarkReceivedEvent w = (WatermarkReceivedEvent) ev;
            if (w.ts == Watermark.MAX_WATERMARK.getTimestamp()) {
                assertFalse(String.format("Max Watermark received twice by %s/%d/%d", w.operatorId, w.subtaskIndex, w.inputId), inputsWithMaxWatermark.get(w.inputId));
                inputsWithMaxWatermark.set(w.inputId);
            }
        } else if (ev instanceof InputEndedEvent) {
            InputEndedEvent w = (InputEndedEvent) ev;
            assertTrue(format("Input %d ended before receiving max watermark by %s[%d]#%d", w.inputId, operatorId, subtaskIndex, w.attemptNumber), inputsWithMaxWatermark.get(w.inputId));
            assertFalse(endedInputs.get(w.inputId));
            endedInputs.set(w.inputId);
        }
    }
    assertEquals(format("Incorrect number of ended inputs for %s[%d]", operatorId, subtaskIndex), getNumInputs(job, operatorId), endedInputs.cardinality());
}
Also used : InputEndedEvent(org.apache.flink.runtime.operators.lifecycle.event.InputEndedEvent) TestEvent(org.apache.flink.runtime.operators.lifecycle.event.TestEvent) BitSet(java.util.BitSet) WatermarkReceivedEvent(org.apache.flink.runtime.operators.lifecycle.event.WatermarkReceivedEvent)

Example 2 with TestEvent

use of org.apache.flink.runtime.operators.lifecycle.event.TestEvent in project flink by apache.

the class TestOperatorLifecycleValidator method checkOperatorsLifecycle.

static void checkOperatorsLifecycle(TestJobWithDescription testJob, TestOperatorLifecycleValidator... validators) {
    Map<Tuple2<String, Integer>, List<TestEvent>> eventsByOperator = new HashMap<>();
    for (TestEvent ev : testJob.eventQueue.getAll()) {
        eventsByOperator.computeIfAbsent(Tuple2.of(ev.operatorId, ev.subtaskIndex), ign -> new ArrayList<>()).add(ev);
    }
    eventsByOperator.forEach((operatorIdAndIndex, operatorEvents) -> {
        String id = operatorIdAndIndex.f0;
        if (testJob.operatorsWithLifecycleTracking.contains(id)) {
            for (TestOperatorLifecycleValidator validator : validators) {
                validator.validateOperatorLifecycle(testJob, id, operatorIdAndIndex.f1, operatorEvents);
            }
        }
    });
}
Also used : List(java.util.List) Tuple2(org.apache.flink.api.java.tuple.Tuple2) Map(java.util.Map) TestJobWithDescription(org.apache.flink.runtime.operators.lifecycle.TestJobWithDescription) HashMap(java.util.HashMap) TestEvent(org.apache.flink.runtime.operators.lifecycle.event.TestEvent) ArrayList(java.util.ArrayList) HashMap(java.util.HashMap) TestEvent(org.apache.flink.runtime.operators.lifecycle.event.TestEvent) Tuple2(org.apache.flink.api.java.tuple.Tuple2) ArrayList(java.util.ArrayList) List(java.util.List) ArrayList(java.util.ArrayList)

Example 3 with TestEvent

use of org.apache.flink.runtime.operators.lifecycle.event.TestEvent in project flink by apache.

the class FinishingValidator method validateOperatorLifecycle.

@Override
public void validateOperatorLifecycle(TestJobWithDescription job, String operatorId, int subtaskIndex, List<TestEvent> operatorEvents) {
    boolean opFinished = false;
    Set<Long> finalCheckpointCandidates = new HashSet<>();
    for (TestEvent ev : operatorEvents) {
        if (ev instanceof OperatorFinishedEvent) {
            opFinished = true;
        } else if (ev instanceof CheckpointStartedEvent) {
            if (opFinished) {
                finalCheckpointCandidates.add(((CheckpointStartedEvent) ev).checkpointID);
            }
        } else if (ev instanceof CheckpointCompletedEvent) {
            if (finalCheckpointCandidates.contains(((CheckpointCompletedEvent) ev).checkpointID)) {
                return;
            }
        } else if (opFinished) {
            fail(format("Unexpected event after operator %s[%d] finished: %s", operatorId, subtaskIndex, ev));
        }
    }
    assertTrue(format("Operator %s[%d] wasn't finished (events: %s)", operatorId, subtaskIndex, operatorEvents), opFinished);
    fail(format("Operator %s[%d] was finished but didn't finish the checkpoint after that;" + "checkpoints started after finish: %s (events (excluding watermarks): %s)", operatorId, subtaskIndex, finalCheckpointCandidates, operatorEvents.stream().filter(ev -> !(ev instanceof WatermarkReceivedEvent)).collect(toList())));
}
Also used : CheckpointStartedEvent(org.apache.flink.runtime.operators.lifecycle.event.CheckpointStartedEvent) CheckpointStartedEvent(org.apache.flink.runtime.operators.lifecycle.event.CheckpointStartedEvent) TestJobWithDescription(org.apache.flink.runtime.operators.lifecycle.TestJobWithDescription) WatermarkReceivedEvent(org.apache.flink.runtime.operators.lifecycle.event.WatermarkReceivedEvent) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) String.format(java.lang.String.format) TestEvent(org.apache.flink.runtime.operators.lifecycle.event.TestEvent) StreamOperator(org.apache.flink.streaming.api.operators.StreamOperator) HashSet(java.util.HashSet) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) OperatorFinishedEvent(org.apache.flink.runtime.operators.lifecycle.event.OperatorFinishedEvent) CheckpointCompletedEvent(org.apache.flink.runtime.operators.lifecycle.event.CheckpointCompletedEvent) Assert.fail(org.junit.Assert.fail) OperatorFinishedEvent(org.apache.flink.runtime.operators.lifecycle.event.OperatorFinishedEvent) TestEvent(org.apache.flink.runtime.operators.lifecycle.event.TestEvent) WatermarkReceivedEvent(org.apache.flink.runtime.operators.lifecycle.event.WatermarkReceivedEvent) CheckpointCompletedEvent(org.apache.flink.runtime.operators.lifecycle.event.CheckpointCompletedEvent) HashSet(java.util.HashSet)

Example 4 with TestEvent

use of org.apache.flink.runtime.operators.lifecycle.event.TestEvent in project flink by apache.

the class TestJobExecutor method waitForFailover.

private void waitForFailover(BlockingQueue<TestEvent> queue) throws Exception {
    int timeoutMs = 10_000;
    Deadline deadline = Deadline.fromNow(Duration.ofMillis(timeoutMs));
    String operatorId = null;
    int subtaskId = -1;
    int attemptNumber = -1;
    while (deadline.hasTimeLeft()) {
        TestEvent e = queue.poll(deadline.timeLeft().toMillis(), MILLISECONDS);
        if (e instanceof TestCommandAckEvent) {
            TestCommandAckEvent ack = (TestCommandAckEvent) e;
            if (ack.getCommand() == FAIL) {
                operatorId = ack.operatorId;
                subtaskId = ack.subtaskIndex;
                attemptNumber = ack.getAttemptNumber();
            }
        } else if (e instanceof OperatorStartedEvent && operatorId != null) {
            OperatorStartedEvent started = (OperatorStartedEvent) e;
            if (started.operatorId.equals(operatorId) && started.subtaskIndex == subtaskId && started.getAttemptNumber() >= attemptNumber) {
                return;
            }
        }
    }
    throw new TimeoutException("No subtask restarted in " + timeoutMs + "ms");
}
Also used : TestEvent(org.apache.flink.runtime.operators.lifecycle.event.TestEvent) Deadline(org.apache.flink.api.common.time.Deadline) TestCommandAckEvent(org.apache.flink.runtime.operators.lifecycle.event.TestCommandAckEvent) OperatorStartedEvent(org.apache.flink.runtime.operators.lifecycle.event.OperatorStartedEvent) TimeoutException(java.util.concurrent.TimeoutException)

Example 5 with TestEvent

use of org.apache.flink.runtime.operators.lifecycle.event.TestEvent in project flink by apache.

the class DrainingValidator method validateOperatorLifecycle.

@Override
public void validateOperatorLifecycle(TestJobWithDescription job, String operatorId, int subtaskIndex, List<TestEvent> operatorEvents) {
    Map<Integer, List<TestEvent>> byAttempt = new HashMap<>();
    Set<Integer> normallyFinishedAttempts = new HashSet<>();
    int lastAttempt = Integer.MIN_VALUE;
    for (TestEvent e : operatorEvents) {
        byAttempt.computeIfAbsent(e.attemptNumber, ign -> new ArrayList<>()).add(e);
        if (isFinishAck(e)) {
            normallyFinishedAttempts.add(e.attemptNumber);
        }
        lastAttempt = Math.max(lastAttempt, e.attemptNumber);
    }
    for (Map.Entry<Integer, List<TestEvent>> entry : byAttempt.entrySet()) {
        // Skip if this or other task from this attempt failed.
        if (lastAttempt == entry.getKey() || normallyFinishedAttempts.contains(entry.getKey())) {
            validateSubtaskAttempt(job, operatorId, subtaskIndex, entry.getValue());
        }
    }
}
Also used : TestCommand(org.apache.flink.runtime.operators.lifecycle.command.TestCommand) JobVertex(org.apache.flink.runtime.jobgraph.JobVertex) TestJobWithDescription(org.apache.flink.runtime.operators.lifecycle.TestJobWithDescription) WatermarkReceivedEvent(org.apache.flink.runtime.operators.lifecycle.event.WatermarkReceivedEvent) Watermark(org.apache.flink.streaming.api.watermark.Watermark) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) HashMap(java.util.HashMap) OperatorIDPair(org.apache.flink.runtime.OperatorIDPair) String.format(java.lang.String.format) InputEndedEvent(org.apache.flink.runtime.operators.lifecycle.event.InputEndedEvent) TestEvent(org.apache.flink.runtime.operators.lifecycle.event.TestEvent) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) List(java.util.List) Assert.assertFalse(org.junit.Assert.assertFalse) Map(java.util.Map) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) BitSet(java.util.BitSet) NoSuchElementException(java.util.NoSuchElementException) TestCommandAckEvent(org.apache.flink.runtime.operators.lifecycle.event.TestCommandAckEvent) Assert.assertEquals(org.junit.Assert.assertEquals) HashMap(java.util.HashMap) TestEvent(org.apache.flink.runtime.operators.lifecycle.event.TestEvent) ArrayList(java.util.ArrayList) ArrayList(java.util.ArrayList) List(java.util.List) HashMap(java.util.HashMap) Map(java.util.Map) HashSet(java.util.HashSet)

Aggregations

TestEvent (org.apache.flink.runtime.operators.lifecycle.event.TestEvent)7 TestJobWithDescription (org.apache.flink.runtime.operators.lifecycle.TestJobWithDescription)4 String.format (java.lang.String.format)3 HashMap (java.util.HashMap)3 List (java.util.List)3 Map (java.util.Map)3 WatermarkReceivedEvent (org.apache.flink.runtime.operators.lifecycle.event.WatermarkReceivedEvent)3 Assert.assertTrue (org.junit.Assert.assertTrue)3 ArrayList (java.util.ArrayList)2 BitSet (java.util.BitSet)2 HashSet (java.util.HashSet)2 Set (java.util.Set)2 OperatorIDPair (org.apache.flink.runtime.OperatorIDPair)2 JobVertex (org.apache.flink.runtime.jobgraph.JobVertex)2 CheckpointCompletedEvent (org.apache.flink.runtime.operators.lifecycle.event.CheckpointCompletedEvent)2 CheckpointStartedEvent (org.apache.flink.runtime.operators.lifecycle.event.CheckpointStartedEvent)2 InputEndedEvent (org.apache.flink.runtime.operators.lifecycle.event.InputEndedEvent)2 OperatorFinishedEvent (org.apache.flink.runtime.operators.lifecycle.event.OperatorFinishedEvent)2 TestCommandAckEvent (org.apache.flink.runtime.operators.lifecycle.event.TestCommandAckEvent)2 Collection (java.util.Collection)1