Examples with MailboxExecutor org.apache.flink.api.common.operators.MailboxExecutor used on opensource projects

Example 1 with MailboxExecutor

use of org.apache.flink.api.common.operators.MailboxExecutor in project flink by apache.

the class StreamTaskTest method testQuiesceOfMailboxRightBeforeSubmittingActionViaTimerService.

@Test
public void testQuiesceOfMailboxRightBeforeSubmittingActionViaTimerService() throws Exception {
    // given: the stream task with configured handle async exception.
    AtomicBoolean submitThroughputFail = new AtomicBoolean();
    MockEnvironment mockEnvironment = new MockEnvironmentBuilder().build();
    final UnAvailableTestInputProcessor inputProcessor = new UnAvailableTestInputProcessor();
    RunningTask<StreamTask<?, ?>> task = runTask(() -> new MockStreamTaskBuilder(mockEnvironment).setHandleAsyncException((str, t) -> submitThroughputFail.set(true)).setStreamInputProcessor(inputProcessor).build());
    waitTaskIsRunning(task.streamTask, task.invocationFuture);
    TimerService timerService = task.streamTask.systemTimerService;
    MailboxExecutor mainMailboxExecutor = task.streamTask.mailboxProcessor.getMainMailboxExecutor();
    CountDownLatch stoppingMailboxLatch = new CountDownLatch(1);
    timerService.registerTimer(timerService.getCurrentProcessingTime(), (time) -> {
        stoppingMailboxLatch.await();
        // The time to the start 'afterInvoke' inside of mailbox.
        // 'afterInvoke' won't finish until this execution won't finish so it is
        // impossible to wait on latch or something else.
        Thread.sleep(5);
        mainMailboxExecutor.submit(() -> {
        }, "test");
    });
    // when: Calling the quiesce for mailbox and finishing the timer service.
    mainMailboxExecutor.submit(() -> {
        stoppingMailboxLatch.countDown();
        task.streamTask.afterInvoke();
    }, "test").get();
    // then: the exception handle wasn't invoked because the such situation is expected.
    assertFalse(submitThroughputFail.get());
    // Correctly shutdown the stream task to avoid hanging.
    inputProcessor.availabilityProvider.getUnavailableToResetAvailable().complete(null);
}

Example 2 with MailboxExecutor

use of org.apache.flink.api.common.operators.MailboxExecutor in project flink by apache.

the class StreamTaskTest method testProcessWithUnAvailableOutput.

@Test
public void testProcessWithUnAvailableOutput() throws Exception {
    final long sleepTimeOutsideMail = 42;
    final long sleepTimeInsideMail = 44;
    @Nullable WaitingThread waitingThread = null;
    try (final MockEnvironment environment = setupEnvironment(true, false)) {
        final int numberOfProcessCalls = 10;
        final AvailabilityTestInputProcessor inputProcessor = new AvailabilityTestInputProcessor(numberOfProcessCalls);
        final StreamTask task = new MockStreamTaskBuilder(environment).setStreamInputProcessor(inputProcessor).build();
        final MailboxExecutor executor = task.mailboxProcessor.getMainMailboxExecutor();
        TaskIOMetricGroup ioMetricGroup = task.getEnvironment().getMetricGroup().getIOMetricGroup();
        final RunnableWithException completeFutureTask = () -> {
            assertEquals(1, inputProcessor.currentNumProcessCalls);
            assertFalse(task.mailboxProcessor.isDefaultActionAvailable());
            environment.getWriter(1).getAvailableFuture().complete(null);
        };
        waitingThread = new WaitingThread(executor, completeFutureTask, sleepTimeInsideMail, sleepTimeOutsideMail, ioMetricGroup.getSoftBackPressuredTimePerSecond());
        // Make sure WaitingThread is started after Task starts processing.
        executor.submit(waitingThread::start, "This task will submit another task to execute after processing input once.");
        long startTs = System.currentTimeMillis();
        task.invoke();
        long totalDuration = System.currentTimeMillis() - startTs;
        assertThat(ioMetricGroup.getSoftBackPressuredTimePerSecond().getCount(), greaterThanOrEqualTo(sleepTimeOutsideMail));
        assertThat(ioMetricGroup.getSoftBackPressuredTimePerSecond().getCount(), Matchers.lessThanOrEqualTo(totalDuration - sleepTimeInsideMail));
        assertThat(ioMetricGroup.getIdleTimeMsPerSecond().getCount(), is(0L));
        assertEquals(numberOfProcessCalls, inputProcessor.currentNumProcessCalls);
    } finally {
        if (waitingThread != null) {
            waitingThread.join();
        }
    }
}

Example 3 with MailboxExecutor

use of org.apache.flink.api.common.operators.MailboxExecutor in project flink by apache.

the class StreamOperatorWrapperTest method setup.

@Before
public void setup() throws Exception {
    this.operatorWrappers = new ArrayList<>();
    this.output = new ConcurrentLinkedQueue<>();
    try (MockEnvironment env = MockEnvironment.builder().build()) {
        this.containingTask = new MockStreamTaskBuilder(env).build();
        // initialize operator wrappers
        for (int i = 0; i < numOperators; i++) {
            MailboxExecutor mailboxExecutor = containingTask.getMailboxExecutorFactory().createExecutor(i);
            TimerMailController timerMailController = new TimerMailController(containingTask, mailboxExecutor);
            ProcessingTimeServiceImpl processingTimeService = new ProcessingTimeServiceImpl(timerService, timerMailController::wrapCallback);
            TestOneInputStreamOperator streamOperator = new TestOneInputStreamOperator("Operator" + i, output, processingTimeService, mailboxExecutor, timerMailController);
            streamOperator.setProcessingTimeService(processingTimeService);
            StreamOperatorWrapper<?, ?> operatorWrapper = new StreamOperatorWrapper<>(streamOperator, Optional.ofNullable(streamOperator.getProcessingTimeService()), mailboxExecutor, i == 0);
            operatorWrappers.add(operatorWrapper);
        }
        StreamOperatorWrapper<?, ?> previous = null;
        for (StreamOperatorWrapper<?, ?> current : operatorWrappers) {
            if (previous != null) {
                previous.setNext(current);
            }
            current.setPrevious(previous);
            previous = current;
        }
    }
}

Example 4 with MailboxExecutor

use of org.apache.flink.api.common.operators.MailboxExecutor in project flink by apache.

the class StreamTaskTest method testProcessWithUnAvailableInput.

@Test
public void testProcessWithUnAvailableInput() throws Exception {
    final long sleepTimeOutsideMail = 42;
    final long sleepTimeInsideMail = 44;
    @Nullable WaitingThread waitingThread = null;
    try (final MockEnvironment environment = setupEnvironment(true, true)) {
        final UnAvailableTestInputProcessor inputProcessor = new UnAvailableTestInputProcessor();
        final StreamTask task = new MockStreamTaskBuilder(environment).setStreamInputProcessor(inputProcessor).build();
        TaskIOMetricGroup ioMetricGroup = task.getEnvironment().getMetricGroup().getIOMetricGroup();
        final MailboxExecutor executor = task.mailboxProcessor.getMainMailboxExecutor();
        final RunnableWithException completeFutureTask = () -> {
            inputProcessor.availabilityProvider.getUnavailableToResetAvailable().complete(null);
        };
        waitingThread = new WaitingThread(executor, completeFutureTask, sleepTimeInsideMail, sleepTimeOutsideMail, ioMetricGroup.getIdleTimeMsPerSecond());
        // Make sure WaitingThread is started after Task starts processing.
        executor.submit(waitingThread::start, "Start WaitingThread after Task starts processing input.");
        SystemClock clock = SystemClock.getInstance();
        long startTs = clock.absoluteTimeMillis();
        task.invoke();
        long totalDuration = clock.absoluteTimeMillis() - startTs;
        assertThat(ioMetricGroup.getIdleTimeMsPerSecond().getCount(), greaterThanOrEqualTo(sleepTimeOutsideMail));
        assertThat(ioMetricGroup.getIdleTimeMsPerSecond().getCount(), Matchers.lessThanOrEqualTo(totalDuration - sleepTimeInsideMail));
        assertThat(ioMetricGroup.getSoftBackPressuredTimePerSecond().getCount(), is(0L));
        assertThat(ioMetricGroup.getHardBackPressuredTimePerSecond().getCount(), is(0L));
    } finally {
        if (waitingThread != null) {
            waitingThread.join();
        }
    }
}

Example 5 with MailboxExecutor

use of org.apache.flink.api.common.operators.MailboxExecutor in project flink by apache.

the class StreamTaskTest method testSyncSavepointWithEndInput.

/**
 * Test for SyncSavepoint and EndInput interactions. Targets following scenarios scenarios:
 *
 * <ol>
 *   <li>Thread1: notify sync savepoint
 *   <li>Thread2: endInput
 *   <li>Thread1: confirm/abort/abortAsync
 *   <li>assert inputEnded: confirmed - no, abort/abortAsync - yes
 * </ol>
 */
private void testSyncSavepointWithEndInput(BiConsumerWithException<StreamTask<?, ?>, Long, IOException> savepointResult, SnapshotType checkpointType, boolean expectEndInput) throws Exception {
    StreamTaskMailboxTestHarness<String> harness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, STRING_TYPE_INFO).addInput(STRING_TYPE_INFO).setupOutputForSingletonOperatorChain(new TestBoundedOneInputStreamOperator()).build();
    final long checkpointId = 1L;
    CountDownLatch savepointTriggeredLatch = new CountDownLatch(1);
    CountDownLatch inputEndedLatch = new CountDownLatch(1);
    MailboxExecutor executor = harness.streamTask.getMailboxExecutorFactory().createExecutor(MAX_PRIORITY);
    executor.execute(() -> {
        try {
            harness.streamTask.triggerCheckpointOnBarrier(new CheckpointMetaData(checkpointId, checkpointId), new CheckpointOptions(checkpointType, getDefault()), new CheckpointMetricsBuilder());
        } catch (IOException e) {
            fail(e.getMessage());
        }
    }, "triggerCheckpointOnBarrier");
    new Thread(() -> {
        try {
            savepointTriggeredLatch.await();
            harness.endInput(expectEndInput);
            inputEndedLatch.countDown();
        } catch (InterruptedException e) {
            fail(e.getMessage());
        }
    }).start();
    // this mails should be executed from the one above (from triggerCheckpointOnBarrier)
    executor.execute(savepointTriggeredLatch::countDown, "savepointTriggeredLatch");
    executor.execute(() -> {
        inputEndedLatch.await();
        savepointResult.accept(harness.streamTask, checkpointId);
    }, "savepointResult");
    harness.processAll();
    Assert.assertEquals(expectEndInput, TestBoundedOneInputStreamOperator.isInputEnded());
}