Examples with MailboxProcessor org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor used on opensource projects

Example 1 with MailboxProcessor

use of org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor in project flink by apache.

the class StreamTask method afterInvoke.

protected void afterInvoke() throws Exception {
    LOG.debug("Finished task {}", getName());
    getCompletionFuture().exceptionally(unused -> null).join();
    Set<CompletableFuture<Void>> terminationConditions = new HashSet<>();
    // continue perform checkpoints.
    if (endOfDataReceived && areCheckpointsWithFinishedTasksEnabled()) {
        LOG.debug("Waiting for all the records processed by the downstream tasks.");
        for (ResultPartitionWriter partitionWriter : getEnvironment().getAllWriters()) {
            terminationConditions.add(partitionWriter.getAllDataProcessedFuture());
        }
        terminationConditions.add(finalCheckpointCompleted);
    }
    if (syncSavepoint != null) {
        terminationConditions.add(finalCheckpointCompleted);
    }
    FutureUtils.waitForAll(terminationConditions).thenRun(mailboxProcessor::allActionsCompleted);
    // Resumes the mailbox processor. The mailbox processor would be completed
    // after all records are processed by the downstream tasks.
    mailboxProcessor.runMailboxLoop();
    // make sure no further checkpoint and notification actions happen.
    // at the same time, this makes sure that during any "regular" exit where still
    actionExecutor.runThrowing(() -> {
        // make sure no new timers can come
        timerService.quiesce().get();
        systemTimerService.quiesce().get();
        // let mailbox execution reject all new letters from this point
        mailboxProcessor.prepareClose();
    });
    // processes the remaining mails; no new mails can be enqueued
    mailboxProcessor.drain();
    // Set isRunning to false after all the mails are drained so that
    // the queued checkpoint requirements could be triggered normally.
    actionExecutor.runThrowing(() -> {
        // only set the StreamTask to not running after all operators have been
        // finished!
        // See FLINK-7430
        isRunning = false;
    });
    LOG.debug("Finished operators for task {}", getName());
    // make sure all buffered data is flushed
    operatorChain.flushOutputs();
    if (areCheckpointsWithFinishedTasksEnabled()) {
        // No new checkpoints could be triggered since mailbox has been drained.
        subtaskCheckpointCoordinator.waitForPendingCheckpoints();
        LOG.debug("All pending checkpoints are finished");
    }
    disableInterruptOnCancel();
    // make an attempt to dispose the operators such that failures in the dispose call
    // still let the computation fail
    closeAllOperators();
}

Example 2 with MailboxProcessor

use of org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor in project flink by apache.

the class StreamTaskTestHarness method waitForInputProcessing.

/**
 * This only returns after all input queues are empty.
 */
public void waitForInputProcessing() throws Exception {
    while (taskThread.isAlive()) {
        boolean allEmpty = true;
        for (int i = 0; i < numInputGates; i++) {
            if (!inputGates[i].allQueuesEmpty()) {
                allEmpty = false;
            }
        }
        if (allEmpty) {
            break;
        }
    }
    // Wait for all currently available input has been processed.
    final AtomicBoolean allInputProcessed = new AtomicBoolean();
    final MailboxProcessor mailboxProcessor = taskThread.task.mailboxProcessor;
    final MailboxExecutor mailboxExecutor = mailboxProcessor.getMainMailboxExecutor();
    while (taskThread.isAlive()) {
        try {
            final CountDownLatch latch = new CountDownLatch(1);
            mailboxExecutor.execute(() -> {
                allInputProcessed.set(!mailboxProcessor.isDefaultActionAvailable());
                latch.countDown();
            }, "query-whether-processInput-has-suspend-itself");
            // Mail could be dropped due to task exception, so we do timed-await here.
            latch.await(1, TimeUnit.SECONDS);
        } catch (RejectedExecutionException ex) {
        // Loop until task thread exit for possible task exception.
        }
        if (allInputProcessed.get()) {
            break;
        }
        try {
            Thread.sleep(1);
        } catch (InterruptedException ignored) {
        }
    }
    Throwable error = taskThread.getError();
    if (error != null) {
        throw new Exception("Exception in the task thread", error);
    }
}