Examples with MailboxExecutor org.apache.flink.api.common.operators.MailboxExecutor used on opensource projects

Example 6 with MailboxExecutor

use of org.apache.flink.api.common.operators.MailboxExecutor in project flink by apache.

the class StreamTaskTestHarness method waitForInputProcessing.

/**
 * This only returns after all input queues are empty.
 */
public void waitForInputProcessing() throws Exception {
    while (taskThread.isAlive()) {
        boolean allEmpty = true;
        for (int i = 0; i < numInputGates; i++) {
            if (!inputGates[i].allQueuesEmpty()) {
                allEmpty = false;
            }
        }
        if (allEmpty) {
            break;
        }
    }
    // Wait for all currently available input has been processed.
    final AtomicBoolean allInputProcessed = new AtomicBoolean();
    final MailboxProcessor mailboxProcessor = taskThread.task.mailboxProcessor;
    final MailboxExecutor mailboxExecutor = mailboxProcessor.getMainMailboxExecutor();
    while (taskThread.isAlive()) {
        try {
            final CountDownLatch latch = new CountDownLatch(1);
            mailboxExecutor.execute(() -> {
                allInputProcessed.set(!mailboxProcessor.isDefaultActionAvailable());
                latch.countDown();
            }, "query-whether-processInput-has-suspend-itself");
            // Mail could be dropped due to task exception, so we do timed-await here.
            latch.await(1, TimeUnit.SECONDS);
        } catch (RejectedExecutionException ex) {
        // Loop until task thread exit for possible task exception.
        }
        if (allInputProcessed.get()) {
            break;
        }
        try {
            Thread.sleep(1);
        } catch (InterruptedException ignored) {
        }
    }
    Throwable error = taskThread.getError();
    if (error != null) {
        throw new Exception("Exception in the task thread", error);
    }
}

Example 7 with MailboxExecutor

use of org.apache.flink.api.common.operators.MailboxExecutor in project flink by apache.

the class TaskMailboxProcessorTest method testAvoidStarvation.

/**
 * FLINK-14304: Avoid newly spawned letters to prevent input processing from ever happening.
 */
@Test
public void testAvoidStarvation() throws Exception {
    final int expectedInvocations = 3;
    final AtomicInteger counter = new AtomicInteger(0);
    MailboxThread mailboxThread = new MailboxThread() {

        @Override
        public void runDefaultAction(Controller controller) {
            if (counter.incrementAndGet() == expectedInvocations) {
                controller.allActionsCompleted();
            }
        }
    };
    mailboxThread.start();
    final MailboxProcessor mailboxProcessor = mailboxThread.getMailboxProcessor();
    final MailboxExecutor mailboxExecutor = mailboxProcessor.getMailboxExecutor(DEFAULT_PRIORITY);
    AtomicInteger index = new AtomicInteger();
    mailboxExecutor.execute(new RunnableWithException() {

        @Override
        public void run() {
            mailboxExecutor.execute(this, "Blocking mail" + index.incrementAndGet());
        }
    }, "Blocking mail" + index.get());
    mailboxThread.signalStart();
    mailboxThread.join();
    Assert.assertEquals(expectedInvocations, counter.get());
    Assert.assertEquals(expectedInvocations, index.get());
}

Example 8 with MailboxExecutor

use of org.apache.flink.api.common.operators.MailboxExecutor in project flink by apache.

the class MailboxOperatorTest method createReplicatingMail.

@Nonnull
private ReplicatingMail createReplicatingMail(int numRecords, StreamTaskMailboxTestHarness<Integer> testHarness, int priority) {
    final MailboxExecutor mailboxExecutor = testHarness.getExecutor(priority);
    final ReplicatingMail mail1 = new ReplicatingMail(mailboxExecutor, numRecords + 1);
    mailboxExecutor.submit(mail1, "Initial mail");
    return mail1;
}

Example 9 with MailboxExecutor

use of org.apache.flink.api.common.operators.MailboxExecutor in project flink by apache.

the class StreamOperatorFactoryUtil method createOperator.

/**
 * Creates a new operator using a factory and makes sure that all special factory traits are
 * properly handled.
 *
 * @param operatorFactory the operator factory.
 * @param containingTask the containing task.
 * @param configuration the configuration of the operator.
 * @param output the output of the operator.
 * @param operatorEventDispatcher the operator event dispatcher for communication between
 *     operator and coordinators.
 * @return a newly created and configured operator, and the {@link ProcessingTimeService}
 *     instance it can access.
 */
public static <OUT, OP extends StreamOperator<OUT>> Tuple2<OP, Optional<ProcessingTimeService>> createOperator(StreamOperatorFactory<OUT> operatorFactory, StreamTask<OUT, ?> containingTask, StreamConfig configuration, Output<StreamRecord<OUT>> output, OperatorEventDispatcher operatorEventDispatcher) {
    MailboxExecutor mailboxExecutor = containingTask.getMailboxExecutorFactory().createExecutor(configuration.getChainIndex());
    if (operatorFactory instanceof YieldingOperatorFactory) {
        ((YieldingOperatorFactory<?>) operatorFactory).setMailboxExecutor(mailboxExecutor);
    }
    final Supplier<ProcessingTimeService> processingTimeServiceFactory = () -> containingTask.getProcessingTimeServiceFactory().createProcessingTimeService(mailboxExecutor);
    final ProcessingTimeService processingTimeService;
    if (operatorFactory instanceof ProcessingTimeServiceAware) {
        processingTimeService = processingTimeServiceFactory.get();
        ((ProcessingTimeServiceAware) operatorFactory).setProcessingTimeService(processingTimeService);
    } else {
        processingTimeService = null;
    }
    // TODO: what to do with ProcessingTimeServiceAware?
    OP op = operatorFactory.createStreamOperator(new StreamOperatorParameters<>(containingTask, configuration, output, processingTimeService != null ? () -> processingTimeService : processingTimeServiceFactory, operatorEventDispatcher));
    return new Tuple2<>(op, Optional.ofNullable(processingTimeService));
}