Search in sources :

Example 1 with TaskMailboxImpl

use of org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl in project flink by apache.

the class CheckpointedInputGateTest method testPriorityBeforeClose.

/**
 * Tests a priority notification happening right before cancellation. The mail would be
 * processed while draining mailbox but can't pull any data anymore.
 */
@Test
public void testPriorityBeforeClose() throws IOException, InterruptedException {
    NetworkBufferPool bufferPool = new NetworkBufferPool(10, 1024);
    try (Closer closer = Closer.create()) {
        closer.register(bufferPool::destroy);
        for (int repeat = 0; repeat < 100; repeat++) {
            setUp();
            SingleInputGate singleInputGate = new SingleInputGateBuilder().setNumberOfChannels(2).setBufferPoolFactory(bufferPool.createBufferPool(2, Integer.MAX_VALUE)).setSegmentProvider(bufferPool).setChannelFactory(InputChannelBuilder::buildRemoteChannel).build();
            singleInputGate.setup();
            ((RemoteInputChannel) singleInputGate.getChannel(0)).requestSubpartition();
            final TaskMailboxImpl mailbox = new TaskMailboxImpl();
            MailboxExecutorImpl mailboxExecutor = new MailboxExecutorImpl(mailbox, 0, StreamTaskActionExecutor.IMMEDIATE);
            ValidatingCheckpointHandler validatingHandler = new ValidatingCheckpointHandler(1);
            SingleCheckpointBarrierHandler barrierHandler = TestBarrierHandlerFactory.forTarget(validatingHandler).create(singleInputGate, new MockChannelStateWriter());
            CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(singleInputGate, barrierHandler, mailboxExecutor, UpstreamRecoveryTracker.forInputGate(singleInputGate));
            final int oldSize = mailbox.size();
            enqueue(checkpointedInputGate, 0, barrier(1));
            // wait for priority mail to be enqueued
            Deadline deadline = Deadline.fromNow(Duration.ofMinutes(1));
            while (deadline.hasTimeLeft() && oldSize >= mailbox.size()) {
                Thread.sleep(1);
            }
            // test the race condition
            // either priority event could be handled, then we expect a checkpoint to be
            // triggered or closing came first in which case we expect a CancelTaskException
            CountDownLatch beforeLatch = new CountDownLatch(2);
            final CheckedThread canceler = new CheckedThread("Canceler") {

                @Override
                public void go() throws IOException {
                    beforeLatch.countDown();
                    singleInputGate.close();
                }
            };
            canceler.start();
            beforeLatch.countDown();
            try {
                while (mailboxExecutor.tryYield()) {
                }
                assertEquals(1L, validatingHandler.triggeredCheckpointCounter);
            } catch (CancelTaskException e) {
            }
            canceler.join();
        }
    }
}
Also used : Closer(org.apache.flink.shaded.guava30.com.google.common.io.Closer) MockChannelStateWriter(org.apache.flink.runtime.checkpoint.channel.MockChannelStateWriter) MailboxExecutorImpl(org.apache.flink.streaming.runtime.tasks.mailbox.MailboxExecutorImpl) Deadline(org.apache.flink.api.common.time.Deadline) CountDownLatch(java.util.concurrent.CountDownLatch) CheckedThread(org.apache.flink.core.testutils.CheckedThread) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel) SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) TaskMailboxImpl(org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl) CancelTaskException(org.apache.flink.runtime.execution.CancelTaskException) Test(org.junit.Test)

Example 2 with TaskMailboxImpl

use of org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl in project flink by apache.

the class CheckpointedInputGateTest method setupInputGateWithAlternatingController.

private CheckpointedInputGate setupInputGateWithAlternatingController(int numberOfChannels, NetworkBufferPool networkBufferPool, AbstractInvokable abstractInvokable, RecordingChannelStateWriter stateWriter) throws Exception {
    ConnectionManager connectionManager = new TestingConnectionManager();
    SingleInputGate singleInputGate = new SingleInputGateBuilder().setBufferPoolFactory(networkBufferPool.createBufferPool(numberOfChannels, Integer.MAX_VALUE)).setSegmentProvider(networkBufferPool).setChannelFactory((builder, gate) -> builder.setConnectionManager(connectionManager).buildRemoteChannel(gate)).setNumberOfChannels(numberOfChannels).setChannelStateWriter(stateWriter).build();
    singleInputGate.setup();
    MailboxExecutorImpl mailboxExecutor = new MailboxExecutorImpl(new TaskMailboxImpl(), 0, StreamTaskActionExecutor.IMMEDIATE);
    SingleCheckpointBarrierHandler barrierHandler = TestBarrierHandlerFactory.forTarget(abstractInvokable).create(singleInputGate, stateWriter);
    CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(singleInputGate, barrierHandler, mailboxExecutor, UpstreamRecoveryTracker.forInputGate(singleInputGate));
    for (int i = 0; i < numberOfChannels; i++) {
        ((RemoteInputChannel) checkpointedInputGate.getChannel(i)).requestSubpartition();
    }
    return checkpointedInputGate;
}
Also used : Deadline(org.apache.flink.api.common.time.Deadline) TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) SystemClock(org.apache.flink.util.clock.SystemClock) EndOfPartitionEvent(org.apache.flink.runtime.io.network.api.EndOfPartitionEvent) HashMap(java.util.HashMap) PartitionRequestClient(org.apache.flink.runtime.io.network.PartitionRequestClient) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) CheckedThread(org.apache.flink.core.testutils.CheckedThread) Duration(java.time.Duration) TaskMailboxImpl(org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl) BufferBuilderTestUtils.buildSomeBuffer(org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.buildSomeBuffer) InputChannelInfo(org.apache.flink.runtime.checkpoint.channel.InputChannelInfo) MockChannelStateWriter(org.apache.flink.runtime.checkpoint.channel.MockChannelStateWriter) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) Before(org.junit.Before) CheckpointStorageLocationReference(org.apache.flink.runtime.state.CheckpointStorageLocationReference) EndOfChannelStateEvent(org.apache.flink.runtime.io.network.partition.consumer.EndOfChannelStateEvent) CheckpointType(org.apache.flink.runtime.checkpoint.CheckpointType) SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) CancelTaskException(org.apache.flink.runtime.execution.CancelTaskException) EventSerializer(org.apache.flink.runtime.io.network.api.serialization.EventSerializer) AbstractEvent(org.apache.flink.runtime.event.AbstractEvent) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel) ConnectionManager(org.apache.flink.runtime.io.network.ConnectionManager) Closer(org.apache.flink.shaded.guava30.com.google.common.io.Closer) InputChannelBuilder(org.apache.flink.runtime.io.network.partition.consumer.InputChannelBuilder) AbstractInvokable(org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable) Assert.assertTrue(org.junit.Assert.assertTrue) ConnectionID(org.apache.flink.runtime.io.network.ConnectionID) Test(org.junit.Test) IOException(java.io.IOException) CheckpointOptions(org.apache.flink.runtime.checkpoint.CheckpointOptions) MailboxExecutorImpl(org.apache.flink.streaming.runtime.tasks.mailbox.MailboxExecutorImpl) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) CountDownLatch(java.util.concurrent.CountDownLatch) RecordingChannelStateWriter(org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter) CheckpointBarrier(org.apache.flink.runtime.io.network.api.CheckpointBarrier) Assert.assertFalse(org.junit.Assert.assertFalse) Optional(java.util.Optional) TestingPartitionRequestClient(org.apache.flink.runtime.io.network.TestingPartitionRequestClient) DummyEnvironment(org.apache.flink.runtime.operators.testutils.DummyEnvironment) BufferOrEvent(org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent) Assert.assertEquals(org.junit.Assert.assertEquals) StreamTaskActionExecutor(org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor) SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) MailboxExecutorImpl(org.apache.flink.streaming.runtime.tasks.mailbox.MailboxExecutorImpl) TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) ConnectionManager(org.apache.flink.runtime.io.network.ConnectionManager) TaskMailboxImpl(org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl) TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel)

Example 3 with TaskMailboxImpl

use of org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl in project flink by apache.

the class CheckpointedInputGateTest method setupInputGate.

private CheckpointedInputGate setupInputGate(int numberOfChannels, NetworkBufferPool networkBufferPool, ConnectionManager connectionManager) throws Exception {
    SingleInputGate singleInputGate = new SingleInputGateBuilder().setBufferPoolFactory(networkBufferPool.createBufferPool(numberOfChannels, Integer.MAX_VALUE)).setSegmentProvider(networkBufferPool).setChannelFactory((builder, gate) -> builder.setConnectionManager(connectionManager).buildRemoteChannel(gate)).setNumberOfChannels(numberOfChannels).build();
    singleInputGate.setup();
    MailboxExecutorImpl mailboxExecutor = new MailboxExecutorImpl(new TaskMailboxImpl(), 0, StreamTaskActionExecutor.IMMEDIATE);
    CheckpointBarrierTracker barrierHandler = new CheckpointBarrierTracker(numberOfChannels, new AbstractInvokable(new DummyEnvironment()) {

        @Override
        public void invoke() {
        }
    }, SystemClock.getInstance(), true);
    CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(singleInputGate, barrierHandler, mailboxExecutor, UpstreamRecoveryTracker.forInputGate(singleInputGate));
    for (int i = 0; i < numberOfChannels; i++) {
        ((RemoteInputChannel) checkpointedInputGate.getChannel(i)).requestSubpartition();
    }
    return checkpointedInputGate;
}
Also used : SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) MailboxExecutorImpl(org.apache.flink.streaming.runtime.tasks.mailbox.MailboxExecutorImpl) TaskMailboxImpl(org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl) DummyEnvironment(org.apache.flink.runtime.operators.testutils.DummyEnvironment) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) AbstractInvokable(org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable) RemoteInputChannel(org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel)

Example 4 with TaskMailboxImpl

use of org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl in project flink by apache.

the class StreamTaskExecutionDecorationTest method before.

@Before
public void before() throws Exception {
    mailbox = new TaskMailboxImpl();
    decorator = new CountingStreamTaskActionExecutor();
    task = new StreamTask<Object, StreamOperator<Object>>(new DeclineDummyEnvironment(), null, FatalExitExceptionHandler.INSTANCE, decorator, mailbox) {

        @Override
        protected void init() {
        }

        @Override
        protected void processInput(MailboxDefaultAction.Controller controller) {
        }
    };
    task.operatorChain = new RegularOperatorChain<>(task, new NonRecordWriter<>());
}
Also used : TaskMailboxImpl(org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl) MailboxDefaultAction(org.apache.flink.streaming.runtime.tasks.mailbox.MailboxDefaultAction) NonRecordWriter(org.apache.flink.runtime.io.network.api.writer.NonRecordWriter) StreamOperator(org.apache.flink.streaming.api.operators.StreamOperator) Before(org.junit.Before)

Aggregations

TaskMailboxImpl (org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl)4 RemoteInputChannel (org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel)3 SingleInputGate (org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate)3 SingleInputGateBuilder (org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder)3 MailboxExecutorImpl (org.apache.flink.streaming.runtime.tasks.mailbox.MailboxExecutorImpl)3 CountDownLatch (java.util.concurrent.CountDownLatch)2 Deadline (org.apache.flink.api.common.time.Deadline)2 CheckedThread (org.apache.flink.core.testutils.CheckedThread)2 MockChannelStateWriter (org.apache.flink.runtime.checkpoint.channel.MockChannelStateWriter)2 CancelTaskException (org.apache.flink.runtime.execution.CancelTaskException)2 NetworkBufferPool (org.apache.flink.runtime.io.network.buffer.NetworkBufferPool)2 AbstractInvokable (org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable)2 DummyEnvironment (org.apache.flink.runtime.operators.testutils.DummyEnvironment)2 Closer (org.apache.flink.shaded.guava30.com.google.common.io.Closer)2 Test (org.junit.Test)2 IOException (java.io.IOException)1 Duration (java.time.Duration)1 HashMap (java.util.HashMap)1 Optional (java.util.Optional)1 CheckpointOptions (org.apache.flink.runtime.checkpoint.CheckpointOptions)1