use of org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl in project flink by apache.
the class CheckpointedInputGateTest method testPriorityBeforeClose.
/**
* Tests a priority notification happening right before cancellation. The mail would be
* processed while draining mailbox but can't pull any data anymore.
*/
@Test
public void testPriorityBeforeClose() throws IOException, InterruptedException {
NetworkBufferPool bufferPool = new NetworkBufferPool(10, 1024);
try (Closer closer = Closer.create()) {
closer.register(bufferPool::destroy);
for (int repeat = 0; repeat < 100; repeat++) {
setUp();
SingleInputGate singleInputGate = new SingleInputGateBuilder().setNumberOfChannels(2).setBufferPoolFactory(bufferPool.createBufferPool(2, Integer.MAX_VALUE)).setSegmentProvider(bufferPool).setChannelFactory(InputChannelBuilder::buildRemoteChannel).build();
singleInputGate.setup();
((RemoteInputChannel) singleInputGate.getChannel(0)).requestSubpartition();
final TaskMailboxImpl mailbox = new TaskMailboxImpl();
MailboxExecutorImpl mailboxExecutor = new MailboxExecutorImpl(mailbox, 0, StreamTaskActionExecutor.IMMEDIATE);
ValidatingCheckpointHandler validatingHandler = new ValidatingCheckpointHandler(1);
SingleCheckpointBarrierHandler barrierHandler = TestBarrierHandlerFactory.forTarget(validatingHandler).create(singleInputGate, new MockChannelStateWriter());
CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(singleInputGate, barrierHandler, mailboxExecutor, UpstreamRecoveryTracker.forInputGate(singleInputGate));
final int oldSize = mailbox.size();
enqueue(checkpointedInputGate, 0, barrier(1));
// wait for priority mail to be enqueued
Deadline deadline = Deadline.fromNow(Duration.ofMinutes(1));
while (deadline.hasTimeLeft() && oldSize >= mailbox.size()) {
Thread.sleep(1);
}
// test the race condition
// either priority event could be handled, then we expect a checkpoint to be
// triggered or closing came first in which case we expect a CancelTaskException
CountDownLatch beforeLatch = new CountDownLatch(2);
final CheckedThread canceler = new CheckedThread("Canceler") {
@Override
public void go() throws IOException {
beforeLatch.countDown();
singleInputGate.close();
}
};
canceler.start();
beforeLatch.countDown();
try {
while (mailboxExecutor.tryYield()) {
}
assertEquals(1L, validatingHandler.triggeredCheckpointCounter);
} catch (CancelTaskException e) {
}
canceler.join();
}
}
}
use of org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl in project flink by apache.
the class CheckpointedInputGateTest method setupInputGateWithAlternatingController.
private CheckpointedInputGate setupInputGateWithAlternatingController(int numberOfChannels, NetworkBufferPool networkBufferPool, AbstractInvokable abstractInvokable, RecordingChannelStateWriter stateWriter) throws Exception {
ConnectionManager connectionManager = new TestingConnectionManager();
SingleInputGate singleInputGate = new SingleInputGateBuilder().setBufferPoolFactory(networkBufferPool.createBufferPool(numberOfChannels, Integer.MAX_VALUE)).setSegmentProvider(networkBufferPool).setChannelFactory((builder, gate) -> builder.setConnectionManager(connectionManager).buildRemoteChannel(gate)).setNumberOfChannels(numberOfChannels).setChannelStateWriter(stateWriter).build();
singleInputGate.setup();
MailboxExecutorImpl mailboxExecutor = new MailboxExecutorImpl(new TaskMailboxImpl(), 0, StreamTaskActionExecutor.IMMEDIATE);
SingleCheckpointBarrierHandler barrierHandler = TestBarrierHandlerFactory.forTarget(abstractInvokable).create(singleInputGate, stateWriter);
CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(singleInputGate, barrierHandler, mailboxExecutor, UpstreamRecoveryTracker.forInputGate(singleInputGate));
for (int i = 0; i < numberOfChannels; i++) {
((RemoteInputChannel) checkpointedInputGate.getChannel(i)).requestSubpartition();
}
return checkpointedInputGate;
}
use of org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl in project flink by apache.
the class CheckpointedInputGateTest method setupInputGate.
private CheckpointedInputGate setupInputGate(int numberOfChannels, NetworkBufferPool networkBufferPool, ConnectionManager connectionManager) throws Exception {
SingleInputGate singleInputGate = new SingleInputGateBuilder().setBufferPoolFactory(networkBufferPool.createBufferPool(numberOfChannels, Integer.MAX_VALUE)).setSegmentProvider(networkBufferPool).setChannelFactory((builder, gate) -> builder.setConnectionManager(connectionManager).buildRemoteChannel(gate)).setNumberOfChannels(numberOfChannels).build();
singleInputGate.setup();
MailboxExecutorImpl mailboxExecutor = new MailboxExecutorImpl(new TaskMailboxImpl(), 0, StreamTaskActionExecutor.IMMEDIATE);
CheckpointBarrierTracker barrierHandler = new CheckpointBarrierTracker(numberOfChannels, new AbstractInvokable(new DummyEnvironment()) {
@Override
public void invoke() {
}
}, SystemClock.getInstance(), true);
CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(singleInputGate, barrierHandler, mailboxExecutor, UpstreamRecoveryTracker.forInputGate(singleInputGate));
for (int i = 0; i < numberOfChannels; i++) {
((RemoteInputChannel) checkpointedInputGate.getChannel(i)).requestSubpartition();
}
return checkpointedInputGate;
}
use of org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl in project flink by apache.
the class StreamTaskExecutionDecorationTest method before.
@Before
public void before() throws Exception {
mailbox = new TaskMailboxImpl();
decorator = new CountingStreamTaskActionExecutor();
task = new StreamTask<Object, StreamOperator<Object>>(new DeclineDummyEnvironment(), null, FatalExitExceptionHandler.INSTANCE, decorator, mailbox) {
@Override
protected void init() {
}
@Override
protected void processInput(MailboxDefaultAction.Controller controller) {
}
};
task.operatorChain = new RegularOperatorChain<>(task, new NonRecordWriter<>());
}
Aggregations