Examples with SingleInputGateBuilder org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder used on opensource projects

Example 1 with SingleInputGateBuilder

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder in project flink by apache.

the class SequentialChannelStateReaderImplTest method withInputGates.

private void withInputGates(ThrowingConsumer<InputGate[], Exception> action) throws Exception {
    SingleInputGate[] gates = new SingleInputGate[parLevel];
    final int segmentsToAllocate = parLevel + parLevel * parLevel * buffersPerChannel;
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(segmentsToAllocate, bufferSize);
    try (Closer poolCloser = Closer.create()) {
        poolCloser.register(networkBufferPool::destroy);
        poolCloser.register(networkBufferPool::destroyAllBufferPools);
        try (Closer gateCloser = Closer.create()) {
            for (int i = 0; i < parLevel; i++) {
                gates[i] = new SingleInputGateBuilder().setNumberOfChannels(parLevel).setSingleInputGateIndex(i).setBufferPoolFactory(networkBufferPool.createBufferPool(1, buffersPerChannel)).setSegmentProvider(networkBufferPool).setChannelFactory((builder, gate) -> builder.setNetworkBuffersPerChannel(buffersPerChannel).buildRemoteRecoveredChannel(gate)).build();
                gates[i].setup();
                gateCloser.register(gates[i]::close);
            }
            action.accept(gates);
        }
        assertEquals(segmentsToAllocate, networkBufferPool.getNumberOfAvailableMemorySegments());
    }
}

Example 2 with SingleInputGateBuilder

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testReceiveBacklogAnnouncement.

/**
 * Verifies that {@link NettyMessage.BacklogAnnouncement} can be handled correctly.
 */
@Test
public void testReceiveBacklogAnnouncement() throws Exception {
    int bufferSize = 1024;
    int numBuffers = 10;
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(numBuffers, bufferSize);
    SingleInputGate inputGate = new SingleInputGateBuilder().setSegmentProvider(networkBufferPool).build();
    RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, null);
    inputGate.setInputChannels(inputChannel);
    try {
        BufferPool bufferPool = networkBufferPool.createBufferPool(8, 8);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
        handler.addInputChannel(inputChannel);
        assertEquals(2, inputChannel.getNumberOfAvailableBuffers());
        assertEquals(0, inputChannel.unsynchronizedGetFloatingBuffersAvailable());
        int backlog = 5;
        NettyMessage.BacklogAnnouncement announcement = new NettyMessage.BacklogAnnouncement(backlog, inputChannel.getInputChannelId());
        handler.channelRead(null, announcement);
        assertEquals(7, inputChannel.getNumberOfAvailableBuffers());
        assertEquals(7, inputChannel.getNumberOfRequiredBuffers());
        assertEquals(backlog, inputChannel.getSenderBacklog());
        assertEquals(5, inputChannel.unsynchronizedGetFloatingBuffersAvailable());
        backlog = 12;
        announcement = new NettyMessage.BacklogAnnouncement(backlog, inputChannel.getInputChannelId());
        handler.channelRead(null, announcement);
        assertEquals(10, inputChannel.getNumberOfAvailableBuffers());
        assertEquals(14, inputChannel.getNumberOfRequiredBuffers());
        assertEquals(backlog, inputChannel.getSenderBacklog());
        assertEquals(8, inputChannel.unsynchronizedGetFloatingBuffersAvailable());
    } finally {
        releaseResource(inputGate, networkBufferPool);
    }
}

Example 3 with SingleInputGateBuilder

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder in project flink by apache.

the class CheckpointedInputGateTest method testPriorityBeforeClose.

/**
 * Tests a priority notification happening right before cancellation. The mail would be
 * processed while draining mailbox but can't pull any data anymore.
 */
@Test
public void testPriorityBeforeClose() throws IOException, InterruptedException {
    NetworkBufferPool bufferPool = new NetworkBufferPool(10, 1024);
    try (Closer closer = Closer.create()) {
        closer.register(bufferPool::destroy);
        for (int repeat = 0; repeat < 100; repeat++) {
            setUp();
            SingleInputGate singleInputGate = new SingleInputGateBuilder().setNumberOfChannels(2).setBufferPoolFactory(bufferPool.createBufferPool(2, Integer.MAX_VALUE)).setSegmentProvider(bufferPool).setChannelFactory(InputChannelBuilder::buildRemoteChannel).build();
            singleInputGate.setup();
            ((RemoteInputChannel) singleInputGate.getChannel(0)).requestSubpartition();
            final TaskMailboxImpl mailbox = new TaskMailboxImpl();
            MailboxExecutorImpl mailboxExecutor = new MailboxExecutorImpl(mailbox, 0, StreamTaskActionExecutor.IMMEDIATE);
            ValidatingCheckpointHandler validatingHandler = new ValidatingCheckpointHandler(1);
            SingleCheckpointBarrierHandler barrierHandler = TestBarrierHandlerFactory.forTarget(validatingHandler).create(singleInputGate, new MockChannelStateWriter());
            CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(singleInputGate, barrierHandler, mailboxExecutor, UpstreamRecoveryTracker.forInputGate(singleInputGate));
            final int oldSize = mailbox.size();
            enqueue(checkpointedInputGate, 0, barrier(1));
            // wait for priority mail to be enqueued
            Deadline deadline = Deadline.fromNow(Duration.ofMinutes(1));
            while (deadline.hasTimeLeft() && oldSize >= mailbox.size()) {
                Thread.sleep(1);
            }
            // test the race condition
            // either priority event could be handled, then we expect a checkpoint to be
            // triggered or closing came first in which case we expect a CancelTaskException
            CountDownLatch beforeLatch = new CountDownLatch(2);
            final CheckedThread canceler = new CheckedThread("Canceler") {

                @Override
                public void go() throws IOException {
                    beforeLatch.countDown();
                    singleInputGate.close();
                }
            };
            canceler.start();
            beforeLatch.countDown();
            try {
                while (mailboxExecutor.tryYield()) {
                }
                assertEquals(1L, validatingHandler.triggeredCheckpointCounter);
            } catch (CancelTaskException e) {
            }
            canceler.join();
        }
    }
}

Example 4 with SingleInputGateBuilder

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder in project flink by apache.

the class UnalignedCheckpointsTest method testEndOfStreamWithPendingCheckpoint.

@Test
public void testEndOfStreamWithPendingCheckpoint() throws Exception {
    final int numberOfChannels = 2;
    final ValidatingCheckpointInvokable invokable = new ValidatingCheckpointInvokable();
    final SingleInputGate inputGate = new SingleInputGateBuilder().setChannelFactory(InputChannelBuilder::buildLocalChannel).setNumberOfChannels(numberOfChannels).build();
    final SingleCheckpointBarrierHandler handler = SingleCheckpointBarrierHandler.createUnalignedCheckpointBarrierHandler(TestSubtaskCheckpointCoordinator.INSTANCE, "test", invokable, SystemClock.getInstance(), false, inputGate);
    // should trigger respective checkpoint
    handler.processBarrier(buildCheckpointBarrier(DEFAULT_CHECKPOINT_ID), new InputChannelInfo(0, 0), false);
    assertTrue(handler.isCheckpointPending());
    assertEquals(DEFAULT_CHECKPOINT_ID, handler.getLatestCheckpointId());
    assertEquals(numberOfChannels, handler.getNumOpenChannels());
    // should abort current checkpoint while processing eof
    handler.processEndOfPartition(new InputChannelInfo(0, 0));
    assertFalse(handler.isCheckpointPending());
    assertEquals(DEFAULT_CHECKPOINT_ID, handler.getLatestCheckpointId());
    assertEquals(numberOfChannels - 1, handler.getNumOpenChannels());
    assertEquals(DEFAULT_CHECKPOINT_ID, invokable.getAbortedCheckpointId());
}

Example 5 with SingleInputGateBuilder

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder in project flink by apache.

the class UnalignedCheckpointsTest method createInputGate.

private CheckpointedInputGate createInputGate(int numberOfChannels, AbstractInvokable toNotify, boolean enableCheckpointsAfterTasksFinished) throws IOException {
    final NettyShuffleEnvironment environment = new NettyShuffleEnvironmentBuilder().build();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(numberOfChannels).setupBufferPoolFactory(environment).build();
    gate.setInputChannels(IntStream.range(0, numberOfChannels).mapToObj(channelIndex -> InputChannelBuilder.newBuilder().setChannelIndex(channelIndex).setStateWriter(channelStateWriter).setupFromNettyShuffleEnvironment(environment).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(gate)).toArray(RemoteInputChannel[]::new));
    sequenceNumbers = new int[numberOfChannels];
    gate.setup();
    gate.requestPartitions();
    return createCheckpointedInputGate(gate, toNotify, enableCheckpointsAfterTasksFinished);
}