Examples with NetworkBufferPool org.apache.flink.runtime.io.network.buffer.NetworkBufferPool used on opensource projects

Example 26 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class ChannelPersistenceITCase method upstreamBlocksAfterRecoveringState.

private void upstreamBlocksAfterRecoveringState(ResultPartitionType type) throws Exception {
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(4, 1024);
    byte[] dataAfterRecovery = randomBytes(1024);
    try {
        BufferWritingResultPartition resultPartition = buildResultPartition(networkBufferPool, type, 0, 1);
        new SequentialChannelStateReaderImpl(new TaskStateSnapshot()).readOutputData(new BufferWritingResultPartition[] { resultPartition }, true);
        resultPartition.emitRecord(ByteBuffer.wrap(dataAfterRecovery), 0);
        ResultSubpartitionView view = resultPartition.createSubpartitionView(0, new NoOpBufferAvailablityListener());
        if (type != ResultPartitionType.PIPELINED_APPROXIMATE) {
            assertEquals(RECOVERY_COMPLETION, view.getNextBuffer().buffer().getDataType());
            assertNull(view.getNextBuffer());
            view.resumeConsumption();
        }
        assertArrayEquals(dataAfterRecovery, collectBytes(view.getNextBuffer().buffer()));
    } finally {
        networkBufferPool.destroy();
    }
}

Example 27 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class CheckpointedInputGateTest method testPriorityBeforeClose.

/**
 * Tests a priority notification happening right before cancellation. The mail would be
 * processed while draining mailbox but can't pull any data anymore.
 */
@Test
public void testPriorityBeforeClose() throws IOException, InterruptedException {
    NetworkBufferPool bufferPool = new NetworkBufferPool(10, 1024);
    try (Closer closer = Closer.create()) {
        closer.register(bufferPool::destroy);
        for (int repeat = 0; repeat < 100; repeat++) {
            setUp();
            SingleInputGate singleInputGate = new SingleInputGateBuilder().setNumberOfChannels(2).setBufferPoolFactory(bufferPool.createBufferPool(2, Integer.MAX_VALUE)).setSegmentProvider(bufferPool).setChannelFactory(InputChannelBuilder::buildRemoteChannel).build();
            singleInputGate.setup();
            ((RemoteInputChannel) singleInputGate.getChannel(0)).requestSubpartition();
            final TaskMailboxImpl mailbox = new TaskMailboxImpl();
            MailboxExecutorImpl mailboxExecutor = new MailboxExecutorImpl(mailbox, 0, StreamTaskActionExecutor.IMMEDIATE);
            ValidatingCheckpointHandler validatingHandler = new ValidatingCheckpointHandler(1);
            SingleCheckpointBarrierHandler barrierHandler = TestBarrierHandlerFactory.forTarget(validatingHandler).create(singleInputGate, new MockChannelStateWriter());
            CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(singleInputGate, barrierHandler, mailboxExecutor, UpstreamRecoveryTracker.forInputGate(singleInputGate));
            final int oldSize = mailbox.size();
            enqueue(checkpointedInputGate, 0, barrier(1));
            // wait for priority mail to be enqueued
            Deadline deadline = Deadline.fromNow(Duration.ofMinutes(1));
            while (deadline.hasTimeLeft() && oldSize >= mailbox.size()) {
                Thread.sleep(1);
            }
            // test the race condition
            // either priority event could be handled, then we expect a checkpoint to be
            // triggered or closing came first in which case we expect a CancelTaskException
            CountDownLatch beforeLatch = new CountDownLatch(2);
            final CheckedThread canceler = new CheckedThread("Canceler") {

                @Override
                public void go() throws IOException {
                    beforeLatch.countDown();
                    singleInputGate.close();
                }
            };
            canceler.start();
            beforeLatch.countDown();
            try {
                while (mailboxExecutor.tryYield()) {
                }
                assertEquals(1L, validatingHandler.triggeredCheckpointCounter);
            } catch (CancelTaskException e) {
            }
            canceler.join();
        }
    }
}

Example 28 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class CheckpointedInputGateTest method setupInputGateWithAlternatingController.

private CheckpointedInputGate setupInputGateWithAlternatingController(int numberOfChannels, NetworkBufferPool networkBufferPool, AbstractInvokable abstractInvokable, RecordingChannelStateWriter stateWriter) throws Exception {
    ConnectionManager connectionManager = new TestingConnectionManager();
    SingleInputGate singleInputGate = new SingleInputGateBuilder().setBufferPoolFactory(networkBufferPool.createBufferPool(numberOfChannels, Integer.MAX_VALUE)).setSegmentProvider(networkBufferPool).setChannelFactory((builder, gate) -> builder.setConnectionManager(connectionManager).buildRemoteChannel(gate)).setNumberOfChannels(numberOfChannels).setChannelStateWriter(stateWriter).build();
    singleInputGate.setup();
    MailboxExecutorImpl mailboxExecutor = new MailboxExecutorImpl(new TaskMailboxImpl(), 0, StreamTaskActionExecutor.IMMEDIATE);
    SingleCheckpointBarrierHandler barrierHandler = TestBarrierHandlerFactory.forTarget(abstractInvokable).create(singleInputGate, stateWriter);
    CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(singleInputGate, barrierHandler, mailboxExecutor, UpstreamRecoveryTracker.forInputGate(singleInputGate));
    for (int i = 0; i < numberOfChannels; i++) {
        ((RemoteInputChannel) checkpointedInputGate.getChannel(i)).requestSubpartition();
    }
    return checkpointedInputGate;
}

Example 29 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class LocalInputChannelTest method testConcurrentConsumeMultiplePartitions.

/**
 * Tests the consumption of multiple subpartitions via local input channels.
 *
 * <p>Multiple producer tasks produce pipelined partitions, which are consumed by multiple tasks
 * via local input channels.
 */
@Test
public void testConcurrentConsumeMultiplePartitions() throws Exception {
    // Config
    final int parallelism = 32;
    final int producerBufferPoolSize = parallelism + 1;
    final int numberOfBuffersPerChannel = 1024;
    // Setup
    // One thread per produced partition and one per consumer
    final ExecutorService executor = Executors.newFixedThreadPool(2 * parallelism);
    final NetworkBufferPool networkBuffers = new NetworkBufferPool((parallelism * producerBufferPoolSize) + (parallelism * parallelism), TestBufferFactory.BUFFER_SIZE);
    final ResultPartitionManager partitionManager = new ResultPartitionManager();
    final ResultPartitionID[] partitionIds = new ResultPartitionID[parallelism];
    final TestPartitionProducer[] partitionProducers = new TestPartitionProducer[parallelism];
    // Create all partitions
    for (int i = 0; i < parallelism; i++) {
        partitionIds[i] = new ResultPartitionID();
        final ResultPartition partition = new ResultPartitionBuilder().setResultPartitionId(partitionIds[i]).setNumberOfSubpartitions(parallelism).setNumTargetKeyGroups(parallelism).setResultPartitionManager(partitionManager).setBufferPoolFactory(() -> networkBuffers.createBufferPool(producerBufferPoolSize, producerBufferPoolSize, parallelism, Integer.MAX_VALUE)).build();
        // Create a buffer pool for this partition
        partition.setup();
        // Create the producer
        partitionProducers[i] = new TestPartitionProducer((BufferWritingResultPartition) partition, false, new TestPartitionProducerBufferSource(parallelism, TestBufferFactory.BUFFER_SIZE, numberOfBuffersPerChannel));
    }
    // Test
    try {
        // Submit producer tasks
        List<CompletableFuture<?>> results = Lists.newArrayListWithCapacity(parallelism + 1);
        for (int i = 0; i < parallelism; i++) {
            results.add(CompletableFuture.supplyAsync(CheckedSupplier.unchecked(partitionProducers[i]::call), executor));
        }
        // Submit consumer
        for (int i = 0; i < parallelism; i++) {
            final TestLocalInputChannelConsumer consumer = new TestLocalInputChannelConsumer(i, parallelism, numberOfBuffersPerChannel, networkBuffers.createBufferPool(parallelism, parallelism), partitionManager, new TaskEventDispatcher(), partitionIds);
            results.add(CompletableFuture.supplyAsync(CheckedSupplier.unchecked(consumer::call), executor));
        }
        FutureUtils.waitForAll(results).get();
    } finally {
        networkBuffers.destroyAllBufferPools();
        networkBuffers.destroy();
        executor.shutdown();
    }
}

Example 30 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class NettyMessageClientSideSerializationTest method setup.

@Before
public void setup() throws IOException, InterruptedException {
    networkBufferPool = new NetworkBufferPool(8, BUFFER_SIZE);
    inputGate = createSingleInputGate(1, networkBufferPool);
    RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, new TestingPartitionRequestClient());
    inputChannel.requestSubpartition();
    inputGate.setInputChannels(inputChannel);
    inputGate.setup();
    CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    handler.addInputChannel(inputChannel);
    channel = new EmbeddedChannel(// For outbound messages
    new NettyMessageEncoder(), // For inbound messages
    new NettyMessageClientDecoderDelegate(handler));
    inputChannelId = inputChannel.getInputChannelId();
}