Examples with NetworkBufferPool org.apache.flink.runtime.io.network.buffer.NetworkBufferPool used on opensource projects

Example 46 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class ResultPartitionTest method testPartitionBufferPool.

private void testPartitionBufferPool(ResultPartitionType type) throws Exception {
    // setup
    final int networkBuffersPerChannel = 2;
    final int floatingNetworkBuffersPerGate = 8;
    final NetworkBufferPool globalPool = new NetworkBufferPool(20, 1);
    final ResultPartition partition = new ResultPartitionBuilder().setResultPartitionType(type).setFileChannelManager(fileChannelManager).setNetworkBuffersPerChannel(networkBuffersPerChannel).setFloatingNetworkBuffersPerGate(floatingNetworkBuffersPerGate).setNetworkBufferPool(globalPool).build();
    try {
        partition.setup();
        BufferPool bufferPool = partition.getBufferPool();
        // verify the amount of buffers in created local pool
        assertEquals(partition.getNumberOfSubpartitions() + 1, bufferPool.getNumberOfRequiredMemorySegments());
        if (type.isBounded()) {
            final int maxNumBuffers = networkBuffersPerChannel * partition.getNumberOfSubpartitions() + floatingNetworkBuffersPerGate;
            assertEquals(maxNumBuffers, bufferPool.getMaxNumberOfMemorySegments());
        } else {
            assertEquals(Integer.MAX_VALUE, bufferPool.getMaxNumberOfMemorySegments());
        }
    } finally {
        // cleanup
        globalPool.destroyAllBufferPools();
        globalPool.destroy();
    }
}

Example 47 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class ResultPartitionTest method testIdleAndBackPressuredTime.

@Test
public void testIdleAndBackPressuredTime() throws IOException, InterruptedException {
    // setup
    int bufferSize = 1024;
    NetworkBufferPool globalPool = new NetworkBufferPool(10, bufferSize);
    BufferPool localPool = globalPool.createBufferPool(1, 1, 1, Integer.MAX_VALUE);
    BufferWritingResultPartition resultPartition = (BufferWritingResultPartition) new ResultPartitionBuilder().setBufferPoolFactory(() -> localPool).build();
    resultPartition.setup();
    resultPartition.emitRecord(ByteBuffer.allocate(bufferSize), 0);
    ResultSubpartitionView readView = resultPartition.createSubpartitionView(0, new NoOpBufferAvailablityListener());
    Buffer buffer = readView.getNextBuffer().buffer();
    assertNotNull(buffer);
    // back-pressured time is zero when there is buffer available.
    assertThat(resultPartition.getHardBackPressuredTimeMsPerSecond().getValue(), equalTo(0L));
    CountDownLatch syncLock = new CountDownLatch(1);
    final Thread requestThread = new Thread(() -> {
        try {
            // notify that the request thread start to run.
            syncLock.countDown();
            // wait for buffer.
            resultPartition.emitRecord(ByteBuffer.allocate(bufferSize), 0);
        } catch (Exception e) {
        }
    });
    requestThread.start();
    // wait until request thread start to run.
    syncLock.await();
    Thread.sleep(100);
    // recycle the buffer
    buffer.recycleBuffer();
    requestThread.join();
    Assert.assertThat(resultPartition.getHardBackPressuredTimeMsPerSecond().getCount(), Matchers.greaterThan(0L));
    assertNotNull(readView.getNextBuffer().buffer());
}

Example 48 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class RemoteInputChannelTest method testConcurrentRecycleAndRelease.

/**
 * Tests to verify that there is no race condition with two things running in parallel:
 * recycling the exclusive or floating buffers and some other thread releasing the input
 * channel.
 */
@Test
public void testConcurrentRecycleAndRelease() throws Exception {
    // Setup
    final int numExclusiveSegments = 120;
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(248, 32);
    final int numFloatingBuffers = 128;
    final ExecutorService executor = Executors.newFixedThreadPool(3);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = InputChannelTestUtils.createRemoteInputChannel(inputGate, numExclusiveSegments);
    inputGate.setInputChannels(inputChannel);
    Throwable thrown = null;
    try {
        final BufferPool bufferPool = networkBufferPool.createBufferPool(numFloatingBuffers, numFloatingBuffers);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        final Callable<Void> releaseTask = new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                inputChannel.releaseAllResources();
                return null;
            }
        };
        // Submit tasks and wait to finish
        submitTasksAndWaitForResults(executor, new Callable[] { recycleBufferTask(inputChannel, bufferPool, numExclusiveSegments, numFloatingBuffers), releaseTask });
        assertEquals("There should be no buffers available in the channel.", 0, inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be " + numFloatingBuffers + " buffers available in local pool.", numFloatingBuffers, bufferPool.getNumberOfAvailableMemorySegments());
        assertEquals("There should be " + numExclusiveSegments + " buffers available in global pool.", numExclusiveSegments, networkBufferPool.getNumberOfAvailableMemorySegments());
    } catch (Throwable t) {
        thrown = t;
    } finally {
        cleanup(networkBufferPool, executor, null, thrown, inputChannel);
    }
}

Example 49 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class ChannelPersistenceITCase method testReadWritten.

@Test
public void testReadWritten() throws Exception {
    byte[] inputChannelInfoData = randomBytes(1024);
    byte[] resultSubpartitionInfoData = randomBytes(1024);
    int partitionIndex = 0;
    SequentialChannelStateReader reader = new SequentialChannelStateReaderImpl(toTaskStateSnapshot(write(1L, singletonMap(new InputChannelInfo(0, 0), inputChannelInfoData), singletonMap(new ResultSubpartitionInfo(partitionIndex, 0), resultSubpartitionInfoData))));
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(4, 1024);
    try {
        int numChannels = 1;
        InputGate gate = buildGate(networkBufferPool, numChannels);
        reader.readInputData(new InputGate[] { gate });
        assertArrayEquals(inputChannelInfoData, collectBytes(gate::pollNext, BufferOrEvent::getBuffer));
        BufferWritingResultPartition resultPartition = buildResultPartition(networkBufferPool, ResultPartitionType.PIPELINED, partitionIndex, numChannels);
        reader.readOutputData(new BufferWritingResultPartition[] { resultPartition }, false);
        ResultSubpartitionView view = resultPartition.createSubpartitionView(0, new NoOpBufferAvailablityListener());
        assertArrayEquals(resultSubpartitionInfoData, collectBytes(() -> Optional.ofNullable(view.getNextBuffer()), BufferAndBacklog::buffer));
    } finally {
        networkBufferPool.destroy();
    }
}

Example 50 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class AlignedCheckpointsMassiveRandomTest method testWithTwoChannelsAndRandomBarriers.

@Test
public void testWithTwoChannelsAndRandomBarriers() throws Exception {
    NetworkBufferPool networkBufferPool1 = null;
    NetworkBufferPool networkBufferPool2 = null;
    try {
        networkBufferPool1 = new NetworkBufferPool(100, PAGE_SIZE);
        networkBufferPool2 = new NetworkBufferPool(100, PAGE_SIZE);
        BufferPool pool1 = networkBufferPool1.createBufferPool(100, 100);
        BufferPool pool2 = networkBufferPool2.createBufferPool(100, 100);
        RandomGeneratingInputGate myIG = new RandomGeneratingInputGate(new BufferPool[] { pool1, pool2 }, new BarrierGenerator[] { new CountBarrier(100000), new RandomBarrier(100000) });
        CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(myIG, SingleCheckpointBarrierHandler.aligned("Testing: No task associated", new DummyCheckpointInvokable(), SystemClock.getInstance(), myIG.getNumberOfInputChannels(), (callable, duration) -> () -> {
        }, true, myIG), new SyncMailboxExecutor());
        for (int i = 0; i < 2000000; i++) {
            BufferOrEvent boe = checkpointedInputGate.pollNext().get();
            if (boe.isBuffer()) {
                boe.getBuffer().recycleBuffer();
            }
        }
    } finally {
        if (networkBufferPool1 != null) {
            networkBufferPool1.destroyAllBufferPools();
            networkBufferPool1.destroy();
        }
        if (networkBufferPool2 != null) {
            networkBufferPool2.destroyAllBufferPools();
            networkBufferPool2.destroy();
        }
    }
}