Examples with NetworkBufferPool org.apache.flink.runtime.io.network.buffer.NetworkBufferPool used on opensource projects

Example 6 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class ResultSubpartitionRecoveredStateHandlerTest method setUp.

@Before
public void setUp() throws IOException {
    // given: Segment provider with defined number of allocated segments.
    channelInfo = new ResultSubpartitionInfo(0, 0);
    networkBufferPool = new NetworkBufferPool(preAllocatedSegments, 1024);
    partition = new ResultPartitionBuilder().setNetworkBufferPool(networkBufferPool).build();
    partition.setup();
    rstHandler = buildResultStateHandler(partition);
}

Example 7 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class SequentialChannelStateReaderImplTest method withInputGates.

private void withInputGates(ThrowingConsumer<InputGate[], Exception> action) throws Exception {
    SingleInputGate[] gates = new SingleInputGate[parLevel];
    final int segmentsToAllocate = parLevel + parLevel * parLevel * buffersPerChannel;
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(segmentsToAllocate, bufferSize);
    try (Closer poolCloser = Closer.create()) {
        poolCloser.register(networkBufferPool::destroy);
        poolCloser.register(networkBufferPool::destroyAllBufferPools);
        try (Closer gateCloser = Closer.create()) {
            for (int i = 0; i < parLevel; i++) {
                gates[i] = new SingleInputGateBuilder().setNumberOfChannels(parLevel).setSingleInputGateIndex(i).setBufferPoolFactory(networkBufferPool.createBufferPool(1, buffersPerChannel)).setSegmentProvider(networkBufferPool).setChannelFactory((builder, gate) -> builder.setNetworkBuffersPerChannel(buffersPerChannel).buildRemoteRecoveredChannel(gate)).build();
                gates[i].setup();
                gateCloser.register(gates[i]::close);
            }
            action.accept(gates);
        }
        assertEquals(segmentsToAllocate, networkBufferPool.getNumberOfAvailableMemorySegments());
    }
}

Example 8 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class SequentialChannelStateReaderImplTest method withResultPartitions.

private void withResultPartitions(ThrowingConsumer<BufferWritingResultPartition[], Exception> action) throws Exception {
    int segmentsToAllocate = parLevel * parLevel * buffersPerChannel;
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(segmentsToAllocate, bufferSize);
    BufferWritingResultPartition[] resultPartitions = range(0, parLevel).mapToObj(i -> new ResultPartitionBuilder().setResultPartitionIndex(i).setNumberOfSubpartitions(parLevel).setNetworkBufferPool(networkBufferPool).build()).toArray(BufferWritingResultPartition[]::new);
    try {
        for (ResultPartition resultPartition : resultPartitions) {
            resultPartition.setup();
        }
        action.accept(resultPartitions);
    } finally {
        for (ResultPartition resultPartition : resultPartitions) {
            resultPartition.close();
        }
        try {
            assertEquals(segmentsToAllocate, networkBufferPool.getNumberOfAvailableMemorySegments());
        } finally {
            networkBufferPool.destroyAllBufferPools();
            networkBufferPool.destroy();
        }
    }
}

Example 9 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class SingleInputGateFactory method create.

/**
 * Creates an input gate and all of its input channels.
 */
public SingleInputGate create(@Nonnull ShuffleIOOwnerContext owner, int gateIndex, @Nonnull InputGateDeploymentDescriptor igdd, @Nonnull PartitionProducerStateProvider partitionProducerStateProvider) {
    SupplierWithException<BufferPool, IOException> bufferPoolFactory = createBufferPoolFactory(networkBufferPool, floatingNetworkBuffersPerGate);
    BufferDecompressor bufferDecompressor = null;
    if (igdd.getConsumedPartitionType().isBlocking() && blockingShuffleCompressionEnabled) {
        bufferDecompressor = new BufferDecompressor(networkBufferSize, compressionCodec);
    }
    final String owningTaskName = owner.getOwnerName();
    final MetricGroup networkInputGroup = owner.getInputGroup();
    SubpartitionIndexRange subpartitionIndexRange = igdd.getConsumedSubpartitionIndexRange();
    SingleInputGate inputGate = new SingleInputGate(owningTaskName, gateIndex, igdd.getConsumedResultId(), igdd.getConsumedPartitionType(), subpartitionIndexRange, calculateNumChannels(igdd.getShuffleDescriptors().length, subpartitionIndexRange), partitionProducerStateProvider, bufferPoolFactory, bufferDecompressor, networkBufferPool, networkBufferSize, new ThroughputCalculator(SystemClock.getInstance()), maybeCreateBufferDebloater(gateIndex, networkInputGroup.addGroup(gateIndex)));
    InputChannelMetrics metrics = new InputChannelMetrics(networkInputGroup, owner.getParentGroup());
    createInputChannels(owningTaskName, igdd, inputGate, subpartitionIndexRange, metrics);
    return inputGate;
}

Example 10 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class NettyShuffleServiceFactory method createNettyShuffleEnvironment.

@VisibleForTesting
static NettyShuffleEnvironment createNettyShuffleEnvironment(NettyShuffleEnvironmentConfiguration config, ResourceID taskExecutorResourceId, TaskEventPublisher taskEventPublisher, ResultPartitionManager resultPartitionManager, MetricGroup metricGroup, Executor ioExecutor) {
    checkNotNull(config);
    checkNotNull(taskExecutorResourceId);
    checkNotNull(taskEventPublisher);
    checkNotNull(resultPartitionManager);
    checkNotNull(metricGroup);
    NettyConfig nettyConfig = config.nettyConfig();
    FileChannelManager fileChannelManager = new FileChannelManagerImpl(config.getTempDirs(), DIR_NAME_PREFIX);
    if (LOG.isInfoEnabled()) {
        LOG.info("Created a new {} for storing result partitions of BLOCKING shuffles. Used directories:\n\t{}", FileChannelManager.class.getSimpleName(), Arrays.stream(fileChannelManager.getPaths()).map(File::getAbsolutePath).collect(Collectors.joining("\n\t")));
    }
    ConnectionManager connectionManager = nettyConfig != null ? new NettyConnectionManager(resultPartitionManager, taskEventPublisher, nettyConfig, config.getMaxNumberOfConnections(), config.isConnectionReuseEnabled()) : new LocalConnectionManager();
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(config.numNetworkBuffers(), config.networkBufferSize(), config.getRequestSegmentsTimeout());
    // we create a separated buffer pool here for batch shuffle instead of reusing the network
    // buffer pool directly to avoid potential side effects of memory contention, for example,
    // dead lock or "insufficient network buffer" error
    BatchShuffleReadBufferPool batchShuffleReadBufferPool = new BatchShuffleReadBufferPool(config.batchShuffleReadMemoryBytes(), config.networkBufferSize());
    // we create a separated IO executor pool here for batch shuffle instead of reusing the
    // TaskManager IO executor pool directly to avoid the potential side effects of execution
    // contention, for example, too long IO or waiting time leading to starvation or timeout
    ExecutorService batchShuffleReadIOExecutor = Executors.newFixedThreadPool(Math.max(1, Math.min(batchShuffleReadBufferPool.getMaxConcurrentRequests(), 4 * Hardware.getNumberCPUCores())), new ExecutorThreadFactory("blocking-shuffle-io"));
    registerShuffleMetrics(metricGroup, networkBufferPool);
    ResultPartitionFactory resultPartitionFactory = new ResultPartitionFactory(resultPartitionManager, fileChannelManager, networkBufferPool, batchShuffleReadBufferPool, batchShuffleReadIOExecutor, config.getBlockingSubpartitionType(), config.networkBuffersPerChannel(), config.floatingNetworkBuffersPerGate(), config.networkBufferSize(), config.isBlockingShuffleCompressionEnabled(), config.getCompressionCodec(), config.getMaxBuffersPerChannel(), config.sortShuffleMinBuffers(), config.sortShuffleMinParallelism(), config.isSSLEnabled());
    SingleInputGateFactory singleInputGateFactory = new SingleInputGateFactory(taskExecutorResourceId, config, connectionManager, resultPartitionManager, taskEventPublisher, networkBufferPool);
    return new NettyShuffleEnvironment(taskExecutorResourceId, config, networkBufferPool, connectionManager, resultPartitionManager, fileChannelManager, resultPartitionFactory, singleInputGateFactory, ioExecutor, batchShuffleReadBufferPool, batchShuffleReadIOExecutor);
}