Examples with SingleInputGate org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate used on opensource projects

Example 46 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class TestCheckpointedInputGateBuilder method buildTestGate.

private SingleInputGate buildTestGate() {
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(numChannels).build();
    TestInputChannel[] channels = new TestInputChannel[numChannels];
    for (int i = 0; i < numChannels; i++) {
        channels[i] = new TestInputChannel(gate, i, false, true);
    }
    gate.setInputChannels(channels);
    return gate;
}

Example 47 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class TestCheckpointedInputGateBuilder method buildRemoteGate.

private SingleInputGate buildRemoteGate() throws IOException {
    int maxUsedBuffers = 10;
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(numChannels * maxUsedBuffers, 4096);
    SingleInputGate gate = new SingleInputGateBuilder().setChannelFactory(InputChannelBuilder::buildRemoteChannel).setNumberOfChannels(numChannels).setSegmentProvider(networkBufferPool).setBufferPoolFactory(networkBufferPool.createBufferPool(numChannels, maxUsedBuffers)).setChannelStateWriter(channelStateWriter).build();
    gate.setup();
    gate.requestPartitions();
    return gate;
}

Example 48 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class NettyShuffleMetricFactory method registerLegacyNetworkMetrics.

/**
 * Registers legacy network metric groups before shuffle service refactoring.
 *
 * <p>Registers legacy metric groups if shuffle service implementation is original default one.
 *
 * @deprecated should be removed in future
 */
@SuppressWarnings("DeprecatedIsStillUsed")
@Deprecated
public static void registerLegacyNetworkMetrics(boolean isDetailedMetrics, MetricGroup metricGroup, ResultPartitionWriter[] producedPartitions, InputGate[] inputGates) {
    checkNotNull(metricGroup);
    checkNotNull(producedPartitions);
    checkNotNull(inputGates);
    // add metrics for buffers
    final MetricGroup buffersGroup = metricGroup.addGroup(METRIC_GROUP_BUFFERS_DEPRECATED);
    // similar to MetricUtils.instantiateNetworkMetrics() but inside this IOMetricGroup
    // (metricGroup)
    final MetricGroup networkGroup = metricGroup.addGroup(METRIC_GROUP_NETWORK_DEPRECATED);
    final MetricGroup outputGroup = networkGroup.addGroup(METRIC_GROUP_OUTPUT);
    final MetricGroup inputGroup = networkGroup.addGroup(METRIC_GROUP_INPUT);
    ResultPartition[] resultPartitions = Arrays.copyOf(producedPartitions, producedPartitions.length, ResultPartition[].class);
    registerOutputMetrics(isDetailedMetrics, outputGroup, buffersGroup, resultPartitions);
    SingleInputGate[] singleInputGates = Arrays.copyOf(inputGates, inputGates.length, SingleInputGate[].class);
    registerInputMetrics(isDetailedMetrics, inputGroup, buffersGroup, singleInputGates);
}

Example 49 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class AbstractBuffersUsageGauge method getValue.

@Override
public Float getValue() {
    int usedBuffers = 0;
    int totalBuffers = 0;
    for (SingleInputGate inputGate : inputGates) {
        usedBuffers += calculateUsedBuffers(inputGate);
        totalBuffers += calculateTotalBuffers(inputGate);
    }
    if (totalBuffers != 0) {
        return ((float) usedBuffers) / totalBuffers;
    } else {
        return 0.0f;
    }
}

Example 50 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testReceiveCompressedBuffer.

/**
 * Verifies that {@link BufferResponse} of compressed {@link Buffer} can be handled correctly.
 */
@Test
public void testReceiveCompressedBuffer() throws Exception {
    int bufferSize = 1024;
    String compressionCodec = "LZ4";
    BufferCompressor compressor = new BufferCompressor(bufferSize, compressionCodec);
    BufferDecompressor decompressor = new BufferDecompressor(bufferSize, compressionCodec);
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, bufferSize);
    SingleInputGate inputGate = new SingleInputGateBuilder().setBufferDecompressor(decompressor).setSegmentProvider(networkBufferPool).build();
    RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, null);
    inputGate.setInputChannels(inputChannel);
    try {
        BufferPool bufferPool = networkBufferPool.createBufferPool(8, 8);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
        handler.addInputChannel(inputChannel);
        Buffer buffer = compressor.compressToOriginalBuffer(TestBufferFactory.createBuffer(bufferSize));
        BufferResponse bufferResponse = createBufferResponse(buffer, 0, inputChannel.getInputChannelId(), 2, new NetworkBufferAllocator(handler));
        assertTrue(bufferResponse.isCompressed);
        handler.channelRead(null, bufferResponse);
        Buffer receivedBuffer = inputChannel.getNextReceivedBuffer();
        assertNotNull(receivedBuffer);
        assertTrue(receivedBuffer.isCompressed());
        receivedBuffer.recycleBuffer();
    } finally {
        releaseResource(inputGate, networkBufferPool);
    }
}