Examples with ThroughputCalculator org.apache.flink.runtime.throughput.ThroughputCalculator used on opensource projects

Example 1 with ThroughputCalculator

use of org.apache.flink.runtime.throughput.ThroughputCalculator in project flink by apache.

the class SingleInputGateFactory method create.

/**
 * Creates an input gate and all of its input channels.
 */
public SingleInputGate create(@Nonnull ShuffleIOOwnerContext owner, int gateIndex, @Nonnull InputGateDeploymentDescriptor igdd, @Nonnull PartitionProducerStateProvider partitionProducerStateProvider) {
    SupplierWithException<BufferPool, IOException> bufferPoolFactory = createBufferPoolFactory(networkBufferPool, floatingNetworkBuffersPerGate);
    BufferDecompressor bufferDecompressor = null;
    if (igdd.getConsumedPartitionType().isBlocking() && blockingShuffleCompressionEnabled) {
        bufferDecompressor = new BufferDecompressor(networkBufferSize, compressionCodec);
    }
    final String owningTaskName = owner.getOwnerName();
    final MetricGroup networkInputGroup = owner.getInputGroup();
    SubpartitionIndexRange subpartitionIndexRange = igdd.getConsumedSubpartitionIndexRange();
    SingleInputGate inputGate = new SingleInputGate(owningTaskName, gateIndex, igdd.getConsumedResultId(), igdd.getConsumedPartitionType(), subpartitionIndexRange, calculateNumChannels(igdd.getShuffleDescriptors().length, subpartitionIndexRange), partitionProducerStateProvider, bufferPoolFactory, bufferDecompressor, networkBufferPool, networkBufferSize, new ThroughputCalculator(SystemClock.getInstance()), maybeCreateBufferDebloater(gateIndex, networkInputGroup.addGroup(gateIndex)));
    InputChannelMetrics metrics = new InputChannelMetrics(networkInputGroup, owner.getParentGroup());
    createInputChannels(owningTaskName, igdd, inputGate, subpartitionIndexRange, metrics);
    return inputGate;
}

Example 2 with ThroughputCalculator

use of org.apache.flink.runtime.throughput.ThroughputCalculator in project flink by apache.

the class StreamTaskTest method testBufferSizeRecalculationStartSuccessfully.

@Test
public void testBufferSizeRecalculationStartSuccessfully() throws Exception {
    int expectedThroughput = 13333;
    int inputChannels = 3;
    // debloat period doesn't matter, we will schedule debloating manually
    Configuration config = new Configuration().set(BUFFER_DEBLOAT_PERIOD, Duration.ofHours(10)).set(BUFFER_DEBLOAT_TARGET, Duration.ofSeconds(1)).set(BUFFER_DEBLOAT_THRESHOLD_PERCENTAGES, 1).set(BUFFER_DEBLOAT_ENABLED, true);
    try (StreamTaskMailboxTestHarness<String> harness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, STRING_TYPE_INFO).setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(config)).addInput(STRING_TYPE_INFO, inputChannels).addInput(STRING_TYPE_INFO, inputChannels).modifyGateBuilder(gateBuilder -> gateBuilder.setThroughputCalculator(bufferDebloatConfiguration -> new ThroughputCalculator(SystemClock.getInstance()) {

        @Override
        public long calculateThroughput() {
            return expectedThroughput;
        }
    })).setupOutputForSingletonOperatorChain(new TestBoundedOneInputStreamOperator()).build()) {
        harness.processAll();
        harness.streamTask.debloat();
        long lastBufferSize = -1;
        for (InputGate inputGate : harness.streamTask.getEnvironment().getAllInputGates()) {
            for (int i = 0; i < inputGate.getNumberOfInputChannels(); i++) {
                long currentBufferSize = ((TestInputChannel) inputGate.getChannel(i)).getCurrentBufferSize();
                assertThat(currentBufferSize, lessThan(MEMORY_SEGMENT_SIZE.defaultValue().getBytes()));
                assertThat(currentBufferSize, greaterThan(0L));
                if (lastBufferSize > 0) {
                    assertThat(lastBufferSize, is(currentBufferSize));
                }
                lastBufferSize = currentBufferSize;
            }
        }
    }
}

Example 3 with ThroughputCalculator

use of org.apache.flink.runtime.throughput.ThroughputCalculator in project flink by apache.

the class StreamTaskTest method testBufferDebloatingMultiGates.

@Test
public void testBufferDebloatingMultiGates() throws Exception {
    // debloat period doesn't matter, we will schedule debloating manually
    Configuration config = new Configuration().set(BUFFER_DEBLOAT_PERIOD, Duration.ofHours(10)).set(BUFFER_DEBLOAT_TARGET, Duration.ofSeconds(1)).set(BUFFER_DEBLOAT_ENABLED, true).set(BUFFER_DEBLOAT_THRESHOLD_PERCENTAGES, // disable the threshold to achieve exact buffer sizes
    0);
    final long throughputGate1 = 1024L;
    final long throughputGate2 = 60 * 1024L;
    final int inputChannelsGate1 = 1;
    final int inputChannelsGate2 = 4;
    final ThroughputCalculator throughputCalculator = new ThroughputCalculator(SystemClock.getInstance()) {

        /* parameters are ignored */
        private int callCount = 0;

        @Override
        public long calculateThroughput() {
            if (callCount++ % 2 == 0) {
                return throughputGate1;
            } else {
                return throughputGate2;
            }
        }
    };
    try (StreamTaskMailboxTestHarness<String> harness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, STRING_TYPE_INFO).setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(config)).addInput(STRING_TYPE_INFO, inputChannelsGate1).addInput(STRING_TYPE_INFO, inputChannelsGate2).modifyGateBuilder(gateBuilder -> gateBuilder.setThroughputCalculator(bufferDebloatConfiguration -> throughputCalculator)).setupOutputForSingletonOperatorChain(new TestBoundedOneInputStreamOperator()).build()) {
        final IndexedInputGate[] inputGates = harness.streamTask.getEnvironment().getAllInputGates();
        harness.processAll();
        // call debloating until the EMA reaches the target buffer size
        while (getCurrentBufferSize(inputGates[0]) == 0 || getCurrentBufferSize(inputGates[0]) > throughputGate1) {
            harness.streamTask.debloat();
        }
        assertThat(getCurrentBufferSize(inputGates[0]), equalTo((int) throughputGate1));
        assertThat(getCurrentBufferSize(inputGates[1]), equalTo((int) throughputGate2 / inputChannelsGate2));
    }
}