Search in sources :

Example 6 with IndexedInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.IndexedInputGate in project flink by apache.

the class StreamTaskTest method testBufferDebloatingMultiGates.

@Test
public void testBufferDebloatingMultiGates() throws Exception {
    // debloat period doesn't matter, we will schedule debloating manually
    Configuration config = new Configuration().set(BUFFER_DEBLOAT_PERIOD, Duration.ofHours(10)).set(BUFFER_DEBLOAT_TARGET, Duration.ofSeconds(1)).set(BUFFER_DEBLOAT_ENABLED, true).set(BUFFER_DEBLOAT_THRESHOLD_PERCENTAGES, // disable the threshold to achieve exact buffer sizes
    0);
    final long throughputGate1 = 1024L;
    final long throughputGate2 = 60 * 1024L;
    final int inputChannelsGate1 = 1;
    final int inputChannelsGate2 = 4;
    final ThroughputCalculator throughputCalculator = new ThroughputCalculator(SystemClock.getInstance()) {

        /* parameters are ignored */
        private int callCount = 0;

        @Override
        public long calculateThroughput() {
            if (callCount++ % 2 == 0) {
                return throughputGate1;
            } else {
                return throughputGate2;
            }
        }
    };
    try (StreamTaskMailboxTestHarness<String> harness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, STRING_TYPE_INFO).setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(config)).addInput(STRING_TYPE_INFO, inputChannelsGate1).addInput(STRING_TYPE_INFO, inputChannelsGate2).modifyGateBuilder(gateBuilder -> gateBuilder.setThroughputCalculator(bufferDebloatConfiguration -> throughputCalculator)).setupOutputForSingletonOperatorChain(new TestBoundedOneInputStreamOperator()).build()) {
        final IndexedInputGate[] inputGates = harness.streamTask.getEnvironment().getAllInputGates();
        harness.processAll();
        // call debloating until the EMA reaches the target buffer size
        while (getCurrentBufferSize(inputGates[0]) == 0 || getCurrentBufferSize(inputGates[0]) > throughputGate1) {
            harness.streamTask.debloat();
        }
        assertThat(getCurrentBufferSize(inputGates[0]), equalTo((int) throughputGate1));
        assertThat(getCurrentBufferSize(inputGates[1]), equalTo((int) throughputGate2 / inputChannelsGate2));
    }
}
Also used : InternalTimeServiceManager(org.apache.flink.streaming.api.operators.InternalTimeServiceManager) NettyShuffleDescriptorBuilder(org.apache.flink.runtime.util.NettyShuffleDescriptorBuilder) ArgumentMatchers.nullable(org.mockito.ArgumentMatchers.nullable) TaskLocalStateStoreImpl(org.apache.flink.runtime.state.TaskLocalStateStoreImpl) Collections.singletonList(java.util.Collections.singletonList) CheckpointException(org.apache.flink.runtime.checkpoint.CheckpointException) Arrays.asList(java.util.Arrays.asList) Duration(java.time.Duration) Map(java.util.Map) Mockito.doAnswer(org.mockito.Mockito.doAnswer) AvailabilityTestResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.AvailabilityTestResultPartitionWriter) FunctionWithException(org.apache.flink.util.function.FunctionWithException) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) FatalExitExceptionHandler(org.apache.flink.util.FatalExitExceptionHandler) Matchers.any(org.mockito.Matchers.any) CountDownLatch(java.util.concurrent.CountDownLatch) AsynchronousException(org.apache.flink.runtime.taskmanager.AsynchronousException) Assert.assertFalse(org.junit.Assert.assertFalse) DummyEnvironment(org.apache.flink.runtime.operators.testutils.DummyEnvironment) Mockito.mock(org.mockito.Mockito.mock) CoreMatchers.equalTo(org.hamcrest.CoreMatchers.equalTo) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) Mockito.spy(org.mockito.Mockito.spy) Answer(org.mockito.stubbing.Answer) OptionalLong(java.util.OptionalLong) MAX_PRIORITY(org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailbox.MAX_PRIORITY) Output(org.apache.flink.streaming.api.operators.Output) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) CheckpointableKeyedStateBackend(org.apache.flink.runtime.state.CheckpointableKeyedStateBackend) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) FlinkRuntimeException(org.apache.flink.util.FlinkRuntimeException) MailboxExecutor(org.apache.flink.api.common.operators.MailboxExecutor) ExecutionState(org.apache.flink.runtime.execution.ExecutionState) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) MockStreamConfig(org.apache.flink.streaming.util.MockStreamConfig) StreamOperator(org.apache.flink.streaming.api.operators.StreamOperator) ExecutionException(java.util.concurrent.ExecutionException) Mockito.never(org.mockito.Mockito.never) JobID(org.apache.flink.api.common.JobID) Matchers.sameInstance(org.hamcrest.Matchers.sameInstance) CheckpointStreamFactory(org.apache.flink.runtime.state.CheckpointStreamFactory) Assert.assertEquals(org.junit.Assert.assertEquals) TaskStateManagerImpl(org.apache.flink.runtime.state.TaskStateManagerImpl) TestTaskBuilder(org.apache.flink.runtime.taskmanager.TestTaskBuilder) SystemClock(org.apache.flink.util.clock.SystemClock) ThroughputCalculator(org.apache.flink.runtime.throughput.ThroughputCalculator) ShuffleEnvironment(org.apache.flink.runtime.shuffle.ShuffleEnvironment) ObjectInputStream(java.io.ObjectInputStream) ExceptionUtils(org.apache.flink.util.ExceptionUtils) FunctionSnapshotContext(org.apache.flink.runtime.state.FunctionSnapshotContext) TimerGauge(org.apache.flink.runtime.metrics.TimerGauge) BasicTypeInfo(org.apache.flink.api.common.typeinfo.BasicTypeInfo) ChainingStrategy(org.apache.flink.streaming.api.operators.ChainingStrategy) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) CheckpointStorageLocationReference.getDefault(org.apache.flink.runtime.state.CheckpointStorageLocationReference.getDefault) Matchers.eq(org.mockito.Matchers.eq) BUFFER_DEBLOAT_ENABLED(org.apache.flink.configuration.TaskManagerOptions.BUFFER_DEBLOAT_ENABLED) TaskStateSnapshot(org.apache.flink.runtime.checkpoint.TaskStateSnapshot) CheckpointType(org.apache.flink.runtime.checkpoint.CheckpointType) BUFFER_DEBLOAT_THRESHOLD_PERCENTAGES(org.apache.flink.configuration.TaskManagerOptions.BUFFER_DEBLOAT_THRESHOLD_PERCENTAGES) MockStreamTaskBuilder(org.apache.flink.streaming.util.MockStreamTaskBuilder) CheckpointOptions(org.apache.flink.runtime.checkpoint.CheckpointOptions) Preconditions(org.apache.flink.util.Preconditions) AbstractKeyedStateBackend(org.apache.flink.runtime.state.AbstractKeyedStateBackend) CloseableIterable(org.apache.flink.util.CloseableIterable) DEFAULT_OUTPUT_FLUSH_THREAD_NAME(org.apache.flink.runtime.io.network.api.writer.RecordWriter.DEFAULT_OUTPUT_FLUSH_THREAD_NAME) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) CheckpointResponder(org.apache.flink.runtime.taskmanager.CheckpointResponder) OneShotLatch(org.apache.flink.core.testutils.OneShotLatch) CheckpointMetaData(org.apache.flink.runtime.checkpoint.CheckpointMetaData) Assert.assertThrows(org.junit.Assert.assertThrows) CoreMatchers.not(org.hamcrest.CoreMatchers.not) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) OperatorStreamStateHandle(org.apache.flink.runtime.state.OperatorStreamStateHandle) ArgumentCaptor(org.mockito.ArgumentCaptor) StreamRecord(org.apache.flink.streaming.runtime.streamrecord.StreamRecord) StreamTaskStateInitializer(org.apache.flink.streaming.api.operators.StreamTaskStateInitializer) RichParallelSourceFunction(org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction) StreamMap(org.apache.flink.streaming.api.operators.StreamMap) MockEnvironment(org.apache.flink.runtime.operators.testutils.MockEnvironment) StreamInputProcessor(org.apache.flink.streaming.runtime.io.StreamInputProcessor) ExecutorService(java.util.concurrent.ExecutorService) RunnableFuture(java.util.concurrent.RunnableFuture) NettyShuffleEnvironmentBuilder(org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder) Configuration(org.apache.flink.configuration.Configuration) StreamOperatorParameters(org.apache.flink.streaming.api.operators.StreamOperatorParameters) MockEnvironmentBuilder(org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder) Mockito.when(org.mockito.Mockito.when) Consumer(java.util.function.Consumer) STATE_BACKEND(org.apache.flink.configuration.StateBackendOptions.STATE_BACKEND) ExecutionAttemptID(org.apache.flink.runtime.executiongraph.ExecutionAttemptID) IndexedInputGate(org.apache.flink.runtime.io.network.partition.consumer.IndexedInputGate) TaskExecutionState(org.apache.flink.runtime.taskmanager.TaskExecutionState) ProcessingTimeCallback(org.apache.flink.api.common.operators.ProcessingTimeService.ProcessingTimeCallback) StreamSource(org.apache.flink.streaming.api.operators.StreamSource) TaskIOMetricGroup(org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup) Arrays(java.util.Arrays) AbstractStreamOperatorFactory(org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory) CheckpointListener(org.apache.flink.api.common.state.CheckpointListener) OperatorStateBackend(org.apache.flink.runtime.state.OperatorStateBackend) TypeSerializer(org.apache.flink.api.common.typeutils.TypeSerializer) CancelTaskException(org.apache.flink.runtime.execution.CancelTaskException) Executors(java.util.concurrent.Executors) StopMode(org.apache.flink.runtime.io.network.api.StopMode) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) SnapshotType(org.apache.flink.runtime.checkpoint.SnapshotType) TaskStateManager(org.apache.flink.runtime.state.TaskStateManager) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.is(org.hamcrest.Matchers.is) OneInputStreamOperator(org.apache.flink.streaming.api.operators.OneInputStreamOperator) StateHandleID(org.apache.flink.runtime.state.StateHandleID) RunnableWithException(org.apache.flink.util.function.RunnableWithException) STRING_TYPE_INFO(org.apache.flink.api.common.typeinfo.BasicTypeInfo.STRING_TYPE_INFO) ArrayList(java.util.ArrayList) MEMORY_SEGMENT_SIZE(org.apache.flink.configuration.TaskManagerOptions.MEMORY_SEGMENT_SIZE) Mockito.timeout(org.mockito.Mockito.timeout) OperatorSnapshotFutures(org.apache.flink.streaming.api.operators.OperatorSnapshotFutures) InvocationOnMock(org.mockito.invocation.InvocationOnMock) SourceFunction(org.apache.flink.streaming.api.functions.source.SourceFunction) FutureUtils(org.apache.flink.util.concurrent.FutureUtils) ReadableConfig(org.apache.flink.configuration.ReadableConfig) Matchers.lessThan(org.hamcrest.Matchers.lessThan) ChannelStateWriter(org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter) Nullable(javax.annotation.Nullable) SnapshotResult(org.apache.flink.runtime.state.SnapshotResult) TimeCharacteristic(org.apache.flink.streaming.api.TimeCharacteristic) MailboxDefaultAction(org.apache.flink.streaming.runtime.tasks.mailbox.MailboxDefaultAction) FSDataInputStream(org.apache.flink.core.fs.FSDataInputStream) StreamStateHandle(org.apache.flink.runtime.state.StreamStateHandle) InMemoryStateChangelogStorage(org.apache.flink.runtime.state.changelog.inmemory.InMemoryStateChangelogStorage) AbstractStreamOperator(org.apache.flink.streaming.api.operators.AbstractStreamOperator) Task(org.apache.flink.runtime.taskmanager.Task) SubtaskState(org.apache.flink.runtime.checkpoint.SubtaskState) TestingUncaughtExceptionHandler(org.apache.flink.util.concurrent.TestingUncaughtExceptionHandler) Assert(org.junit.Assert) CoreMatchers(org.hamcrest.CoreMatchers) CheckpointMetricsBuilder(org.apache.flink.runtime.checkpoint.CheckpointMetricsBuilder) SavepointType(org.apache.flink.runtime.checkpoint.SavepointType) SharedStateRegistry(org.apache.flink.runtime.state.SharedStateRegistry) TimeoutException(java.util.concurrent.TimeoutException) StreamTaskUtil.waitTaskIsRunning(org.apache.flink.streaming.util.StreamTaskUtil.waitTaskIsRunning) PartitionDescriptorBuilder(org.apache.flink.runtime.shuffle.PartitionDescriptorBuilder) StringSerializer(org.apache.flink.api.common.typeutils.base.StringSerializer) MapFunction(org.apache.flink.api.common.functions.MapFunction) BUFFER_DEBLOAT_PERIOD(org.apache.flink.configuration.TaskManagerOptions.BUFFER_DEBLOAT_PERIOD) DataInputStatus(org.apache.flink.streaming.runtime.io.DataInputStatus) TaskManagerActions(org.apache.flink.runtime.taskmanager.TaskManagerActions) TestLogger(org.apache.flink.util.TestLogger) Assert.fail(org.junit.Assert.fail) MockStateBackend(org.apache.flink.runtime.state.ttl.mock.MockStateBackend) KeyGroupRange(org.apache.flink.runtime.state.KeyGroupRange) TestInputChannel(org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel) CheckpointedFunction(org.apache.flink.streaming.api.checkpoint.CheckpointedFunction) BUFFER_DEBLOAT_TARGET(org.apache.flink.configuration.TaskManagerOptions.BUFFER_DEBLOAT_TARGET) ExpectedTestException(org.apache.flink.runtime.operators.testutils.ExpectedTestException) FunctionInitializationContext(org.apache.flink.runtime.state.FunctionInitializationContext) KeyedStateHandle(org.apache.flink.runtime.state.KeyedStateHandle) OperatorSubtaskState(org.apache.flink.runtime.checkpoint.OperatorSubtaskState) NettyShuffleEnvironment(org.apache.flink.runtime.io.network.NettyShuffleEnvironment) MemoryStateBackend(org.apache.flink.runtime.state.memory.MemoryStateBackend) AbstractStateBackend(org.apache.flink.runtime.state.AbstractStateBackend) StateObjectCollection.singleton(org.apache.flink.runtime.checkpoint.StateObjectCollection.singleton) List(java.util.List) ResultPartitionDeploymentDescriptor(org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor) KeyGroupStatePartitionStreamProvider(org.apache.flink.runtime.state.KeyGroupStatePartitionStreamProvider) Optional(java.util.Optional) StateBackendFactory(org.apache.flink.runtime.state.StateBackendFactory) CheckpointMetrics(org.apache.flink.runtime.checkpoint.CheckpointMetrics) Environment(org.apache.flink.runtime.execution.Environment) SupplierWithException(org.apache.flink.util.function.SupplierWithException) BiConsumerWithException(org.apache.flink.util.function.BiConsumerWithException) StatePartitionStreamProvider(org.apache.flink.runtime.state.StatePartitionStreamProvider) NoOpTaskManagerActions(org.apache.flink.runtime.taskmanager.NoOpTaskManagerActions) StreamConfig(org.apache.flink.streaming.api.graph.StreamConfig) SavepointFormatType(org.apache.flink.core.execution.SavepointFormatType) TaskInvokable(org.apache.flink.runtime.jobgraph.tasks.TaskInvokable) UNKNOWN_TASK_CHECKPOINT_NOTIFICATION_FAILURE(org.apache.flink.runtime.checkpoint.CheckpointFailureReason.UNKNOWN_TASK_CHECKPOINT_NOTIFICATION_FAILURE) TestingTaskManagerRuntimeInfo(org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo) Matchers.anyLong(org.mockito.Matchers.anyLong) OperatorStateHandle(org.apache.flink.runtime.state.OperatorStateHandle) ExpectedException(org.junit.rules.ExpectedException) InputGate(org.apache.flink.runtime.io.network.partition.consumer.InputGate) Preconditions.checkState(org.apache.flink.util.Preconditions.checkState) StreamOperatorStateContext(org.apache.flink.streaming.api.operators.StreamOperatorStateContext) Matchers(org.hamcrest.Matchers) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) DoneFuture(org.apache.flink.runtime.state.DoneFuture) Rule(org.junit.Rule) Closeable(java.io.Closeable) Collections(java.util.Collections) StateInitializationContext(org.apache.flink.runtime.state.StateInitializationContext) Configuration(org.apache.flink.configuration.Configuration) IndexedInputGate(org.apache.flink.runtime.io.network.partition.consumer.IndexedInputGate) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) TestingTaskManagerRuntimeInfo(org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo) ThroughputCalculator(org.apache.flink.runtime.throughput.ThroughputCalculator) Test(org.junit.Test)

Example 7 with IndexedInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.IndexedInputGate in project flink by apache.

the class StreamNetworkBenchmarkEnvironment method createInputGate.

private InputGate createInputGate(TaskManagerLocation senderLocation) throws Exception {
    IndexedInputGate[] gates = new IndexedInputGate[partitionIds.length];
    for (int gateIndex = 0; gateIndex < gates.length; ++gateIndex) {
        final InputGateDeploymentDescriptor gateDescriptor = createInputGateDeploymentDescriptor(senderLocation, gateIndex, location);
        final IndexedInputGate gate = createInputGateWithMetrics(gateFactory, gateDescriptor, gateIndex);
        gate.setup();
        gates[gateIndex] = gate;
    }
    if (gates.length > 1) {
        return new UnionInputGate(gates);
    } else {
        return gates[0];
    }
}
Also used : UnionInputGate(org.apache.flink.runtime.io.network.partition.consumer.UnionInputGate) IndexedInputGate(org.apache.flink.runtime.io.network.partition.consumer.IndexedInputGate) InputGateDeploymentDescriptor(org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor)

Aggregations

IndexedInputGate (org.apache.flink.runtime.io.network.partition.consumer.IndexedInputGate)7 ArrayList (java.util.ArrayList)4 StreamConfig (org.apache.flink.streaming.api.graph.StreamConfig)4 IOException (java.io.IOException)2 Arrays (java.util.Arrays)2 Collections (java.util.Collections)2 List (java.util.List)2 CompletableFuture (java.util.concurrent.CompletableFuture)2 StreamEdge (org.apache.flink.streaming.api.graph.StreamEdge)2 Closeable (java.io.Closeable)1 ObjectInputStream (java.io.ObjectInputStream)1 Duration (java.time.Duration)1 Arrays.asList (java.util.Arrays.asList)1 Collections.singletonList (java.util.Collections.singletonList)1 Map (java.util.Map)1 Optional (java.util.Optional)1 OptionalLong (java.util.OptionalLong)1 CompletionException (java.util.concurrent.CompletionException)1 CountDownLatch (java.util.concurrent.CountDownLatch)1 ExecutionException (java.util.concurrent.ExecutionException)1