Search in sources :

Example 6 with InterceptingOperatorMetricGroup

use of org.apache.flink.runtime.metrics.util.InterceptingOperatorMetricGroup in project flink by apache.

the class OneInputStreamTaskTest method testWatermarkMetrics.

@Test
@SuppressWarnings("unchecked")
public void testWatermarkMetrics() throws Exception {
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    OneInputStreamOperator<String, String> headOperator = new WatermarkMetricOperator();
    OperatorID headOperatorId = new OperatorID();
    OneInputStreamOperator<String, String> chainedOperator = new WatermarkMetricOperator();
    OperatorID chainedOperatorId = new OperatorID();
    testHarness.setupOperatorChain(headOperatorId, headOperator).chain(chainedOperatorId, chainedOperator, BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).finish();
    InterceptingOperatorMetricGroup headOperatorMetricGroup = new InterceptingOperatorMetricGroup();
    InterceptingOperatorMetricGroup chainedOperatorMetricGroup = new InterceptingOperatorMetricGroup();
    InterceptingTaskMetricGroup taskMetricGroup = new InterceptingTaskMetricGroup() {

        @Override
        public InternalOperatorMetricGroup getOrAddOperator(OperatorID id, String name) {
            if (id.equals(headOperatorId)) {
                return headOperatorMetricGroup;
            } else if (id.equals(chainedOperatorId)) {
                return chainedOperatorMetricGroup;
            } else {
                return super.getOrAddOperator(id, name);
            }
        }
    };
    StreamMockEnvironment env = new StreamMockEnvironment(testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize, new TestTaskStateManager()) {

        @Override
        public TaskMetricGroup getMetricGroup() {
            return taskMetricGroup;
        }
    };
    testHarness.invoke(env);
    testHarness.waitForTaskRunning();
    Gauge<Long> taskInputWatermarkGauge = (Gauge<Long>) taskMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
    Gauge<Long> headInputWatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
    Gauge<Long> headOutputWatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_OUTPUT_WATERMARK);
    Gauge<Long> chainedInputWatermarkGauge = (Gauge<Long>) chainedOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
    Gauge<Long> chainedOutputWatermarkGauge = (Gauge<Long>) chainedOperatorMetricGroup.get(MetricNames.IO_CURRENT_OUTPUT_WATERMARK);
    Assert.assertEquals("A metric was registered multiple times.", 5, new HashSet<>(Arrays.asList(taskInputWatermarkGauge, headInputWatermarkGauge, headOutputWatermarkGauge, chainedInputWatermarkGauge, chainedOutputWatermarkGauge)).size());
    Assert.assertEquals(Long.MIN_VALUE, taskInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headOutputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, chainedInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, chainedOutputWatermarkGauge.getValue().longValue());
    testHarness.processElement(new Watermark(1L));
    testHarness.waitForInputProcessing();
    Assert.assertEquals(1L, taskInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(1L, headInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, headOutputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, chainedInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(4L, chainedOutputWatermarkGauge.getValue().longValue());
    testHarness.processElement(new Watermark(2L));
    testHarness.waitForInputProcessing();
    Assert.assertEquals(2L, taskInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, headInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(4L, headOutputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(4L, chainedInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(8L, chainedOutputWatermarkGauge.getValue().longValue());
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
}
Also used : OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) Gauge(org.apache.flink.metrics.Gauge) TestTaskStateManager(org.apache.flink.runtime.state.TestTaskStateManager) InterceptingOperatorMetricGroup(org.apache.flink.runtime.metrics.util.InterceptingOperatorMetricGroup) InterceptingTaskMetricGroup(org.apache.flink.runtime.metrics.util.InterceptingTaskMetricGroup) MockInputSplitProvider(org.apache.flink.runtime.operators.testutils.MockInputSplitProvider) Watermark(org.apache.flink.streaming.api.watermark.Watermark) HashSet(java.util.HashSet) Test(org.junit.Test)

Aggregations

InterceptingOperatorMetricGroup (org.apache.flink.runtime.metrics.util.InterceptingOperatorMetricGroup)6 ExecutionConfig (org.apache.flink.api.common.ExecutionConfig)4 OperatorID (org.apache.flink.runtime.jobgraph.OperatorID)4 Test (org.junit.Test)4 Gauge (org.apache.flink.metrics.Gauge)3 InterceptingTaskMetricGroup (org.apache.flink.runtime.metrics.util.InterceptingTaskMetricGroup)3 Watermark (org.apache.flink.streaming.api.watermark.Watermark)3 HashSet (java.util.HashSet)2 JobStatus (org.apache.flink.api.common.JobStatus)2 MockInputSplitProvider (org.apache.flink.runtime.operators.testutils.MockInputSplitProvider)2 TestTaskStateManager (org.apache.flink.runtime.state.TestTaskStateManager)2 ListStateDescriptor (org.apache.flink.api.common.state.ListStateDescriptor)1 ValueStateDescriptor (org.apache.flink.api.common.state.ValueStateDescriptor)1 MockSource (org.apache.flink.api.connector.source.mocks.MockSource)1 MetricOptions (org.apache.flink.configuration.MetricOptions)1 CloseableRegistry (org.apache.flink.core.fs.CloseableRegistry)1 CheckpointException (org.apache.flink.runtime.checkpoint.CheckpointException)1 ExpectedTestException (org.apache.flink.runtime.operators.testutils.ExpectedTestException)1 MockEnvironmentBuilder (org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder)1 InputChannelStateHandle (org.apache.flink.runtime.state.InputChannelStateHandle)1