Examples with WatermarkMetricOperator org.apache.flink.streaming.runtime.tasks.OneInputStreamTaskTest.WatermarkMetricOperator used on opensource projects

Example 1 with WatermarkMetricOperator

use of org.apache.flink.streaming.runtime.tasks.OneInputStreamTaskTest.WatermarkMetricOperator in project flink by apache.

the class MultipleInputStreamTaskTest method testWatermarkMetrics.

@Test
@SuppressWarnings("unchecked")
public void testWatermarkMetrics() throws Exception {
    OperatorID mainOperatorId = new OperatorID();
    OperatorID chainedOperatorId = new OperatorID();
    InterceptingOperatorMetricGroup mainOperatorMetricGroup = new InterceptingOperatorMetricGroup();
    InterceptingOperatorMetricGroup chainedOperatorMetricGroup = new InterceptingOperatorMetricGroup();
    InterceptingTaskMetricGroup taskMetricGroup = new InterceptingTaskMetricGroup() {

        @Override
        public InternalOperatorMetricGroup getOrAddOperator(OperatorID id, String name) {
            if (id.equals(mainOperatorId)) {
                return mainOperatorMetricGroup;
            } else if (id.equals(chainedOperatorId)) {
                return chainedOperatorMetricGroup;
            } else {
                return super.getOrAddOperator(id, name);
            }
        }
    };
    try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).modifyExecutionConfig(applyObjectReuse(objectReuse)).addInput(BasicTypeInfo.STRING_TYPE_INFO).addSourceInput(new SourceOperatorFactory<>(new MockSource(Boundedness.CONTINUOUS_UNBOUNDED, 2, true, false), WatermarkStrategy.forGenerator(ctx -> new RecordToWatermarkGenerator())), BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.DOUBLE_TYPE_INFO).setupOperatorChain(mainOperatorId, new MapToStringMultipleInputOperatorFactory(3)).chain(chainedOperatorId, new WatermarkMetricOperator(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).finish().setTaskMetricGroup(taskMetricGroup).build()) {
        Gauge<Long> taskInputWatermarkGauge = (Gauge<Long>) taskMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
        Gauge<Long> mainInput1WatermarkGauge = (Gauge<Long>) mainOperatorMetricGroup.get(MetricNames.currentInputWatermarkName(1));
        Gauge<Long> mainInput2WatermarkGauge = (Gauge<Long>) mainOperatorMetricGroup.get(MetricNames.currentInputWatermarkName(2));
        Gauge<Long> mainInput3WatermarkGauge = (Gauge<Long>) mainOperatorMetricGroup.get(MetricNames.currentInputWatermarkName(3));
        Gauge<Long> mainInputWatermarkGauge = (Gauge<Long>) mainOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
        Gauge<Long> mainOutputWatermarkGauge = (Gauge<Long>) mainOperatorMetricGroup.get(MetricNames.IO_CURRENT_OUTPUT_WATERMARK);
        Gauge<Long> chainedInputWatermarkGauge = (Gauge<Long>) chainedOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
        Gauge<Long> chainedOutputWatermarkGauge = (Gauge<Long>) chainedOperatorMetricGroup.get(MetricNames.IO_CURRENT_OUTPUT_WATERMARK);
        assertEquals(Long.MIN_VALUE, taskInputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInput1WatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInput2WatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInput3WatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainOutputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, chainedInputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, chainedOutputWatermarkGauge.getValue().longValue());
        testHarness.processElement(new Watermark(1L), 0);
        assertEquals(Long.MIN_VALUE, taskInputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInputWatermarkGauge.getValue().longValue());
        assertEquals(1L, mainInput1WatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInput2WatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInput3WatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainOutputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, chainedInputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, chainedOutputWatermarkGauge.getValue().longValue());
        addSourceRecords(testHarness, 1, 2);
        testHarness.processAll();
        assertEquals(Long.MIN_VALUE, taskInputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInputWatermarkGauge.getValue().longValue());
        assertEquals(1L, mainInput1WatermarkGauge.getValue().longValue());
        assertEquals(2L, mainInput2WatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInput3WatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainOutputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, chainedInputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, chainedOutputWatermarkGauge.getValue().longValue());
        testHarness.processElement(new Watermark(2L), 1);
        assertEquals(1L, taskInputWatermarkGauge.getValue().longValue());
        assertEquals(1L, mainInputWatermarkGauge.getValue().longValue());
        assertEquals(1L, mainInput1WatermarkGauge.getValue().longValue());
        assertEquals(2L, mainInput2WatermarkGauge.getValue().longValue());
        assertEquals(2L, mainInput3WatermarkGauge.getValue().longValue());
        assertEquals(1L, mainOutputWatermarkGauge.getValue().longValue());
        assertEquals(1L, chainedInputWatermarkGauge.getValue().longValue());
        assertEquals(2L, chainedOutputWatermarkGauge.getValue().longValue());
        testHarness.processElement(new Watermark(4L), 0);
        addSourceRecords(testHarness, 1, 3);
        testHarness.processAll();
        assertEquals(2L, taskInputWatermarkGauge.getValue().longValue());
        assertEquals(2L, mainInputWatermarkGauge.getValue().longValue());
        assertEquals(4L, mainInput1WatermarkGauge.getValue().longValue());
        assertEquals(3L, mainInput2WatermarkGauge.getValue().longValue());
        assertEquals(2L, mainInput3WatermarkGauge.getValue().longValue());
        assertEquals(2L, mainOutputWatermarkGauge.getValue().longValue());
        assertEquals(2L, chainedInputWatermarkGauge.getValue().longValue());
        assertEquals(4L, chainedOutputWatermarkGauge.getValue().longValue());
        finishAddingRecords(testHarness, 1);
        testHarness.endInput();
        testHarness.waitForTaskCompletion();
        testHarness.finishProcessing();
    }
}