Examples with LatencyMarker org.apache.flink.streaming.runtime.streamrecord.LatencyMarker used on opensource projects

Example 1 with LatencyMarker

use of org.apache.flink.streaming.runtime.streamrecord.LatencyMarker in project flink by apache.

the class InputTest method setup.

@Before
public void setup() {
    element = new StreamRecord<>(GenericRowData.of(StringData.fromString("123")), 456);
    watermark = new Watermark(1223456789);
    latencyMarker = new LatencyMarker(122345678, new OperatorID(123, 456), 1);
}

Example 2 with LatencyMarker

use of org.apache.flink.streaming.runtime.streamrecord.LatencyMarker in project flink by apache.

the class SourceOperatorLatencyMetricsTest method testLatencyMarkEmission.

private void testLatencyMarkEmission(boolean shouldExpectLatencyMarkers, Configuration taskManagerConfig, ExecutionConfig executionConfig) throws Exception {
    try (SourceOperatorTestHarness testHarness = new SourceOperatorTestHarness(new SourceOperatorFactory(new MockSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), new MockEnvironmentBuilder().setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(taskManagerConfig)).setExecutionConfig(executionConfig).build())) {
        testHarness.open();
        testHarness.setup();
        for (long processingTime = 0; processingTime <= MAX_PROCESSING_TIME; processingTime++) {
            testHarness.getProcessingTimeService().setCurrentTime(processingTime);
            testHarness.emitNext();
        }
        List<LatencyMarker> expectedOutput = new ArrayList<>();
        if (!shouldExpectLatencyMarkers) {
            assertTrue(testHarness.getOutput().isEmpty());
        } else {
            expectedOutput.add(new LatencyMarker(1, testHarness.getOperator().getOperatorID(), 0));
            for (long markedTime = LATENCY_MARK_INTERVAL; markedTime <= MAX_PROCESSING_TIME; markedTime += LATENCY_MARK_INTERVAL) {
                expectedOutput.add(new LatencyMarker(markedTime, testHarness.getOperator().getOperatorID(), 0));
            }
            assertThat((Collection<Object>) testHarness.getOutput(), contains(expectedOutput.toArray()));
        }
    }
}

Example 3 with LatencyMarker

use of org.apache.flink.streaming.runtime.streamrecord.LatencyMarker in project flink by apache.

the class LatencyStatsTest method testLatencyStats.

private static void testLatencyStats(final LatencyStats.Granularity granularity, final Consumer<List<Tuple2<String, Histogram>>> verifier) {
    final AbstractMetricGroup<?> dummyGroup = UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup();
    final List<Tuple2<String, Histogram>> latencyHistograms = new ArrayList<>(4);
    final TestingMetricRegistry registry = TestingMetricRegistry.builder().setRegisterConsumer((metric, metricName, group) -> {
        if (metric instanceof Histogram) {
            latencyHistograms.add(Tuple2.of(group.getMetricIdentifier(metricName), (Histogram) metric));
        }
    }).build();
    final MetricGroup parentGroup = new GenericMetricGroup(registry, dummyGroup, PARENT_GROUP_NAME);
    final LatencyStats latencyStats = new LatencyStats(parentGroup, MetricOptions.LATENCY_HISTORY_SIZE.defaultValue(), OPERATOR_SUBTASK_INDEX, OPERATOR_ID, granularity);
    latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 0));
    latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 0));
    latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 1));
    latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_2, 2));
    latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_2, 3));
    verifier.accept(latencyHistograms);
}

Example 4 with LatencyMarker

use of org.apache.flink.streaming.runtime.streamrecord.LatencyMarker in project flink by apache.

the class MultipleInputStreamTaskTest method testLatencyMarker.

@Test
public void testLatencyMarker() throws Exception {
    final Map<String, Metric> metrics = new ConcurrentHashMap<>();
    final TaskMetricGroup taskMetricGroup = StreamTaskTestHarness.createTaskMetricGroup(metrics);
    try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).addInput(BasicTypeInfo.STRING_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.DOUBLE_TYPE_INFO).setupOutputForSingletonOperatorChain(new MapToStringMultipleInputOperatorFactory(3)).setTaskMetricGroup(taskMetricGroup).build()) {
        ArrayDeque<Object> expectedOutput = new ArrayDeque<>();
        OperatorID sourceId = new OperatorID();
        LatencyMarker latencyMarker = new LatencyMarker(42L, sourceId, 0);
        testHarness.processElement(latencyMarker);
        expectedOutput.add(latencyMarker);
        assertThat(testHarness.getOutput(), contains(expectedOutput.toArray()));
        testHarness.endInput();
        testHarness.waitForTaskCompletion();
    }
}

Example 5 with LatencyMarker

use of org.apache.flink.streaming.runtime.streamrecord.LatencyMarker in project flink by apache.

the class OutputTest method setup.

@Before
public void setup() {
    element = new StreamRecord<>(GenericRowData.of(StringData.fromString("123")), 456);
    watermark = new Watermark(1223456789);
    latencyMarker = new LatencyMarker(122345678, new OperatorID(123, 456), 1);
    serializer = InternalTypeInfo.of(RowType.of(DataTypes.STRING().getLogicalType())).createSerializer(new ExecutionConfig());
}