Examples with InterceptingOperatorMetricGroup org.apache.flink.runtime.metrics.util.InterceptingOperatorMetricGroup used on opensource projects

Example 1 with InterceptingOperatorMetricGroup

use of org.apache.flink.runtime.metrics.util.InterceptingOperatorMetricGroup in project flink by apache.

the class JobStatusMetricsTest method testStatusSelection.

@Test
void testStatusSelection() {
    final InterceptingOperatorMetricGroup metricGroup = new InterceptingOperatorMetricGroup();
    final JobStatusMetrics jobStatusMetrics = new JobStatusMetrics(0L, enable(MetricOptions.JobStatusMetrics.STATE));
    jobStatusMetrics.registerMetrics(metricGroup);
    final Map<JobStatus, StateTimeMetricTest.StatusMetricSet> registeredMetrics = extractMetrics(metricGroup);
    for (JobStatus value : JobStatus.values()) {
        if (value.isTerminalState() || value == JobStatus.RECONCILING) {
            assertThat(registeredMetrics).doesNotContainKey(value);
        } else {
            assertThat(registeredMetrics).containsKey(value);
        }
    }
}

Example 2 with InterceptingOperatorMetricGroup

use of org.apache.flink.runtime.metrics.util.InterceptingOperatorMetricGroup in project flink by apache.

the class StateTimeMetricTest method testMetricSelection.

private static void testMetricSelection(MetricOptions.JobStatusMetrics... selectedMetrics) {
    final EnumSet<MetricOptions.JobStatusMetrics> selectedMetricsSet = EnumSet.noneOf(MetricOptions.JobStatusMetrics.class);
    Arrays.stream(selectedMetrics).forEach(selectedMetricsSet::add);
    final InterceptingOperatorMetricGroup metricGroup = new InterceptingOperatorMetricGroup();
    StateTimeMetric.register(enable(selectedMetrics), metricGroup, new TestStateTimeMetric(), "test");
    final Map<JobStatus, StatusMetricSet> registeredMetrics = extractMetrics(metricGroup);
    for (StatusMetricSet metrics : registeredMetrics.values()) {
        assertThat(metrics.getState().isPresent()).isEqualTo(selectedMetricsSet.contains(MetricOptions.JobStatusMetrics.STATE));
        assertThat(metrics.getCurrentTime().isPresent()).isEqualTo(selectedMetricsSet.contains(MetricOptions.JobStatusMetrics.CURRENT_TIME));
        assertThat(metrics.getTotalTime().isPresent()).isEqualTo(selectedMetricsSet.contains(MetricOptions.JobStatusMetrics.TOTAL_TIME));
    }
}

Example 3 with InterceptingOperatorMetricGroup

use of org.apache.flink.runtime.metrics.util.InterceptingOperatorMetricGroup in project flink by apache.

the class StreamOperatorStateHandlerTest method testFailingBackendSnapshotMethod.

/**
 * Tests that a failing snapshot method call to the keyed state backend will trigger the closing
 * of the StateSnapshotContextSynchronousImpl and the cancellation of the
 * OperatorSnapshotResult. The latter is supposed to also cancel all assigned futures.
 */
@Test
public void testFailingBackendSnapshotMethod() throws Exception {
    final long checkpointId = 42L;
    final long timestamp = 1L;
    try (CloseableRegistry closeableRegistry = new CloseableRegistry()) {
        RunnableFuture<SnapshotResult<KeyedStateHandle>> keyedStateManagedFuture = new CancelableFuture<>();
        RunnableFuture<SnapshotResult<KeyedStateHandle>> keyedStateRawFuture = new CancelableFuture<>();
        RunnableFuture<SnapshotResult<OperatorStateHandle>> operatorStateManagedFuture = new CancelableFuture<>();
        RunnableFuture<SnapshotResult<OperatorStateHandle>> operatorStateRawFuture = new CancelableFuture<>();
        RunnableFuture<SnapshotResult<StateObjectCollection<InputChannelStateHandle>>> inputChannelStateFuture = new CancelableFuture<>();
        RunnableFuture<SnapshotResult<StateObjectCollection<ResultSubpartitionStateHandle>>> resultSubpartitionStateFuture = new CancelableFuture<>();
        OperatorSnapshotFutures operatorSnapshotResult = new OperatorSnapshotFutures(keyedStateManagedFuture, keyedStateRawFuture, operatorStateManagedFuture, operatorStateRawFuture, inputChannelStateFuture, resultSubpartitionStateFuture);
        StateSnapshotContextSynchronousImpl context = new TestStateSnapshotContextSynchronousImpl(checkpointId, timestamp, closeableRegistry);
        context.getRawKeyedOperatorStateOutput();
        context.getRawOperatorStateOutput();
        StreamTaskStateInitializerImpl stateInitializer = new StreamTaskStateInitializerImpl(new MockEnvironmentBuilder().build(), new MemoryStateBackend());
        StreamOperatorStateContext stateContext = stateInitializer.streamOperatorStateContext(new OperatorID(), "whatever", new TestProcessingTimeService(), new UnUsedKeyContext(), IntSerializer.INSTANCE, closeableRegistry, new InterceptingOperatorMetricGroup(), 1.0, false);
        StreamOperatorStateHandler stateHandler = new StreamOperatorStateHandler(stateContext, new ExecutionConfig(), closeableRegistry);
        final String keyedStateField = "keyedStateField";
        final String operatorStateField = "operatorStateField";
        CheckpointedStreamOperator checkpointedStreamOperator = new CheckpointedStreamOperator() {

            @Override
            public void initializeState(StateInitializationContext context) throws Exception {
                context.getKeyedStateStore().getState(new ValueStateDescriptor<>(keyedStateField, LongSerializer.INSTANCE)).update(42L);
                context.getOperatorStateStore().getListState(new ListStateDescriptor<>(operatorStateField, LongSerializer.INSTANCE)).add(42L);
            }

            @Override
            public void snapshotState(StateSnapshotContext context) throws Exception {
                throw new ExpectedTestException();
            }
        };
        stateHandler.setCurrentKey("44");
        stateHandler.initializeOperatorState(checkpointedStreamOperator);
        assertThat(stateContext.operatorStateBackend().getRegisteredStateNames(), is(not(empty())));
        assertThat(((AbstractKeyedStateBackend<?>) stateContext.keyedStateBackend()).numKeyValueStatesByName(), equalTo(1));
        try {
            stateHandler.snapshotState(checkpointedStreamOperator, Optional.of(stateContext.internalTimerServiceManager()), "42", 42, 42, CheckpointOptions.forCheckpointWithDefaultLocation(), new MemCheckpointStreamFactory(1024), operatorSnapshotResult, context, false);
            fail("Exception expected.");
        } catch (CheckpointException e) {
            // as CheckpointException is wrapping the cause with SerializedThrowable
            if (!ExceptionUtils.findThrowableWithMessage(e, ExpectedTestException.MESSAGE).isPresent()) {
                throw e;
            }
        }
        assertTrue(keyedStateManagedFuture.isCancelled());
        assertTrue(keyedStateRawFuture.isCancelled());
        assertTrue(context.getKeyedStateStreamFuture().isCancelled());
        assertTrue(operatorStateManagedFuture.isCancelled());
        assertTrue(operatorStateRawFuture.isCancelled());
        assertTrue(context.getOperatorStateStreamFuture().isCancelled());
        assertTrue(inputChannelStateFuture.isCancelled());
        assertTrue(resultSubpartitionStateFuture.isCancelled());
        stateHandler.dispose();
        assertThat(stateContext.operatorStateBackend().getRegisteredBroadcastStateNames(), is(empty()));
        assertThat(stateContext.operatorStateBackend().getRegisteredStateNames(), is(empty()));
        assertThat(((AbstractKeyedStateBackend<?>) stateContext.keyedStateBackend()).numKeyValueStatesByName(), equalTo(0));
    }
}

Example 4 with InterceptingOperatorMetricGroup

use of org.apache.flink.runtime.metrics.util.InterceptingOperatorMetricGroup in project flink by apache.

the class MultipleInputStreamTaskTest method testWatermarkMetrics.

@Test
@SuppressWarnings("unchecked")
public void testWatermarkMetrics() throws Exception {
    OperatorID mainOperatorId = new OperatorID();
    OperatorID chainedOperatorId = new OperatorID();
    InterceptingOperatorMetricGroup mainOperatorMetricGroup = new InterceptingOperatorMetricGroup();
    InterceptingOperatorMetricGroup chainedOperatorMetricGroup = new InterceptingOperatorMetricGroup();
    InterceptingTaskMetricGroup taskMetricGroup = new InterceptingTaskMetricGroup() {

        @Override
        public InternalOperatorMetricGroup getOrAddOperator(OperatorID id, String name) {
            if (id.equals(mainOperatorId)) {
                return mainOperatorMetricGroup;
            } else if (id.equals(chainedOperatorId)) {
                return chainedOperatorMetricGroup;
            } else {
                return super.getOrAddOperator(id, name);
            }
        }
    };
    try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).modifyExecutionConfig(applyObjectReuse(objectReuse)).addInput(BasicTypeInfo.STRING_TYPE_INFO).addSourceInput(new SourceOperatorFactory<>(new MockSource(Boundedness.CONTINUOUS_UNBOUNDED, 2, true, false), WatermarkStrategy.forGenerator(ctx -> new RecordToWatermarkGenerator())), BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.DOUBLE_TYPE_INFO).setupOperatorChain(mainOperatorId, new MapToStringMultipleInputOperatorFactory(3)).chain(chainedOperatorId, new WatermarkMetricOperator(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).finish().setTaskMetricGroup(taskMetricGroup).build()) {
        Gauge<Long> taskInputWatermarkGauge = (Gauge<Long>) taskMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
        Gauge<Long> mainInput1WatermarkGauge = (Gauge<Long>) mainOperatorMetricGroup.get(MetricNames.currentInputWatermarkName(1));
        Gauge<Long> mainInput2WatermarkGauge = (Gauge<Long>) mainOperatorMetricGroup.get(MetricNames.currentInputWatermarkName(2));
        Gauge<Long> mainInput3WatermarkGauge = (Gauge<Long>) mainOperatorMetricGroup.get(MetricNames.currentInputWatermarkName(3));
        Gauge<Long> mainInputWatermarkGauge = (Gauge<Long>) mainOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
        Gauge<Long> mainOutputWatermarkGauge = (Gauge<Long>) mainOperatorMetricGroup.get(MetricNames.IO_CURRENT_OUTPUT_WATERMARK);
        Gauge<Long> chainedInputWatermarkGauge = (Gauge<Long>) chainedOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
        Gauge<Long> chainedOutputWatermarkGauge = (Gauge<Long>) chainedOperatorMetricGroup.get(MetricNames.IO_CURRENT_OUTPUT_WATERMARK);
        assertEquals(Long.MIN_VALUE, taskInputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInput1WatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInput2WatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInput3WatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainOutputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, chainedInputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, chainedOutputWatermarkGauge.getValue().longValue());
        testHarness.processElement(new Watermark(1L), 0);
        assertEquals(Long.MIN_VALUE, taskInputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInputWatermarkGauge.getValue().longValue());
        assertEquals(1L, mainInput1WatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInput2WatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInput3WatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainOutputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, chainedInputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, chainedOutputWatermarkGauge.getValue().longValue());
        addSourceRecords(testHarness, 1, 2);
        testHarness.processAll();
        assertEquals(Long.MIN_VALUE, taskInputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInputWatermarkGauge.getValue().longValue());
        assertEquals(1L, mainInput1WatermarkGauge.getValue().longValue());
        assertEquals(2L, mainInput2WatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainInput3WatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, mainOutputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, chainedInputWatermarkGauge.getValue().longValue());
        assertEquals(Long.MIN_VALUE, chainedOutputWatermarkGauge.getValue().longValue());
        testHarness.processElement(new Watermark(2L), 1);
        assertEquals(1L, taskInputWatermarkGauge.getValue().longValue());
        assertEquals(1L, mainInputWatermarkGauge.getValue().longValue());
        assertEquals(1L, mainInput1WatermarkGauge.getValue().longValue());
        assertEquals(2L, mainInput2WatermarkGauge.getValue().longValue());
        assertEquals(2L, mainInput3WatermarkGauge.getValue().longValue());
        assertEquals(1L, mainOutputWatermarkGauge.getValue().longValue());
        assertEquals(1L, chainedInputWatermarkGauge.getValue().longValue());
        assertEquals(2L, chainedOutputWatermarkGauge.getValue().longValue());
        testHarness.processElement(new Watermark(4L), 0);
        addSourceRecords(testHarness, 1, 3);
        testHarness.processAll();
        assertEquals(2L, taskInputWatermarkGauge.getValue().longValue());
        assertEquals(2L, mainInputWatermarkGauge.getValue().longValue());
        assertEquals(4L, mainInput1WatermarkGauge.getValue().longValue());
        assertEquals(3L, mainInput2WatermarkGauge.getValue().longValue());
        assertEquals(2L, mainInput3WatermarkGauge.getValue().longValue());
        assertEquals(2L, mainOutputWatermarkGauge.getValue().longValue());
        assertEquals(2L, chainedInputWatermarkGauge.getValue().longValue());
        assertEquals(4L, chainedOutputWatermarkGauge.getValue().longValue());
        finishAddingRecords(testHarness, 1);
        testHarness.endInput();
        testHarness.waitForTaskCompletion();
        testHarness.finishProcessing();
    }
}

Example 5 with InterceptingOperatorMetricGroup

use of org.apache.flink.runtime.metrics.util.InterceptingOperatorMetricGroup in project flink by apache.

the class TwoInputStreamTaskTest method testWatermarkMetrics.

@Test
@SuppressWarnings("unchecked")
public void testWatermarkMetrics() throws Exception {
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness = new TwoInputStreamTaskTestHarness<>(TwoInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    CoStreamMap<String, Integer, String> headOperator = new CoStreamMap<>(new IdentityMap());
    final OperatorID headOperatorId = new OperatorID();
    OneInputStreamTaskTest.WatermarkMetricOperator chainedOperator = new OneInputStreamTaskTest.WatermarkMetricOperator();
    OperatorID chainedOperatorId = new OperatorID();
    testHarness.setupOperatorChain(headOperatorId, headOperator).chain(chainedOperatorId, chainedOperator, BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).finish();
    InterceptingOperatorMetricGroup headOperatorMetricGroup = new InterceptingOperatorMetricGroup();
    InterceptingOperatorMetricGroup chainedOperatorMetricGroup = new InterceptingOperatorMetricGroup();
    InterceptingTaskMetricGroup taskMetricGroup = new InterceptingTaskMetricGroup() {

        @Override
        public InternalOperatorMetricGroup getOrAddOperator(OperatorID id, String name) {
            if (id.equals(headOperatorId)) {
                return headOperatorMetricGroup;
            } else if (id.equals(chainedOperatorId)) {
                return chainedOperatorMetricGroup;
            } else {
                return super.getOrAddOperator(id, name);
            }
        }
    };
    StreamMockEnvironment env = new StreamMockEnvironment(testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize, new TestTaskStateManager()) {

        @Override
        public TaskMetricGroup getMetricGroup() {
            return taskMetricGroup;
        }
    };
    testHarness.invoke(env);
    testHarness.waitForTaskRunning();
    Gauge<Long> taskInputWatermarkGauge = (Gauge<Long>) taskMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
    Gauge<Long> headInput1WatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_1_WATERMARK);
    Gauge<Long> headInput2WatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_2_WATERMARK);
    Gauge<Long> headInputWatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
    Gauge<Long> headOutputWatermarkGauge = (Gauge<Long>) headOperatorMetricGroup.get(MetricNames.IO_CURRENT_OUTPUT_WATERMARK);
    Gauge<Long> chainedInputWatermarkGauge = (Gauge<Long>) chainedOperatorMetricGroup.get(MetricNames.IO_CURRENT_INPUT_WATERMARK);
    Gauge<Long> chainedOutputWatermarkGauge = (Gauge<Long>) chainedOperatorMetricGroup.get(MetricNames.IO_CURRENT_OUTPUT_WATERMARK);
    Assert.assertEquals("A metric was registered multiple times.", 7, new HashSet<>(Arrays.asList(taskInputWatermarkGauge, headInput1WatermarkGauge, headInput2WatermarkGauge, headInputWatermarkGauge, headOutputWatermarkGauge, chainedInputWatermarkGauge, chainedOutputWatermarkGauge)).size());
    Assert.assertEquals(Long.MIN_VALUE, taskInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headInput1WatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headInput2WatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headOutputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, chainedInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, chainedOutputWatermarkGauge.getValue().longValue());
    testHarness.processElement(new Watermark(1L), 0, 0);
    testHarness.waitForInputProcessing();
    Assert.assertEquals(Long.MIN_VALUE, taskInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(1L, headInput1WatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headInput2WatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, headOutputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, chainedInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(Long.MIN_VALUE, chainedOutputWatermarkGauge.getValue().longValue());
    testHarness.processElement(new Watermark(2L), 1, 0);
    testHarness.waitForInputProcessing();
    Assert.assertEquals(1L, taskInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(1L, headInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(1L, headInput1WatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, headInput2WatermarkGauge.getValue().longValue());
    Assert.assertEquals(1L, headOutputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(1L, chainedInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, chainedOutputWatermarkGauge.getValue().longValue());
    testHarness.processElement(new Watermark(3L), 0, 0);
    testHarness.waitForInputProcessing();
    Assert.assertEquals(2L, taskInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, headInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(3L, headInput1WatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, headInput2WatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, headOutputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(2L, chainedInputWatermarkGauge.getValue().longValue());
    Assert.assertEquals(4L, chainedOutputWatermarkGauge.getValue().longValue());
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
}