Examples with Histogram org.apache.flink.metrics.Histogram used on opensource projects

Example 6 with Histogram

use of org.apache.flink.metrics.Histogram in project flink by apache.

the class MetricQueryServiceTest method testHandleOversizedMetricMessage.

@Test
public void testHandleOversizedMetricMessage() throws Exception {
    final long sizeLimit = 200L;
    MetricQueryService queryService = MetricQueryService.createMetricQueryService(rpcService, ResourceID.generate(), sizeLimit);
    queryService.start();
    final TaskManagerMetricGroup tm = UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup();
    final String gaugeValue = "Hello";
    final long requiredGaugesToExceedLimit = sizeLimit / gaugeValue.length() + 1;
    List<Tuple2<String, Gauge<String>>> gauges = LongStream.range(0, requiredGaugesToExceedLimit).mapToObj(x -> Tuple2.of("gauge" + x, (Gauge<String>) () -> "Hello" + x)).collect(Collectors.toList());
    gauges.forEach(gauge -> queryService.addMetric(gauge.f0, gauge.f1, tm));
    queryService.addMetric("counter", new SimpleCounter(), tm);
    queryService.addMetric("histogram", new TestHistogram(), tm);
    queryService.addMetric("meter", new TestMeter(), tm);
    MetricDumpSerialization.MetricSerializationResult dump = queryService.queryMetrics(TIMEOUT).get();
    assertTrue(dump.serializedCounters.length > 0);
    assertEquals(1, dump.numCounters);
    assertTrue(dump.serializedMeters.length > 0);
    assertEquals(1, dump.numMeters);
    // gauges exceeded the size limit and will be excluded
    assertEquals(0, dump.serializedGauges.length);
    assertEquals(0, dump.numGauges);
    assertTrue(dump.serializedHistograms.length > 0);
    assertEquals(1, dump.numHistograms);
    // unregister all but one gauge to ensure gauges are reported again if the remaining fit
    for (int x = 1; x < gauges.size(); x++) {
        queryService.removeMetric(gauges.get(x).f1);
    }
    MetricDumpSerialization.MetricSerializationResult recoveredDump = queryService.queryMetrics(TIMEOUT).get();
    assertTrue(recoveredDump.serializedCounters.length > 0);
    assertEquals(1, recoveredDump.numCounters);
    assertTrue(recoveredDump.serializedMeters.length > 0);
    assertEquals(1, recoveredDump.numMeters);
    assertTrue(recoveredDump.serializedGauges.length > 0);
    assertEquals(1, recoveredDump.numGauges);
    assertTrue(recoveredDump.serializedHistograms.length > 0);
    assertEquals(1, recoveredDump.numHistograms);
}

Example 7 with Histogram

use of org.apache.flink.metrics.Histogram in project flink by apache.

the class LatencyStatsTest method testLatencyStats.

private static void testLatencyStats(final LatencyStats.Granularity granularity, final Consumer<List<Tuple2<String, Histogram>>> verifier) {
    final AbstractMetricGroup<?> dummyGroup = UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup();
    final List<Tuple2<String, Histogram>> latencyHistograms = new ArrayList<>(4);
    final TestingMetricRegistry registry = TestingMetricRegistry.builder().setRegisterConsumer((metric, metricName, group) -> {
        if (metric instanceof Histogram) {
            latencyHistograms.add(Tuple2.of(group.getMetricIdentifier(metricName), (Histogram) metric));
        }
    }).build();
    final MetricGroup parentGroup = new GenericMetricGroup(registry, dummyGroup, PARENT_GROUP_NAME);
    final LatencyStats latencyStats = new LatencyStats(parentGroup, MetricOptions.LATENCY_HISTORY_SIZE.defaultValue(), OPERATOR_SUBTASK_INDEX, OPERATOR_ID, granularity);
    latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 0));
    latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 0));
    latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 1));
    latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_2, 2));
    latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_2, 3));
    verifier.accept(latencyHistograms);
}

Example 8 with Histogram

use of org.apache.flink.metrics.Histogram in project flink by apache.

the class MetricFetcherTest method createRequestDumpAnswer.

private static MetricDumpSerialization.MetricSerializationResult createRequestDumpAnswer(InstanceID tmID, JobID jobID) throws IOException {
    Map<Counter, Tuple2<QueryScopeInfo, String>> counters = new HashMap<>();
    Map<Gauge<?>, Tuple2<QueryScopeInfo, String>> gauges = new HashMap<>();
    Map<Histogram, Tuple2<QueryScopeInfo, String>> histograms = new HashMap<>();
    Map<Meter, Tuple2<QueryScopeInfo, String>> meters = new HashMap<>();
    SimpleCounter c1 = new SimpleCounter();
    SimpleCounter c2 = new SimpleCounter();
    c1.inc(1);
    c2.inc(2);
    counters.put(c1, new Tuple2<QueryScopeInfo, String>(new QueryScopeInfo.OperatorQueryScopeInfo(jobID.toString(), "taskid", 2, "opname", "abc"), "oc"));
    counters.put(c2, new Tuple2<QueryScopeInfo, String>(new QueryScopeInfo.TaskQueryScopeInfo(jobID.toString(), "taskid", 2, "abc"), "tc"));
    meters.put(new Meter() {

        @Override
        public void markEvent() {
        }

        @Override
        public void markEvent(long n) {
        }

        @Override
        public double getRate() {
            return 5;
        }

        @Override
        public long getCount() {
            return 10;
        }
    }, new Tuple2<QueryScopeInfo, String>(new QueryScopeInfo.JobQueryScopeInfo(jobID.toString(), "abc"), "jc"));
    gauges.put(new Gauge<String>() {

        @Override
        public String getValue() {
            return "x";
        }
    }, new Tuple2<QueryScopeInfo, String>(new QueryScopeInfo.TaskManagerQueryScopeInfo(tmID.toString(), "abc"), "gauge"));
    histograms.put(new TestingHistogram(), new Tuple2<QueryScopeInfo, String>(new QueryScopeInfo.JobManagerQueryScopeInfo("abc"), "hist"));
    MetricDumpSerialization.MetricDumpSerializer serializer = new MetricDumpSerialization.MetricDumpSerializer();
    MetricDumpSerialization.MetricSerializationResult dump = serializer.serialize(counters, gauges, histograms, meters);
    serializer.close();
    return dump;
}

Example 9 with Histogram

use of org.apache.flink.metrics.Histogram in project flink by apache.

the class MetricQueryService method onReceive.

@Override
public void onReceive(Object message) {
    try {
        if (message instanceof AddMetric) {
            AddMetric added = (AddMetric) message;
            String metricName = added.metricName;
            Metric metric = added.metric;
            AbstractMetricGroup group = added.group;
            QueryScopeInfo info = group.getQueryServiceMetricInfo(FILTER);
            if (metric instanceof Counter) {
                counters.put((Counter) metric, new Tuple2<>(info, FILTER.filterCharacters(metricName)));
            } else if (metric instanceof Gauge) {
                gauges.put((Gauge<?>) metric, new Tuple2<>(info, FILTER.filterCharacters(metricName)));
            } else if (metric instanceof Histogram) {
                histograms.put((Histogram) metric, new Tuple2<>(info, FILTER.filterCharacters(metricName)));
            } else if (metric instanceof Meter) {
                meters.put((Meter) metric, new Tuple2<>(info, FILTER.filterCharacters(metricName)));
            }
        } else if (message instanceof RemoveMetric) {
            Metric metric = (((RemoveMetric) message).metric);
            if (metric instanceof Counter) {
                this.counters.remove(metric);
            } else if (metric instanceof Gauge) {
                this.gauges.remove(metric);
            } else if (metric instanceof Histogram) {
                this.histograms.remove(metric);
            } else if (metric instanceof Meter) {
                this.meters.remove(metric);
            }
        } else if (message instanceof CreateDump) {
            MetricDumpSerialization.MetricSerializationResult dump = serializer.serialize(counters, gauges, histograms, meters);
            getSender().tell(dump, getSelf());
        } else {
            LOG.warn("MetricQueryServiceActor received an invalid message. " + message.toString());
            getSender().tell(new Status.Failure(new IOException("MetricQueryServiceActor received an invalid message. " + message.toString())), getSelf());
        }
    } catch (Exception e) {
        LOG.warn("An exception occurred while processing a message.", e);
    }
}

Example 10 with Histogram

use of org.apache.flink.metrics.Histogram in project flink by apache.

the class PrometheusReporterTaskScopeTest method histogramsCanBeAddedSeveralTimesIfTheyDifferInLabels.

@Test
void histogramsCanBeAddedSeveralTimesIfTheyDifferInLabels() throws UnirestException {
    Histogram histogram = new TestHistogram();
    taskMetricGroup1.histogram("my_histogram", histogram);
    taskMetricGroup2.histogram("my_histogram", histogram);
    final String exportedMetrics = pollMetrics(reporter.getPort()).getBody();
    assertThat(exportedMetrics).contains(// histogram
    "subtask_index=\"0\",quantile=\"0.5\",} 0.5");
    assertThat(exportedMetrics).contains(// histogram
    "subtask_index=\"1\",quantile=\"0.5\",} 0.5");
    final String[] labelNamesWithQuantile = addToArray(LABEL_NAMES, "quantile");
    for (Double quantile : PrometheusReporter.HistogramSummaryProxy.QUANTILES) {
        assertThat(CollectorRegistry.defaultRegistry.getSampleValue("flink_taskmanager_job_task_my_histogram", labelNamesWithQuantile, addToArray(labelValues1, "" + quantile))).isEqualTo(quantile);
        assertThat(CollectorRegistry.defaultRegistry.getSampleValue("flink_taskmanager_job_task_my_histogram", labelNamesWithQuantile, addToArray(labelValues2, "" + quantile))).isEqualTo(quantile);
    }
}