Examples with Histogram org.apache.flink.metrics.Histogram used on opensource projects

Example 1 with Histogram

use of org.apache.flink.metrics.Histogram in project flink by apache.

the class Slf4jReporter method tryReport.

private void tryReport() {
    // initialize with previous size to avoid repeated resizing of backing array
    // pad the size to allow deviations in the final string, for example due to different double
    // value representations
    StringBuilder builder = new StringBuilder((int) (previousSize * 1.1));
    builder.append(lineSeparator).append("=========================== Starting metrics report ===========================").append(lineSeparator);
    builder.append(lineSeparator).append("-- Counters -------------------------------------------------------------------").append(lineSeparator);
    for (Map.Entry<Counter, String> metric : counters.entrySet()) {
        builder.append(metric.getValue()).append(": ").append(metric.getKey().getCount()).append(lineSeparator);
    }
    builder.append(lineSeparator).append("-- Gauges ---------------------------------------------------------------------").append(lineSeparator);
    for (Map.Entry<Gauge<?>, String> metric : gauges.entrySet()) {
        builder.append(metric.getValue()).append(": ").append(metric.getKey().getValue()).append(lineSeparator);
    }
    builder.append(lineSeparator).append("-- Meters ---------------------------------------------------------------------").append(lineSeparator);
    for (Map.Entry<Meter, String> metric : meters.entrySet()) {
        builder.append(metric.getValue()).append(": ").append(metric.getKey().getRate()).append(lineSeparator);
    }
    builder.append(lineSeparator).append("-- Histograms -----------------------------------------------------------------").append(lineSeparator);
    for (Map.Entry<Histogram, String> metric : histograms.entrySet()) {
        HistogramStatistics stats = metric.getKey().getStatistics();
        builder.append(metric.getValue()).append(": count=").append(stats.size()).append(", min=").append(stats.getMin()).append(", max=").append(stats.getMax()).append(", mean=").append(stats.getMean()).append(", stddev=").append(stats.getStdDev()).append(", p50=").append(stats.getQuantile(0.50)).append(", p75=").append(stats.getQuantile(0.75)).append(", p95=").append(stats.getQuantile(0.95)).append(", p98=").append(stats.getQuantile(0.98)).append(", p99=").append(stats.getQuantile(0.99)).append(", p999=").append(stats.getQuantile(0.999)).append(lineSeparator);
    }
    builder.append(lineSeparator).append("=========================== Finished metrics report ===========================").append(lineSeparator);
    LOG.info(builder.toString());
    previousSize = builder.length();
}

Example 2 with Histogram

use of org.apache.flink.metrics.Histogram in project flink by apache.

the class ScheduledDropwizardReporter method notifyOfAddedMetric.

// ------------------------------------------------------------------------
// adding / removing metrics
// ------------------------------------------------------------------------
@Override
public void notifyOfAddedMetric(Metric metric, String metricName, MetricGroup group) {
    final String fullName = group.getMetricIdentifier(metricName, this);
    synchronized (this) {
        if (metric instanceof Counter) {
            counters.put((Counter) metric, fullName);
            registry.register(fullName, new FlinkCounterWrapper((Counter) metric));
        } else if (metric instanceof Gauge) {
            gauges.put((Gauge<?>) metric, fullName);
            registry.register(fullName, FlinkGaugeWrapper.fromGauge((Gauge<?>) metric));
        } else if (metric instanceof Histogram) {
            Histogram histogram = (Histogram) metric;
            histograms.put(histogram, fullName);
            if (histogram instanceof DropwizardHistogramWrapper) {
                registry.register(fullName, ((DropwizardHistogramWrapper) histogram).getDropwizardHistogram());
            } else {
                registry.register(fullName, new FlinkHistogramWrapper(histogram));
            }
        } else if (metric instanceof Meter) {
            Meter meter = (Meter) metric;
            meters.put(meter, fullName);
            if (meter instanceof DropwizardMeterWrapper) {
                registry.register(fullName, ((DropwizardMeterWrapper) meter).getDropwizardMeter());
            } else {
                registry.register(fullName, new FlinkMeterWrapper(meter));
            }
        } else {
            log.warn("Cannot add metric of type {}. This indicates that the reporter " + "does not support this metric type.", metric.getClass().getName());
        }
    }
}

Example 3 with Histogram

use of org.apache.flink.metrics.Histogram in project flink by apache.

the class MetricMapperTest method testMapHistogram.

@Test
void testMapHistogram() {
    Histogram histogram = new TestHistogram();
    verifyPoint(MetricMapper.map(INFO, TIMESTAMP, histogram), "count=3", "max=6", "mean=4.0", "min=7", "p50=0.5", "p75=0.75", "p95=0.95", "p98=0.98", "p99=0.99", "p999=0.999", "stddev=5.0");
}

Example 4 with Histogram

use of org.apache.flink.metrics.Histogram in project flink by apache.

the class ScheduledDropwizardReporterTest method testMetricCleanup.

/**
 * This test verifies that metrics are properly added and removed to/from the
 * ScheduledDropwizardReporter and the underlying Dropwizard MetricRegistry.
 */
@Test
void testMetricCleanup() {
    TestingScheduledDropwizardReporter rep = new TestingScheduledDropwizardReporter();
    MetricGroup mp = new UnregisteredMetricsGroup();
    Counter c = new SimpleCounter();
    Meter m = new TestMeter();
    Histogram h = new TestHistogram();
    Gauge<?> g = () -> null;
    rep.notifyOfAddedMetric(c, "counter", mp);
    assertThat(rep.getCounters()).hasSize(1);
    assertThat(rep.registry.getCounters()).hasSize(1);
    rep.notifyOfAddedMetric(m, "meter", mp);
    assertThat(rep.getMeters()).hasSize(1);
    assertThat(rep.registry.getMeters()).hasSize(1);
    rep.notifyOfAddedMetric(h, "histogram", mp);
    assertThat(rep.getHistograms()).hasSize(1);
    assertThat(rep.registry.getHistograms()).hasSize(1);
    rep.notifyOfAddedMetric(g, "gauge", mp);
    assertThat(rep.getGauges()).hasSize(1);
    assertThat(rep.registry.getGauges()).hasSize(1);
    rep.notifyOfRemovedMetric(c, "counter", mp);
    assertThat(rep.getCounters()).hasSize(0);
    assertThat(rep.registry.getCounters()).hasSize(0);
    rep.notifyOfRemovedMetric(m, "meter", mp);
    assertThat(rep.getMeters()).hasSize(0);
    assertThat(rep.registry.getMeters()).hasSize(0);
    rep.notifyOfRemovedMetric(h, "histogram", mp);
    assertThat(rep.getHistograms()).hasSize(0);
    assertThat(rep.registry.getHistograms()).hasSize(0);
    rep.notifyOfRemovedMetric(g, "gauge", mp);
    assertThat(rep.getGauges()).hasSize(0);
    assertThat(rep.registry.getGauges()).hasSize(0);
}

Example 5 with Histogram

use of org.apache.flink.metrics.Histogram in project flink by apache.

the class MetricDumpSerializerTest method testSerialization.

@Test
public void testSerialization() throws IOException {
    MetricDumpSerialization.MetricDumpSerializer serializer = new MetricDumpSerialization.MetricDumpSerializer();
    MetricDumpSerialization.MetricDumpDeserializer deserializer = new MetricDumpSerialization.MetricDumpDeserializer();
    Map<Counter, Tuple2<QueryScopeInfo, String>> counters = new HashMap<>();
    Map<Gauge<?>, Tuple2<QueryScopeInfo, String>> gauges = new HashMap<>();
    Map<Histogram, Tuple2<QueryScopeInfo, String>> histograms = new HashMap<>();
    Map<Meter, Tuple2<QueryScopeInfo, String>> meters = new HashMap<>();
    SimpleCounter c1 = new SimpleCounter();
    SimpleCounter c2 = new SimpleCounter();
    c1.inc(1);
    c2.inc(2);
    Gauge<Integer> g1 = new Gauge<Integer>() {

        @Override
        public Integer getValue() {
            return 4;
        }
    };
    Histogram h1 = new TestHistogram();
    Meter m1 = new Meter() {

        @Override
        public void markEvent() {
        }

        @Override
        public void markEvent(long n) {
        }

        @Override
        public double getRate() {
            return 5;
        }

        @Override
        public long getCount() {
            return 10;
        }
    };
    counters.put(c1, new Tuple2<QueryScopeInfo, String>(new QueryScopeInfo.JobManagerQueryScopeInfo("A"), "c1"));
    counters.put(c2, new Tuple2<QueryScopeInfo, String>(new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid", "B"), "c2"));
    meters.put(m1, new Tuple2<QueryScopeInfo, String>(new QueryScopeInfo.JobQueryScopeInfo("jid", "C"), "c3"));
    gauges.put(g1, new Tuple2<QueryScopeInfo, String>(new QueryScopeInfo.TaskQueryScopeInfo("jid", "vid", 2, "D"), "g1"));
    histograms.put(h1, new Tuple2<QueryScopeInfo, String>(new QueryScopeInfo.OperatorQueryScopeInfo("jid", "vid", 2, "opname", "E"), "h1"));
    MetricDumpSerialization.MetricSerializationResult serialized = serializer.serialize(counters, gauges, histograms, meters);
    List<MetricDump> deserialized = deserializer.deserialize(serialized);
    // ===== Counters
    // ==============================================================================================
    assertEquals(5, deserialized.size());
    for (MetricDump metric : deserialized) {
        switch(metric.getCategory()) {
            case METRIC_CATEGORY_COUNTER:
                MetricDump.CounterDump counterDump = (MetricDump.CounterDump) metric;
                switch((byte) counterDump.count) {
                    case 1:
                        assertTrue(counterDump.scopeInfo instanceof QueryScopeInfo.JobManagerQueryScopeInfo);
                        assertEquals("A", counterDump.scopeInfo.scope);
                        assertEquals("c1", counterDump.name);
                        counters.remove(c1);
                        break;
                    case 2:
                        assertTrue(counterDump.scopeInfo instanceof QueryScopeInfo.TaskManagerQueryScopeInfo);
                        assertEquals("B", counterDump.scopeInfo.scope);
                        assertEquals("c2", counterDump.name);
                        assertEquals("tmid", ((QueryScopeInfo.TaskManagerQueryScopeInfo) counterDump.scopeInfo).taskManagerID);
                        counters.remove(c2);
                        break;
                    default:
                        fail();
                }
                break;
            case METRIC_CATEGORY_GAUGE:
                MetricDump.GaugeDump gaugeDump = (MetricDump.GaugeDump) metric;
                assertEquals("4", gaugeDump.value);
                assertEquals("g1", gaugeDump.name);
                assertTrue(gaugeDump.scopeInfo instanceof QueryScopeInfo.TaskQueryScopeInfo);
                QueryScopeInfo.TaskQueryScopeInfo taskInfo = (QueryScopeInfo.TaskQueryScopeInfo) gaugeDump.scopeInfo;
                assertEquals("D", taskInfo.scope);
                assertEquals("jid", taskInfo.jobID);
                assertEquals("vid", taskInfo.vertexID);
                assertEquals(2, taskInfo.subtaskIndex);
                gauges.remove(g1);
                break;
            case METRIC_CATEGORY_HISTOGRAM:
                MetricDump.HistogramDump histogramDump = (MetricDump.HistogramDump) metric;
                assertEquals("h1", histogramDump.name);
                assertEquals(0.5, histogramDump.median, 0.1);
                assertEquals(0.75, histogramDump.p75, 0.1);
                assertEquals(0.90, histogramDump.p90, 0.1);
                assertEquals(0.95, histogramDump.p95, 0.1);
                assertEquals(0.98, histogramDump.p98, 0.1);
                assertEquals(0.99, histogramDump.p99, 0.1);
                assertEquals(0.999, histogramDump.p999, 0.1);
                assertEquals(4, histogramDump.mean, 0.1);
                assertEquals(5, histogramDump.stddev, 0.1);
                assertEquals(6, histogramDump.max);
                assertEquals(7, histogramDump.min);
                assertTrue(histogramDump.scopeInfo instanceof QueryScopeInfo.OperatorQueryScopeInfo);
                QueryScopeInfo.OperatorQueryScopeInfo opInfo = (QueryScopeInfo.OperatorQueryScopeInfo) histogramDump.scopeInfo;
                assertEquals("E", opInfo.scope);
                assertEquals("jid", opInfo.jobID);
                assertEquals("vid", opInfo.vertexID);
                assertEquals(2, opInfo.subtaskIndex);
                assertEquals("opname", opInfo.operatorName);
                histograms.remove(h1);
                break;
            case METRIC_CATEGORY_METER:
                MetricDump.MeterDump meterDump = (MetricDump.MeterDump) metric;
                assertEquals(5.0, meterDump.rate, 0.1);
                assertTrue(meterDump.scopeInfo instanceof QueryScopeInfo.JobQueryScopeInfo);
                assertEquals("C", meterDump.scopeInfo.scope);
                assertEquals("c3", meterDump.name);
                assertEquals("jid", ((QueryScopeInfo.JobQueryScopeInfo) meterDump.scopeInfo).jobID);
                break;
            default:
                fail();
        }
    }
    assertTrue(counters.isEmpty());
    assertTrue(gauges.isEmpty());
    assertTrue(histograms.isEmpty());
}