Examples with MetricReport org.apache.gobblin.metrics.MetricReport used on opensource projects

Example 1 with MetricReport

use of org.apache.gobblin.metrics.MetricReport in project incubator-gobblin by apache.

the class MetricReportReporter method report.

/**
 * Serializes metrics and pushes the byte arrays to Kafka. Uses the serialize* methods in {@link MetricReportReporter}.
 *
 * @param gauges map of {@link com.codahale.metrics.Gauge} to report and their name.
 * @param counters map of {@link com.codahale.metrics.Counter} to report and their name.
 * @param histograms map of {@link com.codahale.metrics.Histogram} to report and their name.
 * @param meters map of {@link com.codahale.metrics.Meter} to report and their name.
 * @param timers map of {@link com.codahale.metrics.Timer} to report and their name.
 */
@Override
protected void report(SortedMap<String, Gauge> gauges, SortedMap<String, Counter> counters, SortedMap<String, Histogram> histograms, SortedMap<String, Meter> meters, SortedMap<String, Timer> timers, Map<String, Object> tags) {
    List<Metric> metrics = Lists.newArrayList();
    for (Map.Entry<String, Gauge> gauge : gauges.entrySet()) {
        metrics.addAll(serializeGauge(gauge.getKey(), gauge.getValue()));
    }
    for (Map.Entry<String, Counter> counter : counters.entrySet()) {
        metrics.addAll(serializeCounter(counter.getKey(), counter.getValue()));
    }
    for (Map.Entry<String, Histogram> histogram : histograms.entrySet()) {
        metrics.addAll(serializeSnapshot(histogram.getKey(), histogram.getValue().getSnapshot()));
        metrics.addAll(serializeCounter(histogram.getKey(), histogram.getValue()));
    }
    for (Map.Entry<String, Meter> meter : meters.entrySet()) {
        metrics.addAll(serializeMetered(meter.getKey(), meter.getValue()));
    }
    for (Map.Entry<String, Timer> timer : timers.entrySet()) {
        metrics.addAll(serializeSnapshot(timer.getKey(), timer.getValue().getSnapshot()));
        metrics.addAll(serializeMetered(timer.getKey(), timer.getValue()));
        metrics.addAll(serializeSingleValue(timer.getKey(), timer.getValue().getCount(), Measurements.COUNT.getName()));
    }
    Map<String, Object> allTags = Maps.newHashMap();
    allTags.putAll(tags);
    allTags.putAll(this.tags);
    Map<String, String> allTagsString = Maps.transformValues(allTags, new Function<Object, String>() {

        @Nullable
        @Override
        public String apply(Object input) {
            return input.toString();
        }
    });
    MetricReport report = new MetricReport(allTagsString, System.currentTimeMillis(), metrics);
    emitReport(report);
}

Example 2 with MetricReport

use of org.apache.gobblin.metrics.MetricReport in project incubator-gobblin by apache.

the class GobblinMetricsPinotFlattenerConverterTest method test.

@Test
public void test() throws Exception {
    MetricReport metricReport = new MetricReport();
    metricReport.setTags(ImmutableMap.of("tag", "value", "tag2", "value2"));
    metricReport.setTimestamp(10L);
    metricReport.setMetrics(Lists.newArrayList(new Metric("metric", 1.0), new Metric("metric2", 2.0)));
    AvroSerializer<MetricReport> serializer = new AvroBinarySerializer<>(MetricReport.SCHEMA$, new NoopSchemaVersionWriter());
    serializer.serializeRecord(metricReport);
    Schema metricReportUtf8 = new Schema.Parser().parse(this.getClass().getClassLoader().getResourceAsStream("MetricReport.avsc"));
    GenericRecord genericRecordMetric = AvroUtils.slowDeserializeGenericRecord(serializer.serializeRecord(metricReport), metricReportUtf8);
    GobblinMetricsPinotFlattenerConverter converter = new GobblinMetricsPinotFlattenerConverter();
    Schema outputSchema = converter.convertSchema(MetricReport.SCHEMA$, new WorkUnitState());
    Iterable<GenericRecord> converted = converter.convertRecord(outputSchema, genericRecordMetric, new WorkUnitState());
    List<GenericRecord> convertedList = Lists.newArrayList(converted);
    Assert.assertEquals(convertedList.size(), 2);
    Assert.assertEquals(Sets.newHashSet((List<Utf8>) convertedList.get(0).get("tags")), Sets.newHashSet("tag:value", "tag2:value2"));
    Assert.assertEquals(convertedList.get(0).get("timestamp"), 10L);
    Assert.assertEquals(convertedList.get(0).get("metricName").toString(), "metric");
    Assert.assertEquals(convertedList.get(0).get("metricValue"), 1.0);
    Assert.assertEquals(Sets.newHashSet((List<Utf8>) convertedList.get(1).get("tags")), Sets.newHashSet("tag:value", "tag2:value2"));
    Assert.assertEquals(convertedList.get(1).get("timestamp"), 10L);
    Assert.assertEquals(convertedList.get(1).get("metricName").toString(), "metric2");
    Assert.assertEquals(convertedList.get(1).get("metricValue"), 2.0);
}

Example 3 with MetricReport

use of org.apache.gobblin.metrics.MetricReport in project incubator-gobblin by apache.

the class KafkaReporterTest method kafkaReporterTagsTest.

@Test
public void kafkaReporterTagsTest() throws IOException {
    MetricContext metricContext = MetricContext.builder(this.getClass().getCanonicalName() + ".kafkaReporterTagsTest").build();
    Counter counter = metricContext.counter("com.linkedin.example.counter");
    Tag<?> tag1 = new Tag<>("tag1", "value1");
    Tag<?> tag2 = new Tag<>("tag2", 2);
    MockKafkaPusher pusher = new MockKafkaPusher();
    KafkaReporter kafkaReporter = getBuilder(pusher).withTags(Lists.newArrayList(tag1, tag2)).build("localhost:0000", "topic", new Properties());
    counter.inc();
    kafkaReporter.report(metricContext);
    try {
        Thread.sleep(1000);
    } catch (InterruptedException ex) {
        Thread.currentThread().interrupt();
    }
    MetricReport metricReport = nextReport(pusher.messageIterator());
    Assert.assertEquals(4, metricReport.getTags().size());
    Assert.assertTrue(metricReport.getTags().containsKey(tag1.getKey()));
    Assert.assertEquals(metricReport.getTags().get(tag1.getKey()), tag1.getValue().toString());
    Assert.assertTrue(metricReport.getTags().containsKey(tag2.getKey()));
    Assert.assertEquals(metricReport.getTags().get(tag2.getKey()), tag2.getValue().toString());
}

Example 4 with MetricReport

use of org.apache.gobblin.metrics.MetricReport in project incubator-gobblin by apache.

the class KafkaReporterTest method kafkaReporterContextTest.

@Test
public void kafkaReporterContextTest() throws IOException {
    Tag<?> tag1 = new Tag<>("tag1", "value1");
    MetricContext context = MetricContext.builder("context").addTag(tag1).build();
    Counter counter = context.counter("com.linkedin.example.counter");
    MockKafkaPusher pusher = new MockKafkaPusher();
    KafkaReporter kafkaReporter = getBuilderFromContext(pusher).build("localhost:0000", "topic", new Properties());
    counter.inc();
    kafkaReporter.report(context);
    try {
        Thread.sleep(1000);
    } catch (InterruptedException ex) {
        Thread.currentThread().interrupt();
    }
    MetricReport metricReport = nextReport(pusher.messageIterator());
    Assert.assertEquals(3, metricReport.getTags().size());
    Assert.assertTrue(metricReport.getTags().containsKey(tag1.getKey()));
    Assert.assertEquals(metricReport.getTags().get(tag1.getKey()), tag1.getValue().toString());
}

Example 5 with MetricReport

use of org.apache.gobblin.metrics.MetricReport in project incubator-gobblin by apache.

the class KafkaAvroJobMonitorTest method testWrongSchema.

@Test
public void testWrongSchema() throws Exception {
    TestKafkaAvroJobMonitor monitor = new TestKafkaAvroJobMonitor(GobblinTrackingEvent.SCHEMA$, new NoopSchemaVersionWriter());
    monitor.buildMetricsContextAndMetrics();
    AvroSerializer<MetricReport> serializer = new AvroBinarySerializer<>(MetricReport.SCHEMA$, new NoopSchemaVersionWriter());
    MetricReport event = new MetricReport(Maps.<String, String>newHashMap(), 0L, Lists.<Metric>newArrayList());
    Collection<Either<JobSpec, URI>> results = monitor.parseJobSpec(serializer.serializeRecord(event));
    Assert.assertEquals(results.size(), 0);
    Assert.assertEquals(monitor.events.size(), 0);
    Assert.assertEquals(monitor.getMessageParseFailures().getCount(), 1);
    monitor.shutdownMetrics();
}