Examples with MetricContext org.apache.gobblin.metrics.MetricContext used on opensource projects

Example 21 with MetricContext

use of org.apache.gobblin.metrics.MetricContext in project incubator-gobblin by apache.

the class ScheduledReporterTest method testScheduledReporter.

@Test
public void testScheduledReporter() throws Exception {
    long reportingIntervalMillis = 1000;
    String context1Name = ScheduledReporterTest.class.getSimpleName() + "_1";
    String context2Name = ScheduledReporterTest.class.getSimpleName() + "_2";
    String context3Name = "SomeOtherName";
    // Create a context name (to check that initialized reporter gets existing contexts correctly)
    MetricContext context1 = MetricContext.builder(context1Name).build();
    // Set up config for reporter
    Properties props = new Properties();
    ScheduledReporter.setReportingInterval(props, reportingIntervalMillis, TimeUnit.MILLISECONDS);
    Config config = ConfigUtils.propertiesToConfig(props);
    config = PrefixContextFilter.setPrefixString(config, ScheduledReporterTest.class.getSimpleName());
    config = ContextFilterFactory.setContextFilterClass(config, PrefixContextFilter.class);
    // Create reporter
    ContextStoreReporter reporter = new ContextStoreReporter("testContext", config);
    // Check that reporter correctly found created context
    Set<String> contextNames = getContextNames(reporter);
    Assert.assertEquals(contextNames.size(), 1);
    Assert.assertTrue(contextNames.contains(context1Name));
    // Create two more contexts
    MetricContext context2 = context1.childBuilder(context2Name).build();
    context1.childBuilder(context3Name).build();
    // Check that reporter correctly found new reporter, but skipped the one that does not satisfy filter
    contextNames = getContextNames(reporter);
    Assert.assertEquals(contextNames.size(), 2);
    Assert.assertTrue(contextNames.contains(context1Name));
    Assert.assertTrue(contextNames.contains(context2Name));
    // Check that nothing has been reported
    Assert.assertEquals(reporter.getReportedContexts().size(), 0);
    // Start reporter
    reporter.start();
    // Wait for up to 10 reporting intervals for 3 reports to run
    long maxWaitMillis = 10 * reportingIntervalMillis;
    long totalWait = 0;
    while (reporter.getReportedContexts().size() < 6 && maxWaitMillis > 0) {
        long wait = 100;
        Thread.sleep(wait);
        maxWaitMillis -= wait;
        totalWait += wait;
    }
    // stop reporter
    reporter.stop();
    // Check wait makes sense given reporting interval (e.g. if wait = 100 millis, then 2 reports in 100 millis,
    // something is wrong with schedule).
    Assert.assertTrue(totalWait > reportingIntervalMillis);
    Assert.assertTrue(reporter.getReportedContexts().size() >= 6);
    // Check that it didn't report excessively
    Assert.assertTrue(reporter.getReportedContexts().size() <= 10);
    // Check that first report indeed reported the correct contexts
    Set<String> firstReport = Sets.newHashSet(reporter.getReportedContexts().get(0).getName(), reporter.getReportedContexts().get(1).getName());
    Assert.assertEquals(firstReport, Sets.newHashSet(context1Name, context2Name));
    // Check that second report indeed reported the correct contexts
    Set<String> secondReport = Sets.newHashSet(reporter.getReportedContexts().get(2).getName(), reporter.getReportedContexts().get(3).getName());
    Assert.assertEquals(secondReport, Sets.newHashSet(context1Name, context2Name));
    int totalReports = reporter.getReportedContexts().size();
    // Wait for reporting interval to make sure reporting has actually stopped
    Thread.sleep(2 * reportingIntervalMillis);
    Assert.assertEquals(reporter.getReportedContexts().size(), totalReports);
    reporter.getReportedContexts().clear();
    // Dereference context 2 to ensure that it gets reported
    context2 = null;
    // Wait for context to be GCed
    maxWaitMillis = 2000;
    System.gc();
    while (reporter.getReportedContexts().size() < 1 && maxWaitMillis > 0) {
        System.gc();
        long wait = 100;
        Thread.sleep(wait);
        maxWaitMillis -= wait;
    }
    // Check that GCed context was reported
    Assert.assertEquals(reporter.getReportedContexts().size(), 1);
    Assert.assertEquals(reporter.getReportedContexts().get(0).getName(), context2Name);
    // Test close method
    reporter.close();
}

Example 22 with MetricContext

use of org.apache.gobblin.metrics.MetricContext in project incubator-gobblin by apache.

the class FileFailureEventReporterTest method testReport.

@Test
public void testReport() throws IOException {
    MetricContext testContext = MetricContext.builder(getClass().getCanonicalName()).build();
    FileSystem fs = mock(FileSystem.class);
    Path failureLogPath = mock(Path.class);
    FSDataOutputStream outputStream = mock(FSDataOutputStream.class);
    FileFailureEventReporter reporter = new FileFailureEventReporter(testContext, fs, failureLogPath);
    when(fs.exists(any())).thenReturn(true);
    when(fs.append(any())).thenReturn(outputStream);
    final String eventName = "testEvent";
    final String eventNamespace = "testNamespace";
    GobblinTrackingEvent event = new GobblinTrackingEvent(0L, eventNamespace, eventName, Maps.newHashMap());
    // Noop on normal event
    testContext.submitEvent(event);
    verify(fs, never()).append(failureLogPath);
    verify(outputStream, never()).write(anyByte());
    // Process failure event
    FailureEventBuilder failureEvent = new FailureEventBuilder(eventName, eventNamespace);
    failureEvent.submit(testContext);
    reporter.report();
    // Failure log output is setup
    verify(fs, times(1)).append(failureLogPath);
    // Report successfully
    doAnswer(invocation -> null).when(outputStream).write(any(byte[].class), anyInt(), anyInt());
    verify(outputStream, times(1)).write(any(byte[].class), anyInt(), anyInt());
}

Example 23 with MetricContext

use of org.apache.gobblin.metrics.MetricContext in project incubator-gobblin by apache.

the class LimiterServerResourceTest method testMetrics.

@Test
public void testMetrics() throws Exception {
    ThrottlingGuiceServletConfig guiceServletConfig = new ThrottlingGuiceServletConfig();
    guiceServletConfig.initialize(ConfigFactory.empty());
    Injector injector = guiceServletConfig.getInjector();
    LimiterServerResource limiterServer = injector.getInstance(LimiterServerResource.class);
    PermitRequest request = new PermitRequest();
    request.setPermits(10);
    request.setResource("myResource");
    limiterServer.getSync(new ComplexResourceKey<>(request, new EmptyRecord()));
    limiterServer.getSync(new ComplexResourceKey<>(request, new EmptyRecord()));
    limiterServer.getSync(new ComplexResourceKey<>(request, new EmptyRecord()));
    MetricContext metricContext = limiterServer.metricContext;
    Timer timer = metricContext.timer(LimiterServerResource.REQUEST_TIMER_NAME);
    Assert.assertEquals(timer.getCount(), 3);
}

Example 24 with MetricContext

use of org.apache.gobblin.metrics.MetricContext in project incubator-gobblin by apache.

the class Task method submitTaskCommittedEvent.

protected void submitTaskCommittedEvent() {
    MetricContext taskMetricContext = TaskMetrics.get(this.taskState).getMetricContext();
    EventSubmitter eventSubmitter = new EventSubmitter.Builder(taskMetricContext, "gobblin.runtime.task").build();
    eventSubmitter.submit(TaskEvent.TASK_COMMITTED_EVENT_NAME, ImmutableMap.of(TaskEvent.METADATA_TASK_ID, this.taskId, TaskEvent.METADATA_TASK_ATTEMPT_ID, this.taskState.getTaskAttemptId().or("")));
}

Example 25 with MetricContext

use of org.apache.gobblin.metrics.MetricContext in project incubator-gobblin by apache.

the class InfluxDBReporterTest method testWithTags.

@Test
public void testWithTags() throws IOException {
    try (MetricContext metricContext = MetricContext.builder(this.getClass().getCanonicalName() + ".testGraphiteReporter").addTag(new Tag<String>("taskId", "task_testjob_123")).addTag(new Tag<String>("forkBranchName", "fork_1")).build();
        InfluxDBReporter influxDBReporter = InfluxDBReporter.Factory.newBuilder().withInfluxDBPusher(influxDBPusher).withMetricContextName(CONTEXT_NAME).build(new Properties())) {
        Counter counter = metricContext.counter(MetricRegistry.name(METRIC_PREFIX, COUNTER));
        counter.inc(5l);
        influxDBReporter.report(new TreeMap<String, Gauge>(), metricContext.getCounters(), new TreeMap<String, Histogram>(), new TreeMap<String, Meter>(), new TreeMap<String, Timer>(), metricContext.getTagMap());
        // InfluxDB converts all values to float64 internally
        Assert.assertEquals(getMetricValue("task_testjob_123.fork_1." + METRIC_PREFIX, COUNTER, Measurements.COUNT), Float.toString(5f));
    }
}