Examples with GobblinTrackingEvent org.apache.gobblin.metrics.GobblinTrackingEvent used on opensource projects

Example 6 with GobblinTrackingEvent

use of org.apache.gobblin.metrics.GobblinTrackingEvent in project incubator-gobblin by apache.

the class KafkaAvroJobMonitorTest method testUsingSchemaVersion.

@Test
public void testUsingSchemaVersion() throws Exception {
    TestKafkaAvroJobMonitor monitor = new TestKafkaAvroJobMonitor(GobblinTrackingEvent.SCHEMA$, new FixedSchemaVersionWriter());
    monitor.buildMetricsContextAndMetrics();
    AvroSerializer<GobblinTrackingEvent> serializer = new AvroBinarySerializer<>(GobblinTrackingEvent.SCHEMA$, new FixedSchemaVersionWriter());
    GobblinTrackingEvent event = new GobblinTrackingEvent(0L, "namespace", "event", Maps.<String, String>newHashMap());
    Collection<Either<JobSpec, URI>> results = monitor.parseJobSpec(serializer.serializeRecord(event));
    Assert.assertEquals(results.size(), 1);
    Assert.assertEquals(monitor.events.size(), 1);
    Assert.assertEquals(monitor.events.get(0), event);
    monitor.shutdownMetrics();
}

Example 7 with GobblinTrackingEvent

use of org.apache.gobblin.metrics.GobblinTrackingEvent in project incubator-gobblin by apache.

the class SLAEventKafkaJobMonitorTest method testParseJobSpec.

@Test
public void testParseJobSpec() throws Exception {
    SLAEventKafkaJobMonitor monitor = new SLAEventKafkaJobMonitor("topic", null, new URI("/base/URI"), HighLevelConsumerTest.getSimpleConfig(Optional.of(KafkaJobMonitor.KAFKA_JOB_MONITOR_PREFIX)), new NoopSchemaVersionWriter(), Optional.<Pattern>absent(), Optional.<Pattern>absent(), this.templateURI, ImmutableMap.of("metadataKey1", "key1"));
    monitor.buildMetricsContextAndMetrics();
    GobblinTrackingEvent event = createSLAEvent("DatasetPublish", new URI("/data/myDataset"), ImmutableMap.of("metadataKey1", "value1", "key1", "value2"));
    Collection<Either<JobSpec, URI>> jobSpecs = monitor.parseJobSpec(event);
    Assert.assertEquals(jobSpecs.size(), 1);
    JobSpec jobSpec = (JobSpec) jobSpecs.iterator().next().get();
    Assert.assertEquals(jobSpec.getUri(), new URI("/base/URI/data/myDataset"));
    Assert.assertEquals(jobSpec.getTemplateURI().get(), templateURI);
    // should insert configuration from metadata
    Assert.assertEquals(jobSpec.getConfig().getString("key1"), "value1");
    monitor.shutdownMetrics();
}

Example 8 with GobblinTrackingEvent

use of org.apache.gobblin.metrics.GobblinTrackingEvent in project incubator-gobblin by apache.

the class SLAEventKafkaJobMonitorTest method testFilterByName.

@Test
public void testFilterByName() throws Exception {
    SLAEventKafkaJobMonitor monitor = new SLAEventKafkaJobMonitor("topic", null, new URI("/base/URI"), HighLevelConsumerTest.getSimpleConfig(Optional.of(KafkaJobMonitor.KAFKA_JOB_MONITOR_PREFIX)), new NoopSchemaVersionWriter(), Optional.<Pattern>absent(), Optional.of(Pattern.compile("^accept.*")), this.templateURI, ImmutableMap.<String, String>of());
    monitor.buildMetricsContextAndMetrics();
    GobblinTrackingEvent event;
    Collection<Either<JobSpec, URI>> jobSpecs;
    event = createSLAEvent("acceptthis", new URI("/data/myDataset"), Maps.<String, String>newHashMap());
    jobSpecs = monitor.parseJobSpec(event);
    Assert.assertEquals(jobSpecs.size(), 1);
    Assert.assertEquals(monitor.getRejectedEvents().getCount(), 0);
    event = createSLAEvent("donotacceptthis", new URI("/data/myDataset"), Maps.<String, String>newHashMap());
    jobSpecs = monitor.parseJobSpec(event);
    Assert.assertEquals(jobSpecs.size(), 0);
    Assert.assertEquals(monitor.getRejectedEvents().getCount(), 1);
    monitor.shutdownMetrics();
}

Example 9 with GobblinTrackingEvent

use of org.apache.gobblin.metrics.GobblinTrackingEvent in project incubator-gobblin by apache.

the class KafkaEventReporterTest method testKafkaEventReporter.

@Test
public void testKafkaEventReporter() throws IOException {
    MetricContext context = MetricContext.builder("context").build();
    MockKafkaPusher pusher = new MockKafkaPusher();
    KafkaEventReporter kafkaReporter = getBuilder(context, pusher).build("localhost:0000", "topic");
    String namespace = "gobblin.metrics.test";
    String eventName = "testEvent";
    GobblinTrackingEvent event = new GobblinTrackingEvent();
    event.setName(eventName);
    event.setNamespace(namespace);
    Map<String, String> metadata = Maps.newHashMap();
    metadata.put("m1", "v1");
    metadata.put("m2", null);
    event.setMetadata(metadata);
    context.submitEvent(event);
    try {
        Thread.sleep(100);
    } catch (InterruptedException ex) {
        Thread.currentThread().interrupt();
    }
    kafkaReporter.report();
    try {
        Thread.sleep(100);
    } catch (InterruptedException ex) {
        Thread.currentThread().interrupt();
    }
    GobblinTrackingEvent retrievedEvent = nextEvent(pusher.messageIterator());
    Assert.assertEquals(retrievedEvent.getNamespace(), namespace);
    Assert.assertEquals(retrievedEvent.getName(), eventName);
    Assert.assertEquals(retrievedEvent.getMetadata().size(), 4);
}

Example 10 with GobblinTrackingEvent

use of org.apache.gobblin.metrics.GobblinTrackingEvent in project incubator-gobblin by apache.

the class KafkaAvroJobMonitorTest method testSimple.

@Test
public void testSimple() throws Exception {
    TestKafkaAvroJobMonitor monitor = new TestKafkaAvroJobMonitor(GobblinTrackingEvent.SCHEMA$, new NoopSchemaVersionWriter());
    monitor.buildMetricsContextAndMetrics();
    AvroSerializer<GobblinTrackingEvent> serializer = new AvroBinarySerializer<>(GobblinTrackingEvent.SCHEMA$, new NoopSchemaVersionWriter());
    GobblinTrackingEvent event = new GobblinTrackingEvent(0L, "namespace", "event", Maps.<String, String>newHashMap());
    Collection<Either<JobSpec, URI>> results = monitor.parseJobSpec(serializer.serializeRecord(event));
    Assert.assertEquals(results.size(), 1);
    Assert.assertEquals(monitor.events.size(), 1);
    Assert.assertEquals(monitor.events.get(0), event);
    monitor.shutdownMetrics();
}