Examples with GobblinTrackingEvent org.apache.gobblin.metrics.GobblinTrackingEvent used on opensource projects

Example 11 with GobblinTrackingEvent

use of org.apache.gobblin.metrics.GobblinTrackingEvent in project incubator-gobblin by apache.

the class SLAEventKafkaJobMonitorTest method testFilterByDatasetURN.

@Test
public void testFilterByDatasetURN() throws Exception {
    Properties props = new Properties();
    props.put(SLAEventKafkaJobMonitor.TEMPLATE_KEY, templateURI.toString());
    props.put(SLAEventKafkaJobMonitor.DATASET_URN_FILTER_KEY, "^/accept.*");
    Config config = ConfigFactory.parseProperties(props).withFallback(superConfig);
    SLAEventKafkaJobMonitor monitor = new SLAEventKafkaJobMonitor("topic", null, new URI("/base/URI"), HighLevelConsumerTest.getSimpleConfig(Optional.of(KafkaJobMonitor.KAFKA_JOB_MONITOR_PREFIX)), new NoopSchemaVersionWriter(), Optional.of(Pattern.compile("^/accept.*")), Optional.<Pattern>absent(), this.templateURI, ImmutableMap.<String, String>of());
    monitor.buildMetricsContextAndMetrics();
    GobblinTrackingEvent event;
    Collection<Either<JobSpec, URI>> jobSpecs;
    event = createSLAEvent("event", new URI("/accept/myDataset"), Maps.<String, String>newHashMap());
    jobSpecs = monitor.parseJobSpec(event);
    Assert.assertEquals(jobSpecs.size(), 1);
    Assert.assertEquals(monitor.getRejectedEvents().getCount(), 0);
    event = createSLAEvent("event", new URI("/reject/myDataset"), Maps.<String, String>newHashMap());
    jobSpecs = monitor.parseJobSpec(event);
    Assert.assertEquals(jobSpecs.size(), 0);
    Assert.assertEquals(monitor.getRejectedEvents().getCount(), 1);
    monitor.shutdownMetrics();
}

Example 12 with GobblinTrackingEvent

use of org.apache.gobblin.metrics.GobblinTrackingEvent in project incubator-gobblin by apache.

the class CopySource method submitUnfulfilledRequestEventsHelper.

private void submitUnfulfilledRequestEventsHelper(List<FileSet<CopyEntity>> fileSetList, String eventName) {
    for (FileSet<CopyEntity> fileSet : fileSetList) {
        GobblinTrackingEvent event = GobblinTrackingEvent.newBuilder().setName(eventName).setNamespace(CopySource.class.getName()).setMetadata(ImmutableMap.<String, String>builder().put(ConfigurationKeys.DATASET_URN_KEY, fileSet.getDataset().getUrn()).put(FILESET_TOTAL_ENTITIES, Integer.toString(fileSet.getTotalEntities())).put(FILESET_TOTAL_SIZE_IN_BYTES, Long.toString(fileSet.getTotalSizeInBytes())).put(FILESET_NAME, fileSet.getName()).build()).build();
        this.metricContext.submitEvent(event);
    }
}

Example 13 with GobblinTrackingEvent

use of org.apache.gobblin.metrics.GobblinTrackingEvent in project incubator-gobblin by apache.

the class FileFailureEventReporterTest method testReport.

@Test
public void testReport() throws IOException {
    MetricContext testContext = MetricContext.builder(getClass().getCanonicalName()).build();
    FileSystem fs = mock(FileSystem.class);
    Path failureLogPath = mock(Path.class);
    FSDataOutputStream outputStream = mock(FSDataOutputStream.class);
    FileFailureEventReporter reporter = new FileFailureEventReporter(testContext, fs, failureLogPath);
    when(fs.exists(any())).thenReturn(true);
    when(fs.append(any())).thenReturn(outputStream);
    final String eventName = "testEvent";
    final String eventNamespace = "testNamespace";
    GobblinTrackingEvent event = new GobblinTrackingEvent(0L, eventNamespace, eventName, Maps.newHashMap());
    // Noop on normal event
    testContext.submitEvent(event);
    verify(fs, never()).append(failureLogPath);
    verify(outputStream, never()).write(anyByte());
    // Process failure event
    FailureEventBuilder failureEvent = new FailureEventBuilder(eventName, eventNamespace);
    failureEvent.submit(testContext);
    reporter.report();
    // Failure log output is setup
    verify(fs, times(1)).append(failureLogPath);
    // Report successfully
    doAnswer(invocation -> null).when(outputStream).write(any(byte[].class), anyInt(), anyInt());
    verify(outputStream, times(1)).write(any(byte[].class), anyInt(), anyInt());
}

Example 14 with GobblinTrackingEvent

use of org.apache.gobblin.metrics.GobblinTrackingEvent in project incubator-gobblin by apache.

the class OutputStreamEventReporter method reportEventQueue.

@Override
public void reportEventQueue(Queue<GobblinTrackingEvent> queue) {
    if (queue.size() <= 0) {
        return;
    }
    this.outputBuffer.reset();
    GobblinTrackingEvent nextEvent;
    final String dateTime = dateFormat.format(new Date());
    printWithBanner(dateTime, '=');
    this.outputBufferPrintStream.println();
    printWithBanner("-- Events", '-');
    while (null != (nextEvent = queue.poll())) {
        this.outputBufferPrintStream.println(new String(this.serializer.serializeRecord(nextEvent), Charsets.UTF_8));
    }
    this.outputBufferPrintStream.println();
    try {
        this.outputBuffer.writeTo(this.output);
    } catch (IOException exception) {
        LOGGER.warn("Failed to write events to output stream.");
    }
}

Example 15 with GobblinTrackingEvent

use of org.apache.gobblin.metrics.GobblinTrackingEvent in project incubator-gobblin by apache.

the class EventSubmitter method submit.

/**
 * Submits the {@link org.apache.gobblin.metrics.GobblinTrackingEvent} to the {@link org.apache.gobblin.metrics.MetricContext}.
 * @param name Name of the event.
 * @param additionalMetadata Additional metadata to be added to the event.
 */
public void submit(String name, Map<String, String> additionalMetadata) {
    if (this.metricContext.isPresent()) {
        Map<String, String> finalMetadata = Maps.newHashMap(this.metadata);
        if (!additionalMetadata.isEmpty()) {
            finalMetadata.putAll(additionalMetadata);
        }
        // Timestamp is set by metric context.
        this.metricContext.get().submitEvent(new GobblinTrackingEvent(0l, this.namespace, name, finalMetadata));
    }
}