Examples with GobblinEventBuilder org.apache.gobblin.metrics.event.GobblinEventBuilder used on opensource projects

Example 1 with GobblinEventBuilder

use of org.apache.gobblin.metrics.event.GobblinEventBuilder in project incubator-gobblin by apache.

the class IcebergMetadataWriter method submitSnapshotCommitEvent.

private void submitSnapshotCommitEvent(Snapshot snapshot, TableMetadata tableMetadata, String dbName, String tableName, Map<String, String> props, Long highWaterMark) {
    GobblinEventBuilder gobblinTrackingEvent = new GobblinEventBuilder(IcebergMCEMetadataKeys.ICEBERG_COMMIT_EVENT_NAME);
    long currentSnapshotID = snapshot.snapshotId();
    long endToEndLag = System.currentTimeMillis() - tableMetadata.lowestGMCEEmittedTime;
    TableIdentifier tid = TableIdentifier.of(dbName, tableName);
    String gmceTopicPartition = tableTopicPartitionMap.get(tid);
    // Add information to automatically trigger repair jon when data loss happen
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.GMCE_TOPIC_NAME, gmceTopicPartition.split("-")[0]);
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.GMCE_TOPIC_PARTITION, gmceTopicPartition.split("-")[1]);
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.GMCE_HIGH_WATERMARK, highWaterMark.toString());
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.GMCE_LOW_WATERMARK, tableMetadata.lowWatermark.get().toString());
    // Add information for lag monitoring
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.LAG_KEY_NAME, Long.toString(endToEndLag));
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.SNAPSHOT_KEY_NAME, Long.toString(currentSnapshotID));
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.MANIFEST_LOCATION, snapshot.manifestListLocation());
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.SNAPSHOT_INFORMATION_KEY_NAME, Joiner.on(",").withKeyValueSeparator("=").join(snapshot.summary()));
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.TABLE_KEY_NAME, tableName);
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.DATABASE_KEY_NAME, dbName);
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.DATASET_HDFS_PATH, tableMetadata.datasetName);
    for (Map.Entry<String, String> entry : props.entrySet()) {
        if (entry.getKey().startsWith(OFFSET_RANGE_KEY_PREFIX)) {
            gobblinTrackingEvent.addMetadata(entry.getKey(), entry.getValue());
        }
    }
    eventSubmitter.submit(gobblinTrackingEvent);
}

Example 2 with GobblinEventBuilder

use of org.apache.gobblin.metrics.event.GobblinEventBuilder in project incubator-gobblin by apache.

the class GobblinTaskRunner method submitEvent.

private void submitEvent(ContainerHealthCheckFailureEvent event) {
    EventSubmitter eventSubmitter = new EventSubmitter.Builder(RootMetricContext.get(), getClass().getPackage().getName()).build();
    GobblinEventBuilder eventBuilder = new GobblinEventBuilder(event.getClass().getSimpleName());
    State taskState = ConfigUtils.configToState(event.getConfig());
    // Add task metadata such as Helix taskId, containerId, and workflowId if configured
    TaskEventMetadataGenerator taskEventMetadataGenerator = TaskEventMetadataUtils.getTaskEventMetadataGenerator(taskState);
    eventBuilder.addAdditionalMetadata(taskEventMetadataGenerator.getMetadata(taskState, event.getClass().getSimpleName()));
    eventBuilder.addAdditionalMetadata(event.getMetadata());
    eventSubmitter.submit(eventBuilder);
}

Example 3 with GobblinEventBuilder

use of org.apache.gobblin.metrics.event.GobblinEventBuilder in project incubator-gobblin by apache.

the class GobblinMCEWriter method submitFailureEvent.

/**
 * Submit event indicating that a specific set of GMCEs have been skipped, so there is a gap in the registration
 */
private void submitFailureEvent(GobblinMetadataException exception) {
    log.warn(String.format("Sending GTE to indicate table flush failure for %s.%s", exception.dbName, exception.tableName));
    GobblinEventBuilder gobblinTrackingEvent = new GobblinEventBuilder(IcebergMCEMetadataKeys.METADATA_WRITER_FAILURE_EVENT);
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.DATASET_HDFS_PATH, exception.datasetPath);
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.FAILURE_EVENT_DB_NAME, exception.dbName);
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.FAILURE_EVENT_TABLE_NAME, exception.tableName);
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.GMCE_TOPIC_PARTITION, exception.GMCETopicPartition);
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.GMCE_HIGH_WATERMARK, Long.toString(exception.highWatermark));
    gobblinTrackingEvent.addMetadata(IcebergMCEMetadataKeys.GMCE_LOW_WATERMARK, Long.toString(exception.lowWatermark));
    eventSubmitter.submit(gobblinTrackingEvent);
}