Examples with ProvenanceEventRecordDTO com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO used on opensource projects

Example 11 with ProvenanceEventRecordDTO

use of com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO in project kylo by Teradata.

the class ProvenanceEventDtoBuilder method build.

/**
 * Build the event record
 * @return the event record
 */
public ProvenanceEventRecordDTO build() {
    if (this.eventId == null) {
        this.eventId = LongIdGenerator.nextId();
    }
    if (this.eventType == null) {
        this.eventType = EventType.DROP;
    }
    if (StringUtils.isBlank(componentType)) {
        componentType = "CustomComponent";
    }
    Long now = DateTime.now().getMillis();
    if (startTime == null) {
        startTime = now;
    }
    if (eventTime == null) {
        eventTime = now;
    }
    if (jobFlowFileId == null) {
        jobFlowFileId = flowFileId;
    }
    ProvenanceEventRecordDTO event = new ProvenanceEventRecordDTO();
    event.setFeedName(feedName);
    event.setFirstEventProcessorId(firstEventProcessorId);
    event.setJobFlowFileId(jobFlowFileId);
    event.setFlowFileUuid(flowFileId);
    event.setComponentName(componentName);
    event.setStartTime(startTime);
    event.setEventTime(eventTime);
    event.setStream(stream);
    event.setIsFinalJobEvent(endingEvent);
    event.setComponentId(UUID.randomUUID().toString());
    event.setEventId(eventId);
    event.setIsStartOfJob(startingEvent);
    event.setEventType(eventType.name());
    if (startingEvent) {
        event.setEventType("CREATE");
    }
    event.setComponentType(componentType);
    Map<String, String> attrs = new HashMap<>();
    if (updatedAttributes == null) {
        updatedAttributes = attrs;
    }
    if (previousAttributes == null) {
        previousAttributes = new HashMap<>();
    }
    event.setUpdatedAttributes(updatedAttributes);
    event.setPreviousAttributes(previousAttributes);
    Map<String, String> attributeMap = updatedAttributes;
    if (!previousAttributes.isEmpty()) {
        attributeMap = new HashMap<>();
        attributeMap.putAll(previousAttributes);
        attributeMap.putAll(updatedAttributes);
    }
    event.setAttributeMap(attributeMap);
    return event;
}

Example 12 with ProvenanceEventRecordDTO

use of com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO in project kylo by Teradata.

the class JpaBatchJobExecutionProvider method getOrCreateStreamJobExecution.

private JpaBatchJobExecution getOrCreateStreamJobExecution(ProvenanceEventRecordDTO event, OpsManagerFeed feed) {
    JpaBatchJobExecution jobExecution = null;
    boolean isNew = false;
    try {
        BatchJobExecution latestJobExecution = latestStreamingJobByFeedName.get(event.getFeedName());
        if (latestJobExecution == null) {
            latestJobExecution = findLatestJobForFeed(event.getFeedName());
        } else {
            if (clusterService.isClustered()) {
                latestJobExecution = jobExecutionRepository.findOne(latestJobExecution.getJobExecutionId());
            }
        }
        if (latestJobExecution == null || (latestJobExecution != null && !latestJobExecution.isStream())) {
            // If the latest Job is not set to be a Stream and its still running we need to fail it and create the new streaming job.
            if (latestJobExecution != null && !latestJobExecution.isFinished()) {
                ProvenanceEventRecordDTO tempFailedEvent = new ProvenanceEventRecordDTO();
                tempFailedEvent.setFeedName(event.getFeedName());
                tempFailedEvent.setAttributeMap(new HashMap<>());
                tempFailedEvent.setIsFailure(true);
                tempFailedEvent.setDetails("Failed Running Batch event as this Feed has now become a Stream");
                finishJob(tempFailedEvent, (JpaBatchJobExecution) latestJobExecution);
                latestJobExecution.setExitMessage("Failed Running Batch event as this Feed has now become a Stream");
                save(latestJobExecution);
            }
            jobExecution = createNewJobExecution(event, feed);
            jobExecution.setStream(true);
            latestStreamingJobByFeedName.put(event.getFeedName(), jobExecution);
            log.info("Created new Streaming Job Execution with id of {} and starting event {} ", jobExecution.getJobExecutionId(), event);
        } else {
            jobExecution = (JpaBatchJobExecution) latestJobExecution;
        }
        if (jobExecution != null) {
            latestStreamingJobByFeedName.put(event.getFeedName(), jobExecution);
        }
    } catch (OptimisticLockException e) {
        // read
        jobExecution = (JpaBatchJobExecution) findLatestJobForFeed(event.getFeedName());
    }
    boolean save = isNew;
    if (!jobExecution.isStream()) {
        jobExecution.setStream(true);
        save = true;
    }
    if (save) {
        save(jobExecution);
    }
    return jobExecution;
}

Example 13 with ProvenanceEventRecordDTO

use of com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO in project kylo by Teradata.

the class SparkProvenance method run.

private void run(@Nonnull final PrintStream out, @Nonnull final String... args) {
    // Check how many arguments were passed in
    if (args.length < 5) {
        String msg = "Proper Usage is: <flowfile-id> <job-flowfile-id> <feed-name (category.feed)> <connection-url (url to connect to JMS or KAFAK)> <type (JMS, KAFKA)>" + "You provided " + args.length + " args which are (comma separated): " + StringUtils.join(args, ",");
        out.println(msg);
        throw new IllegalArgumentException(msg);
    }
    ProvenanceEventService provenanceEventService = null;
    final SparkContext sparkContext = SparkContext.getOrCreate();
    try {
        final SparkProvenanceConfiguration params = new SparkProvenanceConfiguration(args);
        // Get the proper ProvenanceService
        provenanceEventService = ProvenanceServiceFactory.getProvenanceEventService(params);
        // Collection of custom Provenance Events we will be sending to Kylo
        List<ProvenanceEventRecordDTO> events = new ArrayList<>();
        // do some work.  Look up the database names in Hive
        final HiveContext hiveContext = new HiveContext(sparkContext);
        // Do some work... i.e. look up the Databases in Hive
        ProvenanceEventRecordDTO event = newEvent("Databases", params);
        Dataset df = hiveContext.sql("show databases");
        event.getAttributeMap().put("databases", df.toJSON().collectAsList().toString());
        event.setEventTime(System.currentTimeMillis());
        events.add(event);
        event = newEvent("Another Step", params);
        event.getAttributeMap().put("UUID 1", UUID.randomUUID().toString());
        event.setEventTime(System.currentTimeMillis());
        event.getAttributeMap().put("timestamp", String.valueOf(System.currentTimeMillis()));
        events.add(event);
        // Send the events off
        provenanceEventService.sendEvents(events);
        log.info("Spark app finished");
    } catch (Exception e) {
        log.error("Failed to run Spark Provenance Job: {}", e.toString(), e);
    } finally {
        provenanceEventService.closeConnection();
        sparkContext.stop();
        log.info("Exiting!!!!");
        System.exit(0);
    }
}