Examples with ProvenanceEventRecordDTO com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO used on opensource projects

Example 1 with ProvenanceEventRecordDTO

use of com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO in project kylo by Teradata.

the class ProvenanceRestController method addProvenance.

@POST
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("add custom provenance events to a job")
@ApiResponses(@ApiResponse(code = 200, message = "add custom provenance events to a job", response = String.class))
public Response addProvenance(ProvenanceEventRecordDTOHolder eventRecordDTOHolder) {
    ProvenanceEventRecordDTOHolder batchEventEntity = new ProvenanceEventRecordDTOHolder();
    List<ProvenanceEventRecordDTO> events = eventRecordDTOHolder.getEvents();
    List<ProvenanceEventRecordDTO> batchEvents = new ArrayList<>();
    for (ProvenanceEventRecordDTO event : events) {
        if (!event.isStream()) {
            batchEvents.add(event);
        }
    }
    // reassign the events
    batchEventEntity.setEvents(batchEvents);
    AggregatedFeedProcessorStatisticsHolder stats = GroupedStatsUtil.gatherStats(events);
    log.info("Processing {} batch  events", batchEventEntity);
    provenanceEventReceiver.receiveEvents(batchEventEntity);
    log.info("Processing {} stats ", stats);
    statsJmsReceiver.receiveTopic(stats);
    return Response.ok(new RestResponseStatus.ResponseStatusBuilder().message("Processed " + eventRecordDTOHolder).buildSuccess()).build();
}

Example 2 with ProvenanceEventRecordDTO

use of com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO in project kylo by Teradata.

the class ProvenanceEventReceiver method queryForNiFiErrorBulletins.

/**
 * Make a REST call to NiFi and query for the NiFi Bulletins that have a flowfile id matching for this job execution and write the bulletin message to the {@link
 * BatchJobExecution#setExitMessage(String)}
 *
 * @param event a provenance event
 */
private void queryForNiFiErrorBulletins(ProvenanceEventRecordDTO event) {
    try {
        metadataAccess.commit(() -> {
            // query for nifi logs
            List<String> relatedFlowFiles = batchJobExecutionProvider.findRelatedFlowFiles(event.getFlowFileUuid());
            if (relatedFlowFiles == null) {
                relatedFlowFiles = new ArrayList<>();
            }
            if (relatedFlowFiles.isEmpty()) {
                relatedFlowFiles.add(event.getFlowFileUuid());
            }
            log.debug("Failed Job {}/{}. Found {} related flow files. ", event.getEventId(), event.getFlowFileUuid(), relatedFlowFiles.size());
            List<BulletinDTO> bulletinDTOS = nifiBulletinExceptionExtractor.getErrorBulletinsForFlowFiles(relatedFlowFiles);
            if (bulletinDTOS != null && !bulletinDTOS.isEmpty()) {
                // write them back to the job
                BatchJobExecution jobExecution = batchJobExecutionProvider.findJobExecution(event);
                if (jobExecution != null) {
                    String msg = jobExecution.getExitMessage() != null ? jobExecution.getExitMessage() + "\n" : "";
                    msg += "NiFi exceptions: \n" + bulletinDTOS.stream().map(bulletinDTO -> bulletinDTO.getMessage()).collect(Collectors.joining("\n"));
                    jobExecution.setExitMessage(msg);
                    this.batchJobExecutionProvider.save(jobExecution);
                }
            }
        }, MetadataAccess.SERVICE);
    } catch (Exception e) {
        log.error("Unable to query NiFi and save exception bulletins for job failure eventid/flowfile : {} / {}. Exception Message:  {}", event.getEventId(), event.getFlowFileUuid(), e.getMessage(), e);
    }
}

Example 3 with ProvenanceEventRecordDTO

use of com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO in project kylo by Teradata.

the class NifiStatsJmsReceiver method ensureStreamingJobExecutionRecord.

private void ensureStreamingJobExecutionRecord(NifiFeedProcessorStats stats) {
    if (stats.getJobsStarted() > 0 || stats.getJobsFinished() > 0) {
        OpsManagerFeed feed = provenanceEventFeedUtil.getFeed(stats.getFeedName());
        if (feed != null && feed.isStream()) {
            ProvenanceEventRecordDTO event = new ProvenanceEventRecordDTO();
            event.setEventId(stats.getMaxEventId());
            event.setEventTime(stats.getMinEventTime().getMillis());
            event.setEventDuration(stats.getDuration());
            event.setFlowFileUuid(stats.getLatestFlowFileId());
            event.setJobFlowFileId(stats.getLatestFlowFileId());
            event.setComponentId(stats.getProcessorId());
            event.setComponentName(stats.getProcessorName());
            event.setIsFailure(stats.getFailedCount() > 0L);
            event.setStream(feed.isStream());
            event.setIsStartOfJob(stats.getJobsStarted() > 0L);
            event.setIsFinalJobEvent(stats.getJobsFinished() > 0L);
            event.setFeedProcessGroupId(stats.getFeedProcessGroupId());
            event.setFeedName(stats.getFeedName());
            ProvenanceEventRecordDTOHolder holder = new ProvenanceEventRecordDTOHolder();
            List<ProvenanceEventRecordDTO> events = new ArrayList<>();
            events.add(event);
            holder.setEvents(events);
            log.debug("Ensuring Streaming Feed Event: {} has a respective JobExecution record ", event);
            provenanceEventReceiver.receiveEvents(holder);
        }
    }
}

Example 4 with ProvenanceEventRecordDTO

use of com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO in project kylo by Teradata.

the class KyloKafkaProvenanceEventService method sendEvents.

@Override
public void sendEvents(List<ProvenanceEventRecordDTO> events) throws ProvenanceException {
    try {
        List<Future<RecordMetadata>> resultFutures = new ArrayList<>();
        ProvenanceEventRecordDTOHolder eventRecordDTOHolder = new ProvenanceEventRecordDTOHolder();
        List<ProvenanceEventRecordDTO> batchEvents = new ArrayList<>();
        for (ProvenanceEventRecordDTO event : events) {
            if (!event.isStream()) {
                batchEvents.add(event);
            }
        }
        eventRecordDTOHolder.setEvents(batchEvents);
        byte[] data = SerializationUtils.serialize(eventRecordDTOHolder);
        ProducerRecord<byte[], byte[]> eventsMessage = new ProducerRecord<>(KYLO_BATCH_EVENT_TOPIC, data);
        log.info("Sending {} events to Kafka ", eventRecordDTOHolder);
        resultFutures.add(kafkaProducer.send(eventsMessage));
        AggregatedFeedProcessorStatisticsHolder stats = GroupedStatsUtil.gatherStats(events);
        data = SerializationUtils.serialize(stats);
        ProducerRecord<byte[], byte[]> statsMessage = new ProducerRecord<>(KYLO_EVENT_STATS_TOPIC, data);
        resultFutures.add(kafkaProducer.send(statsMessage));
        processAcks(resultFutures);
    } catch (Exception e) {
        throw new ProvenanceException(e);
    }
}

Example 5 with ProvenanceEventRecordDTO

use of com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO in project kylo by Teradata.

the class JpaBatchJobExecutionProvider method getOrCreateStreamJobExecution.

private JpaBatchJobExecution getOrCreateStreamJobExecution(ProvenanceEventRecordDTO event, OpsManagerFeed feed) {
    JpaBatchJobExecution jobExecution = null;
    boolean isNew = false;
    try {
        BatchJobExecution latestJobExecution = latestStreamingJobByFeedName.get(event.getFeedName());
        if (latestJobExecution == null) {
            latestJobExecution = findLatestJobForFeed(event.getFeedName());
        } else {
            if (clusterService.isClustered()) {
                latestJobExecution = jobExecutionRepository.findOne(latestJobExecution.getJobExecutionId());
            }
        }
        if (latestJobExecution == null || (latestJobExecution != null && !latestJobExecution.isStream())) {
            // If the latest Job is not set to be a Stream and its still running we need to fail it and create the new streaming job.
            if (latestJobExecution != null && !latestJobExecution.isFinished()) {
                ProvenanceEventRecordDTO tempFailedEvent = new ProvenanceEventRecordDTO();
                tempFailedEvent.setFeedName(event.getFeedName());
                tempFailedEvent.setAttributeMap(new HashMap<>());
                tempFailedEvent.setIsFailure(true);
                tempFailedEvent.setDetails("Failed Running Batch event as this Feed has now become a Stream");
                finishJob(tempFailedEvent, (JpaBatchJobExecution) latestJobExecution);
                latestJobExecution.setExitMessage("Failed Running Batch event as this Feed has now become a Stream");
                save(latestJobExecution);
            }
            jobExecution = createNewJobExecution(event, feed);
            jobExecution.setStream(true);
            latestStreamingJobByFeedName.put(event.getFeedName(), jobExecution);
            log.info("Created new Streaming Job Execution with id of {} and starting event {} ", jobExecution.getJobExecutionId(), event);
        } else {
            jobExecution = (JpaBatchJobExecution) latestJobExecution;
        }
        if (jobExecution != null) {
            latestStreamingJobByFeedName.put(event.getFeedName(), jobExecution);
        }
    } catch (OptimisticLockException e) {
        // read
        jobExecution = (JpaBatchJobExecution) findLatestJobForFeed(event.getFeedName());
    }
    boolean save = isNew;
    if (!jobExecution.isStream()) {
        jobExecution.setStream(true);
        save = true;
    }
    if (save) {
        save(jobExecution);
    }
    return jobExecution;
}