Examples with JpaNifiFeedProcessorStats com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.JpaNifiFeedProcessorStats used on opensource projects

Example 1 with JpaNifiFeedProcessorStats

use of com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.JpaNifiFeedProcessorStats in project kylo by Teradata.

the class NifiStatsJmsReceiver method toSummaryStats.

private NifiFeedProcessorStats toSummaryStats(GroupedStats groupedStats) {
    NifiFeedProcessorStats nifiFeedProcessorStats = new JpaNifiFeedProcessorStats();
    nifiFeedProcessorStats.setTotalCount(groupedStats.getTotalCount());
    nifiFeedProcessorStats.setFlowFilesFinished(groupedStats.getFlowFilesFinished());
    nifiFeedProcessorStats.setFlowFilesStarted(groupedStats.getFlowFilesStarted());
    nifiFeedProcessorStats.setCollectionId(groupedStats.getGroupKey());
    nifiFeedProcessorStats.setBytesIn(groupedStats.getBytesIn());
    nifiFeedProcessorStats.setBytesOut(groupedStats.getBytesOut());
    nifiFeedProcessorStats.setDuration(groupedStats.getDuration());
    nifiFeedProcessorStats.setJobsFinished(groupedStats.getJobsFinished());
    nifiFeedProcessorStats.setJobsStarted(groupedStats.getJobsStarted());
    nifiFeedProcessorStats.setProcessorsFailed(groupedStats.getProcessorsFailed());
    nifiFeedProcessorStats.setCollectionTime(new DateTime(groupedStats.getTime()));
    nifiFeedProcessorStats.setMinEventTime(new DateTime(groupedStats.getMinTime()));
    nifiFeedProcessorStats.setMinEventTimeMillis(nifiFeedProcessorStats.getMinEventTime().getMillis());
    nifiFeedProcessorStats.setMaxEventTime(new DateTime(groupedStats.getMaxTime()));
    nifiFeedProcessorStats.setJobsFailed(groupedStats.getJobsFailed());
    nifiFeedProcessorStats.setSuccessfulJobDuration(groupedStats.getSuccessfulJobDuration());
    nifiFeedProcessorStats.setJobDuration(groupedStats.getJobDuration());
    nifiFeedProcessorStats.setMaxEventId(groupedStats.getMaxEventId());
    nifiFeedProcessorStats.setFailedCount(groupedStats.getProcessorsFailed());
    if (groupedStats instanceof GroupedStatsV2) {
        nifiFeedProcessorStats.setLatestFlowFileId(((GroupedStatsV2) groupedStats).getLatestFlowFileId());
    }
    if (provenanceEventFeedUtil.isFailure(groupedStats.getSourceConnectionIdentifier())) {
        nifiFeedProcessorStats.setFailedCount(groupedStats.getTotalCount() + groupedStats.getProcessorsFailed());
    }
    return nifiFeedProcessorStats;
}

Example 2 with JpaNifiFeedProcessorStats

use of com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.JpaNifiFeedProcessorStats in project kylo by Teradata.

the class NifiStatsJmsReceiver method receiveTopic.

@JmsListener(id = JMS_LISTENER_ID, destination = Queues.PROVENANCE_EVENT_STATS_QUEUE, containerFactory = JmsConstants.QUEUE_LISTENER_CONTAINER_FACTORY)
public void receiveTopic(AggregatedFeedProcessorStatisticsHolder stats) {
    if (readyToProcess(stats)) {
        if (ensureValidRetryAttempt(stats)) {
            final List<AggregatedFeedProcessorStatistics> unregisteredEvents = new ArrayList<>();
            metadataAccess.commit(() -> {
                List<NifiFeedProcessorStats> summaryStats = createSummaryStats(stats, unregisteredEvents);
                List<JpaNifiFeedProcessorStats> failedStatsWithFlowFiles = new ArrayList<>();
                for (NifiFeedProcessorStats stat : summaryStats) {
                    NifiFeedProcessorStats savedStats = nifiEventStatisticsProvider.create(stat);
                    if (savedStats.getFailedCount() > 0L && savedStats.getLatestFlowFileId() != null) {
                        // offload the query to nifi and merge back in
                        failedStatsWithFlowFiles.add((JpaNifiFeedProcessorStats) savedStats);
                    }
                    ensureStreamingJobExecutionRecord(stat);
                }
                if (stats instanceof AggregatedFeedProcessorStatisticsHolderV2) {
                    saveFeedStats((AggregatedFeedProcessorStatisticsHolderV2) stats, summaryStats);
                }
                if (!failedStatsWithFlowFiles.isEmpty()) {
                    assignNiFiBulletinErrors(failedStatsWithFlowFiles);
                }
                return summaryStats;
            }, MetadataAccess.SERVICE);
            if (clusterService.isClustered() && !unregisteredEvents.isEmpty()) {
                // reprocess with delay
                if (retryProvenanceEventWithDelay != null) {
                    retryProvenanceEventWithDelay.delay(stats, unregisteredEvents);
                }
            }
        } else {
            // stop processing the events
            log.info("Unable find the feed in Ops Manager.  Not processing {} stats ", stats.getFeedStatistics().values().size());
        }
    } else {
        log.info("NiFi is not up yet.  Sending back to JMS for later dequeue ");
        throw new JmsProcessingException("Unable to process Statistics Events.  NiFi is either not up, or there is an error trying to populate the Kylo NiFi Flow Cache. ");
    }
}