Examples with AggregatedFeedProcessorStatistics com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatistics used on opensource projects

Example 1 with AggregatedFeedProcessorStatistics

use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatistics in project kylo by Teradata.

the class NifiStatsJmsReceiver method receiveTopic.

@JmsListener(id = JMS_LISTENER_ID, destination = Queues.PROVENANCE_EVENT_STATS_QUEUE, containerFactory = JmsConstants.QUEUE_LISTENER_CONTAINER_FACTORY)
public void receiveTopic(AggregatedFeedProcessorStatisticsHolder stats) {
    if (readyToProcess(stats)) {
        if (ensureValidRetryAttempt(stats)) {
            final List<AggregatedFeedProcessorStatistics> unregisteredEvents = new ArrayList<>();
            metadataAccess.commit(() -> {
                List<NifiFeedProcessorStats> summaryStats = createSummaryStats(stats, unregisteredEvents);
                List<JpaNifiFeedProcessorStats> failedStatsWithFlowFiles = new ArrayList<>();
                for (NifiFeedProcessorStats stat : summaryStats) {
                    NifiFeedProcessorStats savedStats = nifiEventStatisticsProvider.create(stat);
                    if (savedStats.getFailedCount() > 0L && savedStats.getLatestFlowFileId() != null) {
                        // offload the query to nifi and merge back in
                        failedStatsWithFlowFiles.add((JpaNifiFeedProcessorStats) savedStats);
                    }
                    ensureStreamingJobExecutionRecord(stat);
                }
                if (stats instanceof AggregatedFeedProcessorStatisticsHolderV2) {
                    saveFeedStats((AggregatedFeedProcessorStatisticsHolderV2) stats, summaryStats);
                }
                if (!failedStatsWithFlowFiles.isEmpty()) {
                    assignNiFiBulletinErrors(failedStatsWithFlowFiles);
                }
                return summaryStats;
            }, MetadataAccess.SERVICE);
            if (clusterService.isClustered() && !unregisteredEvents.isEmpty()) {
                // reprocess with delay
                if (retryProvenanceEventWithDelay != null) {
                    retryProvenanceEventWithDelay.delay(stats, unregisteredEvents);
                }
            }
        } else {
            // stop processing the events
            log.info("Unable find the feed in Ops Manager.  Not processing {} stats ", stats.getFeedStatistics().values().size());
        }
    } else {
        log.info("NiFi is not up yet.  Sending back to JMS for later dequeue ");
        throw new JmsProcessingException("Unable to process Statistics Events.  NiFi is either not up, or there is an error trying to populate the Kylo NiFi Flow Cache. ");
    }
}

Example 2 with AggregatedFeedProcessorStatistics

use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatistics in project kylo by Teradata.

the class FeedStatisticsManager method gatherStatistics.

public void gatherStatistics() {
    lock.lock();
    List<ProvenanceEventRecordDTO> eventsToSend = null;
    Map<String, AggregatedFeedProcessorStatistics> statsToSend = null;
    try {
        // Gather Events and Stats to send Ops Manager
        // filter out the streaming feeds
        ensureStreamingFeedMetadata();
        eventsToSend = feedStatisticsMap.values().stream().flatMap(stats -> stats.getEventsToSend().stream().filter(event -> !FeedEventStatistics.getInstance().streamingFeedProcessorIdsList.contains(event.getFirstEventProcessorId()))).sorted(Comparator.comparing(ProvenanceEventRecordDTO::getEventTime).thenComparing(ProvenanceEventRecordDTO::getEventId)).collect(Collectors.toList());
        final String collectionId = UUID.randomUUID().toString();
        Map<String, Long> runningFlowsCount = new HashMap<>();
        for (FeedStatistics feedStatistics : feedStatisticsMap.values()) {
            if (feedStatistics.hasStats()) {
                if (statsToSend == null) {
                    statsToSend = new ConcurrentHashMap<>();
                }
                AggregatedFeedProcessorStatistics feedProcessorStatistics = statsToSend.computeIfAbsent(feedStatistics.getFeedProcessorId(), feedProcessorId -> new AggregatedFeedProcessorStatistics(feedStatistics.getFeedProcessorId(), collectionId, sendJmsTimeMillis));
                AggregatedProcessorStatistics processorStatistics = feedProcessorStatistics.getProcessorStats().computeIfAbsent(feedStatistics.getProcessorId(), processorId -> new AggregatedProcessorStatisticsV2(feedStatistics.getProcessorId(), null, collectionId));
                // accumulate the stats together into the processorStatistics object grouped by source connection id
                feedStatistics.getStats().stream().forEach(stats -> {
                    FeedProcessorStatisticsAggregator.getInstance().addStats1(processorStatistics.getStats(stats.getSourceConnectionIdentifier()), stats);
                });
            }
        }
        if ((eventsToSend != null && !eventsToSend.isEmpty()) || (statsToSend != null && !statsToSend.isEmpty())) {
            // send it off to jms on a different thread
            JmsSender jmsSender = new JmsSender(eventsToSend, statsToSend.values(), FeedEventStatistics.getInstance().getRunningFeedFlowsForFeed(statsToSend.keySet()));
            this.jmsService.submit(new JmsSenderConsumer(jmsSender));
        } else {
            // if we are empty but the runningFlows have changed, then send off as well
            if (FeedEventStatistics.getInstance().isFeedProcessorRunningFeedFlowsChanged()) {
                JmsSender jmsSender = new JmsSender(null, null, FeedEventStatistics.getInstance().getRunningFeedFlowsChanged());
                this.jmsService.submit(new JmsSenderConsumer(jmsSender));
            }
        }
    } finally {
        FeedEventStatistics.getInstance().markFeedProcessorRunningFeedFlowsUnchanged();
        feedStatisticsMap.values().stream().forEach(stats -> stats.clear());
        lock.unlock();
    }
}

Example 3 with AggregatedFeedProcessorStatistics

use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatistics in project kylo by Teradata.

the class GroupedStatsUtil method gatherStats.

/**
 * Gather feed stats for a list of events
 */
public static AggregatedFeedProcessorStatisticsHolder gatherStats(final List<ProvenanceEventRecordDTO> events) {
    Map<String, Map<GroupedStatsIdentity, List<GroupedStats>>> feedStatsByProcessor = new ConcurrentHashMap<>();
    // events.stream().forEach(e -> {
    for (ProvenanceEventRecordDTO e : events) {
        if (!feedStatsByProcessor.containsKey(e.getFeedName())) {
            feedStatsByProcessor.put(e.getFeedName(), new ConcurrentHashMap<GroupedStatsIdentity, List<GroupedStats>>());
        }
        // feedStatsByProcessor.putIfAbsent(e.getFeedName(), );
        Map<GroupedStatsIdentity, List<GroupedStats>> feedStats = feedStatsByProcessor.get(e.getFeedName());
        GroupedStatsIdentity identity = new GroupedStatsIdentity(e.getComponentId(), e.getComponentName());
        if (!feedStats.containsKey(identity)) {
            feedStats.put(identity, new ArrayList<GroupedStats>());
        }
        // feedStats.putIfAbsent(identity, new ArrayList<>());
        List<GroupedStats> feedProcessorStats = feedStats.get(identity);
        // Add the new stats
        GroupedStats statsV2 = GroupedStatsUtil.add(new GroupedStatsV2(), e);
        feedProcessorStats.add(statsV2);
    }
    // );
    List<AggregatedFeedProcessorStatistics> statsList = new ArrayList<>();
    for (Map.Entry<String, Map<GroupedStatsIdentity, List<GroupedStats>>> feedStats : feedStatsByProcessor.entrySet()) {
        AggregatedFeedProcessorStatistics feedProcessorStatistics = GroupedStatsUtil.groupStatsByProcessor(feedStats.getKey(), feedStats.getValue());
        statsList.add(feedProcessorStatistics);
    }
    /* feedStatsByProcessor.entrySet().stream().forEach(feedStats -> {
            AggregatedFeedProcessorStatistics feedProcessorStatistics = GroupedStatsUtil.groupStatsByProcessor(feedStats.getKey(), feedStats.getValue());
            statsList.add(feedProcessorStatistics);
        });
        */
    AggregatedFeedProcessorStatisticsHolderV3 feedProcessorStatisticsHolderV3 = new AggregatedFeedProcessorStatisticsHolderV3();
    feedProcessorStatisticsHolderV3.setFeedStatistics(statsList);
    return feedProcessorStatisticsHolderV3;
}