Examples with AggregatedProcessorStatisticsV2 com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedProcessorStatisticsV2 used on opensource projects

Example 1 with AggregatedProcessorStatisticsV2

use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedProcessorStatisticsV2 in project kylo by Teradata.

the class GroupedStatsUtil method groupStatsByProcessor.

/**
 * Group stats together by processor for a given feed
 *
 * @param feedName     feed name
 * @param groupedStats Map of processorIdentity to list of stats for that processor
 */
public static AggregatedFeedProcessorStatistics groupStatsByProcessor(String feedName, Map<GroupedStatsIdentity, List<GroupedStats>> groupedStats) {
    Long sendJmsTimeMillis = 3000L;
    String collectionId = UUID.randomUUID().toString();
    // Create a random id for the starting processor.
    // the starting feed processor id is not needed as we already know the feed name associated with these stats
    String startingProcessorId = UUID.randomUUID().toString();
    AggregatedFeedProcessorStatisticsV2 feedProcessorStatistics = new AggregatedFeedProcessorStatisticsV2(startingProcessorId, collectionId, sendJmsTimeMillis);
    feedProcessorStatistics.setFeedName(feedName);
    for (Map.Entry<GroupedStatsIdentity, List<GroupedStats>> stats : groupedStats.entrySet()) {
        String processorName = stats.getKey().getProcessorName();
        String processorId = stats.getKey().getProcessorId();
        Map<String, AggregatedProcessorStatistics> processorStatsMap = feedProcessorStatistics.getProcessorStats();
        if (!processorStatsMap.containsKey(processorName)) {
            processorStatsMap.put(processorName, new AggregatedProcessorStatisticsV2(processorId, processorName, collectionId));
        }
        AggregatedProcessorStatistics processorStatistics = processorStatsMap.get(processorName);
        for (GroupedStats s : stats.getValue()) {
            GroupedStatsUtil.addStats1(processorStatistics.getStats(GroupedStats.DEFAULT_SOURCE_CONNECTION_ID), s);
        }
    /*
            AggregatedProcessorStatistics
                processorStatistics =
                feedProcessorStatistics.getProcessorStats()
                    .computeIfAbsent(processorName, k -> new AggregatedProcessorStatisticsV2(processorId, k, collectionId));


            stats.getValue().stream().forEach( s ->  GroupedStatsUtil.add(processorStatistics.getStats(GroupedStats.DEFAULT_SOURCE_CONNECTION_ID), s));
            */
    }
    return feedProcessorStatistics;
}

Example 2 with AggregatedProcessorStatisticsV2

use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedProcessorStatisticsV2 in project kylo by Teradata.

the class FeedStatisticsManager method gatherStatistics.

public void gatherStatistics() {
    lock.lock();
    List<ProvenanceEventRecordDTO> eventsToSend = null;
    Map<String, AggregatedFeedProcessorStatistics> statsToSend = null;
    try {
        // Gather Events and Stats to send Ops Manager
        // filter out the streaming feeds
        ensureStreamingFeedMetadata();
        eventsToSend = feedStatisticsMap.values().stream().flatMap(stats -> stats.getEventsToSend().stream().filter(event -> !FeedEventStatistics.getInstance().streamingFeedProcessorIdsList.contains(event.getFirstEventProcessorId()))).sorted(Comparator.comparing(ProvenanceEventRecordDTO::getEventTime).thenComparing(ProvenanceEventRecordDTO::getEventId)).collect(Collectors.toList());
        final String collectionId = UUID.randomUUID().toString();
        Map<String, Long> runningFlowsCount = new HashMap<>();
        for (FeedStatistics feedStatistics : feedStatisticsMap.values()) {
            if (feedStatistics.hasStats()) {
                if (statsToSend == null) {
                    statsToSend = new ConcurrentHashMap<>();
                }
                AggregatedFeedProcessorStatistics feedProcessorStatistics = statsToSend.computeIfAbsent(feedStatistics.getFeedProcessorId(), feedProcessorId -> new AggregatedFeedProcessorStatistics(feedStatistics.getFeedProcessorId(), collectionId, sendJmsTimeMillis));
                AggregatedProcessorStatistics processorStatistics = feedProcessorStatistics.getProcessorStats().computeIfAbsent(feedStatistics.getProcessorId(), processorId -> new AggregatedProcessorStatisticsV2(feedStatistics.getProcessorId(), null, collectionId));
                // accumulate the stats together into the processorStatistics object grouped by source connection id
                feedStatistics.getStats().stream().forEach(stats -> {
                    FeedProcessorStatisticsAggregator.getInstance().addStats1(processorStatistics.getStats(stats.getSourceConnectionIdentifier()), stats);
                });
            }
        }
        boolean runningFlowsChanged = FeedEventStatistics.getInstance().isFeedProcessorRunningFeedFlowsChanged();
        if ((eventsToSend != null && !eventsToSend.isEmpty()) || (statsToSend != null && !statsToSend.isEmpty())) {
            // send it off to jms on a different thread
            JmsSender jmsSender = new JmsSender(eventsToSend, statsToSend.values(), FeedEventStatistics.getInstance().getRunningFeedFlowsForFeed(statsToSend.keySet()), runningFlowsChanged);
            this.jmsService.submit(new JmsSenderConsumer(jmsSender));
        } else {
            JmsSender jmsSender = new JmsSender(null, null, FeedEventStatistics.getInstance().getRunningFeedFlowsChanged(), runningFlowsChanged);
            this.jmsService.submit(new JmsSenderConsumer(jmsSender));
        }
    } finally {
        FeedEventStatistics.getInstance().markFeedProcessorRunningFeedFlowsUnchanged();
        feedStatisticsMap.values().stream().forEach(stats -> stats.clear());
        lock.unlock();
    }
}