use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedProcessorStatistics in project kylo by Teradata.
the class FeedStatisticsManager method gatherStatistics.
public void gatherStatistics() {
lock.lock();
List<ProvenanceEventRecordDTO> eventsToSend = null;
Map<String, AggregatedFeedProcessorStatistics> statsToSend = null;
try {
// Gather Events and Stats to send Ops Manager
// filter out the streaming feeds
ensureStreamingFeedMetadata();
eventsToSend = feedStatisticsMap.values().stream().flatMap(stats -> stats.getEventsToSend().stream().filter(event -> !FeedEventStatistics.getInstance().streamingFeedProcessorIdsList.contains(event.getFirstEventProcessorId()))).sorted(Comparator.comparing(ProvenanceEventRecordDTO::getEventTime).thenComparing(ProvenanceEventRecordDTO::getEventId)).collect(Collectors.toList());
final String collectionId = UUID.randomUUID().toString();
Map<String, Long> runningFlowsCount = new HashMap<>();
for (FeedStatistics feedStatistics : feedStatisticsMap.values()) {
if (feedStatistics.hasStats()) {
if (statsToSend == null) {
statsToSend = new ConcurrentHashMap<>();
}
AggregatedFeedProcessorStatistics feedProcessorStatistics = statsToSend.computeIfAbsent(feedStatistics.getFeedProcessorId(), feedProcessorId -> new AggregatedFeedProcessorStatistics(feedStatistics.getFeedProcessorId(), collectionId, sendJmsTimeMillis));
AggregatedProcessorStatistics processorStatistics = feedProcessorStatistics.getProcessorStats().computeIfAbsent(feedStatistics.getProcessorId(), processorId -> new AggregatedProcessorStatisticsV2(feedStatistics.getProcessorId(), null, collectionId));
// accumulate the stats together into the processorStatistics object grouped by source connection id
feedStatistics.getStats().stream().forEach(stats -> {
FeedProcessorStatisticsAggregator.getInstance().addStats1(processorStatistics.getStats(stats.getSourceConnectionIdentifier()), stats);
});
}
}
if ((eventsToSend != null && !eventsToSend.isEmpty()) || (statsToSend != null && !statsToSend.isEmpty())) {
// send it off to jms on a different thread
JmsSender jmsSender = new JmsSender(eventsToSend, statsToSend.values(), FeedEventStatistics.getInstance().getRunningFeedFlowsForFeed(statsToSend.keySet()));
this.jmsService.submit(new JmsSenderConsumer(jmsSender));
} else {
// if we are empty but the runningFlows have changed, then send off as well
if (FeedEventStatistics.getInstance().isFeedProcessorRunningFeedFlowsChanged()) {
JmsSender jmsSender = new JmsSender(null, null, FeedEventStatistics.getInstance().getRunningFeedFlowsChanged());
this.jmsService.submit(new JmsSenderConsumer(jmsSender));
}
}
} finally {
FeedEventStatistics.getInstance().markFeedProcessorRunningFeedFlowsUnchanged();
feedStatisticsMap.values().stream().forEach(stats -> stats.clear());
lock.unlock();
}
}
use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedProcessorStatistics in project kylo by Teradata.
the class GroupedStatsUtil method groupStatsByProcessor.
/**
* Group stats together by processor for a given feed
*
* @param feedName feed name
* @param groupedStats Map of processorIdentity to list of stats for that processor
*/
public static AggregatedFeedProcessorStatistics groupStatsByProcessor(String feedName, Map<GroupedStatsIdentity, List<GroupedStats>> groupedStats) {
Long sendJmsTimeMillis = 3000L;
String collectionId = UUID.randomUUID().toString();
// Create a random id for the starting processor.
// the starting feed processor id is not needed as we already know the feed name associated with these stats
String startingProcessorId = UUID.randomUUID().toString();
AggregatedFeedProcessorStatisticsV2 feedProcessorStatistics = new AggregatedFeedProcessorStatisticsV2(startingProcessorId, collectionId, sendJmsTimeMillis);
feedProcessorStatistics.setFeedName(feedName);
for (Map.Entry<GroupedStatsIdentity, List<GroupedStats>> stats : groupedStats.entrySet()) {
String processorName = stats.getKey().getProcessorName();
String processorId = stats.getKey().getProcessorId();
Map<String, AggregatedProcessorStatistics> processorStatsMap = feedProcessorStatistics.getProcessorStats();
if (!processorStatsMap.containsKey(processorName)) {
processorStatsMap.put(processorName, new AggregatedProcessorStatisticsV2(processorId, processorName, collectionId));
}
AggregatedProcessorStatistics processorStatistics = processorStatsMap.get(processorName);
for (GroupedStats s : stats.getValue()) {
GroupedStatsUtil.addStats1(processorStatistics.getStats(GroupedStats.DEFAULT_SOURCE_CONNECTION_ID), s);
}
/*
AggregatedProcessorStatistics
processorStatistics =
feedProcessorStatistics.getProcessorStats()
.computeIfAbsent(processorName, k -> new AggregatedProcessorStatisticsV2(processorId, k, collectionId));
stats.getValue().stream().forEach( s -> GroupedStatsUtil.add(processorStatistics.getStats(GroupedStats.DEFAULT_SOURCE_CONNECTION_ID), s));
*/
}
return feedProcessorStatistics;
}
Aggregations