use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolderV2 in project kylo by Teradata.
the class NifiStatsJmsReceiver method saveFeedStats.
/**
* Save the running totals for the feed
*/
private Map<String, JpaNifiFeedStats> saveFeedStats(AggregatedFeedProcessorStatisticsHolderV2 holder, List<NifiFeedProcessorStats> summaryStats) {
Map<String, JpaNifiFeedStats> feedStatsMap = new HashMap<>();
if (summaryStats != null) {
Map<String, Long> feedLatestTimestamp = summaryStats.stream().collect(Collectors.toMap(NifiFeedProcessorStats::getFeedName, stats -> stats.getMinEventTime().getMillis(), Long::max));
feedLatestTimestamp.entrySet().stream().forEach(e -> {
String feedName = e.getKey();
Long timestamp = e.getValue();
JpaNifiFeedStats stats = feedStatsMap.computeIfAbsent(feedName, name -> new JpaNifiFeedStats(feedName));
OpsManagerFeed opsManagerFeed = provenanceEventFeedUtil.getFeed(feedName);
if (opsManagerFeed != null) {
stats.setFeedId(new JpaNifiFeedStats.OpsManagerFeedId(opsManagerFeed.getId().toString()));
}
stats.setLastActivityTimestamp(timestamp);
});
}
if (holder.getProcessorIdRunningFlows() != null) {
holder.getProcessorIdRunningFlows().entrySet().stream().forEach(e -> {
String feedProcessorId = e.getKey();
Long runningCount = e.getValue();
// ensure not null
String feedName = provenanceEventFeedUtil.getFeedName(feedProcessorId);
if (StringUtils.isNotBlank(feedName)) {
JpaNifiFeedStats stats = feedStatsMap.computeIfAbsent(feedName, name -> new JpaNifiFeedStats(feedName));
OpsManagerFeed opsManagerFeed = provenanceEventFeedUtil.getFeed(feedName);
if (opsManagerFeed != null) {
stats.setFeedId(new JpaNifiFeedStats.OpsManagerFeedId(opsManagerFeed.getId().toString()));
stats.setStream(opsManagerFeed.isStream());
}
stats.addRunningFeedFlows(runningCount);
if (holder instanceof AggregatedFeedProcessorStatisticsHolderV3) {
stats.setTime(((AggregatedFeedProcessorStatisticsHolderV3) holder).getTimestamp());
if (stats.getLastActivityTimestamp() == null) {
stats.setLastActivityTimestamp(((AggregatedFeedProcessorStatisticsHolderV3) holder).getTimestamp());
}
} else {
stats.setTime(DateTime.now().getMillis());
}
if (stats.getLastActivityTimestamp() == null) {
log.warn("The JpaNifiFeedStats.lastActivityTimestamp for the feed {} is NULL. The JMS Class was: {}", feedName, holder.getClass().getSimpleName());
}
}
});
}
// group stats to save together by feed name
if (!feedStatsMap.isEmpty()) {
// only save those that have changed
List<NifiFeedStats> updatedStats = feedStatsMap.entrySet().stream().map(e -> e.getValue()).collect(Collectors.toList());
// if the running flows are 0 and its streaming we should try back to see if this feed is running or not
updatedStats.stream().filter(s -> s.isStream()).forEach(stats -> {
latestStatsCache.put(stats.getFeedName(), (JpaNifiFeedStats) stats);
if (stats.getRunningFeedFlows() == 0L) {
batchJobExecutionProvider.markStreamingFeedAsStopped(stats.getFeedName());
} else {
batchJobExecutionProvider.markStreamingFeedAsStarted(stats.getFeedName());
}
});
nifiFeedStatisticsProvider.saveLatestFeedStats(updatedStats);
}
return feedStatsMap;
}
use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolderV2 in project kylo by Teradata.
the class NifiStatsJmsReceiver method receiveTopic.
@JmsListener(id = JMS_LISTENER_ID, destination = Queues.PROVENANCE_EVENT_STATS_QUEUE, containerFactory = JmsConstants.QUEUE_LISTENER_CONTAINER_FACTORY)
public void receiveTopic(AggregatedFeedProcessorStatisticsHolder stats) {
if (readyToProcess(stats)) {
if (ensureValidRetryAttempt(stats)) {
final List<AggregatedFeedProcessorStatistics> unregisteredEvents = new ArrayList<>();
metadataAccess.commit(() -> {
List<NifiFeedProcessorStats> summaryStats = createSummaryStats(stats, unregisteredEvents);
List<JpaNifiFeedProcessorStats> failedStatsWithFlowFiles = new ArrayList<>();
for (NifiFeedProcessorStats stat : summaryStats) {
NifiFeedProcessorStats savedStats = nifiEventStatisticsProvider.create(stat);
if (savedStats.getFailedCount() > 0L && savedStats.getLatestFlowFileId() != null) {
// offload the query to nifi and merge back in
failedStatsWithFlowFiles.add((JpaNifiFeedProcessorStats) savedStats);
}
ensureStreamingJobExecutionRecord(stat);
}
if (stats instanceof AggregatedFeedProcessorStatisticsHolderV2) {
saveFeedStats((AggregatedFeedProcessorStatisticsHolderV2) stats, summaryStats);
}
if (!failedStatsWithFlowFiles.isEmpty()) {
assignNiFiBulletinErrors(failedStatsWithFlowFiles);
}
return summaryStats;
}, MetadataAccess.SERVICE);
if (clusterService.isClustered() && !unregisteredEvents.isEmpty()) {
// reprocess with delay
if (retryProvenanceEventWithDelay != null) {
retryProvenanceEventWithDelay.delay(stats, unregisteredEvents);
}
}
} else {
// stop processing the events
log.info("Unable find the feed in Ops Manager. Not processing {} stats ", stats.getFeedStatistics().values().size());
}
} else {
log.info("NiFi is not up yet. Sending back to JMS for later dequeue ");
throw new JmsProcessingException("Unable to process Statistics Events. NiFi is either not up, or there is an error trying to populate the Kylo NiFi Flow Cache. ");
}
}
use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolderV2 in project kylo by Teradata.
the class NifiStatsJmsReceiver method createSummaryStats.
private List<NifiFeedProcessorStats> createSummaryStats(AggregatedFeedProcessorStatisticsHolder holder, final List<AggregatedFeedProcessorStatistics> unregisteredEvents) {
List<NifiFeedProcessorStats> nifiFeedProcessorStatsList = new ArrayList<>();
holder.getFeedStatistics().values().stream().forEach(feedProcessorStats -> {
Long collectionIntervalMillis = feedProcessorStats.getCollectionIntervalMillis();
String feedProcessorId = feedProcessorStats.getStartingProcessorId();
String feedName = getFeedName(feedProcessorStats);
if (StringUtils.isNotBlank(feedName)) {
String feedProcessGroupId = provenanceEventFeedUtil.getFeedProcessGroupId(feedProcessorId);
feedProcessorStats.getProcessorStats().values().forEach(processorStats -> {
processorStats.getStats().values().stream().forEach(stats -> {
NifiFeedProcessorStats nifiFeedProcessorStats = toSummaryStats(stats);
nifiFeedProcessorStats.setFeedName(feedName);
nifiFeedProcessorStats.setProcessorId(processorStats.getProcessorId());
nifiFeedProcessorStats.setCollectionIntervalSeconds((collectionIntervalMillis / 1000));
if (holder instanceof AggregatedFeedProcessorStatisticsHolderV2) {
nifiFeedProcessorStats.setCollectionId(((AggregatedFeedProcessorStatisticsHolderV2) holder).getCollectionId());
}
String processorName = provenanceEventFeedUtil.getProcessorName(processorStats.getProcessorId());
if (processorName == null) {
processorName = processorStats.getProcessorName();
}
nifiFeedProcessorStats.setProcessorName(processorName);
nifiFeedProcessorStats.setFeedProcessGroupId(feedProcessGroupId);
nifiFeedProcessorStatsList.add(nifiFeedProcessorStats);
});
});
} else {
unregisteredEvents.add(feedProcessorStats);
}
});
return nifiFeedProcessorStatsList;
}
Aggregations