use of com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.JpaNifiFeedStats in project kylo by Teradata.
the class NifiStatsJmsReceiver method saveFeedStats.
/**
* Save the running totals for the feed
*/
private Map<String, JpaNifiFeedStats> saveFeedStats(AggregatedFeedProcessorStatisticsHolderV2 holder, List<NifiFeedProcessorStats> summaryStats) {
Map<String, JpaNifiFeedStats> feedStatsMap = new HashMap<>();
if (summaryStats != null) {
Map<String, Long> feedLatestTimestamp = summaryStats.stream().collect(Collectors.toMap(NifiFeedProcessorStats::getFeedName, stats -> stats.getMinEventTime().getMillis(), Long::max));
feedLatestTimestamp.entrySet().stream().forEach(e -> {
String feedName = e.getKey();
Long timestamp = e.getValue();
JpaNifiFeedStats stats = feedStatsMap.computeIfAbsent(feedName, name -> new JpaNifiFeedStats(feedName));
OpsManagerFeed opsManagerFeed = provenanceEventFeedUtil.getFeed(feedName);
if (opsManagerFeed != null) {
stats.setFeedId(new JpaNifiFeedStats.OpsManagerFeedId(opsManagerFeed.getId().toString()));
}
stats.setLastActivityTimestamp(timestamp);
});
}
if (holder.getProcessorIdRunningFlows() != null) {
holder.getProcessorIdRunningFlows().entrySet().stream().forEach(e -> {
String feedProcessorId = e.getKey();
Long runningCount = e.getValue();
// ensure not null
String feedName = provenanceEventFeedUtil.getFeedName(feedProcessorId);
if (StringUtils.isNotBlank(feedName)) {
JpaNifiFeedStats stats = feedStatsMap.computeIfAbsent(feedName, name -> new JpaNifiFeedStats(feedName));
OpsManagerFeed opsManagerFeed = provenanceEventFeedUtil.getFeed(feedName);
if (opsManagerFeed != null) {
stats.setFeedId(new JpaNifiFeedStats.OpsManagerFeedId(opsManagerFeed.getId().toString()));
stats.setStream(opsManagerFeed.isStream());
}
stats.addRunningFeedFlows(runningCount);
if (holder instanceof AggregatedFeedProcessorStatisticsHolderV3) {
stats.setTime(((AggregatedFeedProcessorStatisticsHolderV3) holder).getTimestamp());
if (stats.getLastActivityTimestamp() == null) {
stats.setLastActivityTimestamp(((AggregatedFeedProcessorStatisticsHolderV3) holder).getTimestamp());
}
} else {
stats.setTime(DateTime.now().getMillis());
}
if (stats.getLastActivityTimestamp() == null) {
log.warn("The JpaNifiFeedStats.lastActivityTimestamp for the feed {} is NULL. The JMS Class was: {}", feedName, holder.getClass().getSimpleName());
}
}
});
}
// group stats to save together by feed name
if (!feedStatsMap.isEmpty()) {
// only save those that have changed
List<NifiFeedStats> updatedStats = feedStatsMap.entrySet().stream().map(e -> e.getValue()).collect(Collectors.toList());
// if the running flows are 0 and its streaming we should try back to see if this feed is running or not
updatedStats.stream().filter(s -> s.isStream()).forEach(stats -> {
latestStatsCache.put(stats.getFeedName(), (JpaNifiFeedStats) stats);
if (stats.getRunningFeedFlows() == 0L) {
batchJobExecutionProvider.markStreamingFeedAsStopped(stats.getFeedName());
} else {
batchJobExecutionProvider.markStreamingFeedAsStarted(stats.getFeedName());
}
});
nifiFeedStatisticsProvider.saveLatestFeedStats(updatedStats);
}
return feedStatsMap;
}
use of com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.JpaNifiFeedStats in project kylo by Teradata.
the class OpsFeedManagerFeedProvider method save.
@Override
public OpsManagerFeed save(OpsManagerFeed.ID feedId, String systemName, boolean isStream, Long timeBetweenBatchJobs) {
OpsManagerFeed feed = repository.findByIdWithoutAcl(feedId);
if (feed == null) {
if (ensureUniqueFeedName) {
ensureAndRemoveDuplicateFeedsWithTheSameName(systemName, feedId);
}
feed = new JpaOpsManagerFeed();
((JpaOpsManagerFeed) feed).setName(systemName);
((JpaOpsManagerFeed) feed).setId((OpsManagerFeedId) feedId);
((JpaOpsManagerFeed) feed).setStream(isStream);
((JpaOpsManagerFeed) feed).setTimeBetweenBatchJobs(timeBetweenBatchJobs);
NifiFeedStats stats = feedStatisticsProvider.findLatestStatsForFeedWithoutAccessControl(systemName);
if (stats == null) {
JpaNifiFeedStats newStats = new JpaNifiFeedStats(systemName, new JpaNifiFeedStats.OpsManagerFeedId(feedId.toString()));
newStats.setRunningFeedFlows(0L);
feedStatisticsProvider.saveLatestFeedStats(Lists.newArrayList(newStats));
}
} else {
((JpaOpsManagerFeed) feed).setStream(isStream);
((JpaOpsManagerFeed) feed).setTimeBetweenBatchJobs(timeBetweenBatchJobs);
}
feed = save(feed);
return feed;
}
Aggregations