Search in sources :

Example 1 with AggregatedFeedProcessorStatisticsHolderV2

use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolderV2 in project kylo by Teradata.

the class NifiStatsJmsReceiver method saveFeedStats.

/**
 * Save the running totals for the feed
 */
private Map<String, JpaNifiFeedStats> saveFeedStats(AggregatedFeedProcessorStatisticsHolderV2 holder, List<NifiFeedProcessorStats> summaryStats) {
    Map<String, JpaNifiFeedStats> feedStatsMap = new HashMap<>();
    if (summaryStats != null) {
        Map<String, Long> feedLatestTimestamp = summaryStats.stream().collect(Collectors.toMap(NifiFeedProcessorStats::getFeedName, stats -> stats.getMinEventTime().getMillis(), Long::max));
        feedLatestTimestamp.entrySet().stream().forEach(e -> {
            String feedName = e.getKey();
            Long timestamp = e.getValue();
            JpaNifiFeedStats stats = feedStatsMap.computeIfAbsent(feedName, name -> new JpaNifiFeedStats(feedName));
            OpsManagerFeed opsManagerFeed = provenanceEventFeedUtil.getFeed(feedName);
            if (opsManagerFeed != null) {
                stats.setFeedId(new JpaNifiFeedStats.OpsManagerFeedId(opsManagerFeed.getId().toString()));
            }
            stats.setLastActivityTimestamp(timestamp);
        });
    }
    if (holder.getProcessorIdRunningFlows() != null) {
        holder.getProcessorIdRunningFlows().entrySet().stream().forEach(e -> {
            String feedProcessorId = e.getKey();
            Long runningCount = e.getValue();
            // ensure not null
            String feedName = provenanceEventFeedUtil.getFeedName(feedProcessorId);
            if (StringUtils.isNotBlank(feedName)) {
                JpaNifiFeedStats stats = feedStatsMap.computeIfAbsent(feedName, name -> new JpaNifiFeedStats(feedName));
                OpsManagerFeed opsManagerFeed = provenanceEventFeedUtil.getFeed(feedName);
                if (opsManagerFeed != null) {
                    stats.setFeedId(new JpaNifiFeedStats.OpsManagerFeedId(opsManagerFeed.getId().toString()));
                    stats.setStream(opsManagerFeed.isStream());
                }
                stats.addRunningFeedFlows(runningCount);
                if (holder instanceof AggregatedFeedProcessorStatisticsHolderV3) {
                    stats.setTime(((AggregatedFeedProcessorStatisticsHolderV3) holder).getTimestamp());
                    if (stats.getLastActivityTimestamp() == null) {
                        stats.setLastActivityTimestamp(((AggregatedFeedProcessorStatisticsHolderV3) holder).getTimestamp());
                    }
                } else {
                    stats.setTime(DateTime.now().getMillis());
                }
                if (stats.getLastActivityTimestamp() == null) {
                    log.warn("The JpaNifiFeedStats.lastActivityTimestamp for the feed {} is NULL.  The JMS Class was: {}", feedName, holder.getClass().getSimpleName());
                }
            }
        });
    }
    // group stats to save together by feed name
    if (!feedStatsMap.isEmpty()) {
        // only save those that have changed
        List<NifiFeedStats> updatedStats = feedStatsMap.entrySet().stream().map(e -> e.getValue()).collect(Collectors.toList());
        // if the running flows are 0 and its streaming we should try back to see if this feed is running or not
        updatedStats.stream().filter(s -> s.isStream()).forEach(stats -> {
            latestStatsCache.put(stats.getFeedName(), (JpaNifiFeedStats) stats);
            if (stats.getRunningFeedFlows() == 0L) {
                batchJobExecutionProvider.markStreamingFeedAsStopped(stats.getFeedName());
            } else {
                batchJobExecutionProvider.markStreamingFeedAsStarted(stats.getFeedName());
            }
        });
        nifiFeedStatisticsProvider.saveLatestFeedStats(updatedStats);
    }
    return feedStatsMap;
}
Also used : JpaNifiFeedStats(com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.JpaNifiFeedStats) DateTimeZone(org.joda.time.DateTimeZone) LoadingCache(com.google.common.cache.LoadingCache) JobScheduler(com.thinkbiganalytics.scheduler.JobScheduler) ClusterServiceMessageReceiver(com.thinkbiganalytics.cluster.ClusterServiceMessageReceiver) ProvenanceEventRecordDTOHolder(com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTOHolder) LoggerFactory(org.slf4j.LoggerFactory) AggregatedFeedProcessorStatistics(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatistics) NifiFeedProcessorStats(com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedProcessorStats) StringUtils(org.apache.commons.lang3.StringUtils) Queues(com.thinkbiganalytics.jms.Queues) NifiFeedProcessorErrors(com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedProcessorErrors) NifiFeedProcessorStatisticsProvider(com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedProcessorStatisticsProvider) DefaultTriggerIdentifier(com.thinkbiganalytics.scheduler.model.DefaultTriggerIdentifier) Map(java.util.Map) OpsManagerFeed(com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed) JmsListener(org.springframework.jms.annotation.JmsListener) MetadataAccess(com.thinkbiganalytics.metadata.api.MetadataAccess) NifiFeedStatisticsProvider(com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedStatisticsProvider) BulletinDTO(org.apache.nifi.web.api.dto.BulletinDTO) AggregatedFeedProcessorStatisticsHolderV3(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolderV3) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) AggregatedFeedProcessorStatisticsHolderV2(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolderV2) ObjectAlreadyExistsException(org.quartz.ObjectAlreadyExistsException) Set(java.util.Set) ClusterMessage(com.thinkbiganalytics.cluster.ClusterMessage) Collectors(java.util.stream.Collectors) CacheLoader(com.google.common.cache.CacheLoader) BatchJobExecutionProvider(com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecutionProvider) List(java.util.List) AggregatedFeedProcessorStatisticsV2(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsV2) GroupedStatsV2(com.thinkbiganalytics.nifi.provenance.model.stats.GroupedStatsV2) DefaultJobIdentifier(com.thinkbiganalytics.scheduler.model.DefaultJobIdentifier) PostConstruct(javax.annotation.PostConstruct) ClusterService(com.thinkbiganalytics.cluster.ClusterService) CacheBuilder(com.google.common.cache.CacheBuilder) Queue(java.util.Queue) QuartzScheduler(com.thinkbiganalytics.scheduler.QuartzScheduler) AggregatedFeedProcessorStatisticsHolder(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolder) SerializationUtils(org.apache.commons.lang.SerializationUtils) HashMap(java.util.HashMap) JpaNifiFeedStats(com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.JpaNifiFeedStats) ArrayList(java.util.ArrayList) Value(org.springframework.beans.factory.annotation.Value) HashSet(java.util.HashSet) Inject(javax.inject.Inject) SchedulerException(org.quartz.SchedulerException) EvictingQueue(com.google.common.collect.EvictingQueue) JmsConstants(com.thinkbiganalytics.jms.JmsConstants) ProvenanceEventRecordDTO(com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO) JobIdentifier(com.thinkbiganalytics.scheduler.JobIdentifier) Logger(org.slf4j.Logger) DateTime(org.joda.time.DateTime) NifiFeedStats(com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedStats) JpaNifiFeedProcessorStats(com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.JpaNifiFeedProcessorStats) TriggerIdentifier(com.thinkbiganalytics.scheduler.TriggerIdentifier) Collections(java.util.Collections) GroupedStats(com.thinkbiganalytics.nifi.provenance.model.stats.GroupedStats) OpsManagerFeed(com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) JpaNifiFeedStats(com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.JpaNifiFeedStats) NifiFeedStats(com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedStats) AggregatedFeedProcessorStatisticsHolderV3(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolderV3)

Example 2 with AggregatedFeedProcessorStatisticsHolderV2

use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolderV2 in project kylo by Teradata.

the class NifiStatsJmsReceiver method receiveTopic.

@JmsListener(id = JMS_LISTENER_ID, destination = Queues.PROVENANCE_EVENT_STATS_QUEUE, containerFactory = JmsConstants.QUEUE_LISTENER_CONTAINER_FACTORY)
public void receiveTopic(AggregatedFeedProcessorStatisticsHolder stats) {
    if (readyToProcess(stats)) {
        if (ensureValidRetryAttempt(stats)) {
            final List<AggregatedFeedProcessorStatistics> unregisteredEvents = new ArrayList<>();
            metadataAccess.commit(() -> {
                List<NifiFeedProcessorStats> summaryStats = createSummaryStats(stats, unregisteredEvents);
                List<JpaNifiFeedProcessorStats> failedStatsWithFlowFiles = new ArrayList<>();
                for (NifiFeedProcessorStats stat : summaryStats) {
                    NifiFeedProcessorStats savedStats = nifiEventStatisticsProvider.create(stat);
                    if (savedStats.getFailedCount() > 0L && savedStats.getLatestFlowFileId() != null) {
                        // offload the query to nifi and merge back in
                        failedStatsWithFlowFiles.add((JpaNifiFeedProcessorStats) savedStats);
                    }
                    ensureStreamingJobExecutionRecord(stat);
                }
                if (stats instanceof AggregatedFeedProcessorStatisticsHolderV2) {
                    saveFeedStats((AggregatedFeedProcessorStatisticsHolderV2) stats, summaryStats);
                }
                if (!failedStatsWithFlowFiles.isEmpty()) {
                    assignNiFiBulletinErrors(failedStatsWithFlowFiles);
                }
                return summaryStats;
            }, MetadataAccess.SERVICE);
            if (clusterService.isClustered() && !unregisteredEvents.isEmpty()) {
                // reprocess with delay
                if (retryProvenanceEventWithDelay != null) {
                    retryProvenanceEventWithDelay.delay(stats, unregisteredEvents);
                }
            }
        } else {
            // stop processing the events
            log.info("Unable find the feed in Ops Manager.  Not processing {} stats ", stats.getFeedStatistics().values().size());
        }
    } else {
        log.info("NiFi is not up yet.  Sending back to JMS for later dequeue ");
        throw new JmsProcessingException("Unable to process Statistics Events.  NiFi is either not up, or there is an error trying to populate the Kylo NiFi Flow Cache. ");
    }
}
Also used : JpaNifiFeedProcessorStats(com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.JpaNifiFeedProcessorStats) ArrayList(java.util.ArrayList) AggregatedFeedProcessorStatistics(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatistics) NifiFeedProcessorStats(com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedProcessorStats) JpaNifiFeedProcessorStats(com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.JpaNifiFeedProcessorStats) AggregatedFeedProcessorStatisticsHolderV2(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolderV2) JmsListener(org.springframework.jms.annotation.JmsListener)

Example 3 with AggregatedFeedProcessorStatisticsHolderV2

use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolderV2 in project kylo by Teradata.

the class NifiStatsJmsReceiver method createSummaryStats.

private List<NifiFeedProcessorStats> createSummaryStats(AggregatedFeedProcessorStatisticsHolder holder, final List<AggregatedFeedProcessorStatistics> unregisteredEvents) {
    List<NifiFeedProcessorStats> nifiFeedProcessorStatsList = new ArrayList<>();
    holder.getFeedStatistics().values().stream().forEach(feedProcessorStats -> {
        Long collectionIntervalMillis = feedProcessorStats.getCollectionIntervalMillis();
        String feedProcessorId = feedProcessorStats.getStartingProcessorId();
        String feedName = getFeedName(feedProcessorStats);
        if (StringUtils.isNotBlank(feedName)) {
            String feedProcessGroupId = provenanceEventFeedUtil.getFeedProcessGroupId(feedProcessorId);
            feedProcessorStats.getProcessorStats().values().forEach(processorStats -> {
                processorStats.getStats().values().stream().forEach(stats -> {
                    NifiFeedProcessorStats nifiFeedProcessorStats = toSummaryStats(stats);
                    nifiFeedProcessorStats.setFeedName(feedName);
                    nifiFeedProcessorStats.setProcessorId(processorStats.getProcessorId());
                    nifiFeedProcessorStats.setCollectionIntervalSeconds((collectionIntervalMillis / 1000));
                    if (holder instanceof AggregatedFeedProcessorStatisticsHolderV2) {
                        nifiFeedProcessorStats.setCollectionId(((AggregatedFeedProcessorStatisticsHolderV2) holder).getCollectionId());
                    }
                    String processorName = provenanceEventFeedUtil.getProcessorName(processorStats.getProcessorId());
                    if (processorName == null) {
                        processorName = processorStats.getProcessorName();
                    }
                    nifiFeedProcessorStats.setProcessorName(processorName);
                    nifiFeedProcessorStats.setFeedProcessGroupId(feedProcessGroupId);
                    nifiFeedProcessorStatsList.add(nifiFeedProcessorStats);
                });
            });
        } else {
            unregisteredEvents.add(feedProcessorStats);
        }
    });
    return nifiFeedProcessorStatsList;
}
Also used : ArrayList(java.util.ArrayList) NifiFeedProcessorStats(com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedProcessorStats) JpaNifiFeedProcessorStats(com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.JpaNifiFeedProcessorStats) AggregatedFeedProcessorStatisticsHolderV2(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolderV2)

Aggregations

NifiFeedProcessorStats (com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedProcessorStats)3 JpaNifiFeedProcessorStats (com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.JpaNifiFeedProcessorStats)3 AggregatedFeedProcessorStatisticsHolderV2 (com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsHolderV2)3 ArrayList (java.util.ArrayList)3 AggregatedFeedProcessorStatistics (com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatistics)2 JmsListener (org.springframework.jms.annotation.JmsListener)2 CacheBuilder (com.google.common.cache.CacheBuilder)1 CacheLoader (com.google.common.cache.CacheLoader)1 LoadingCache (com.google.common.cache.LoadingCache)1 EvictingQueue (com.google.common.collect.EvictingQueue)1 ClusterMessage (com.thinkbiganalytics.cluster.ClusterMessage)1 ClusterService (com.thinkbiganalytics.cluster.ClusterService)1 ClusterServiceMessageReceiver (com.thinkbiganalytics.cluster.ClusterServiceMessageReceiver)1 JmsConstants (com.thinkbiganalytics.jms.JmsConstants)1 Queues (com.thinkbiganalytics.jms.Queues)1 MetadataAccess (com.thinkbiganalytics.metadata.api.MetadataAccess)1 OpsManagerFeed (com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed)1 BatchJobExecutionProvider (com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecutionProvider)1 NifiFeedProcessorErrors (com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedProcessorErrors)1 NifiFeedProcessorStatisticsProvider (com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedProcessorStatisticsProvider)1