Search in sources :

Example 1 with AggregatedProcessorStatistics

use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedProcessorStatistics in project kylo by Teradata.

the class FeedStatisticsManager method gatherStatistics.

public void gatherStatistics() {
    lock.lock();
    List<ProvenanceEventRecordDTO> eventsToSend = null;
    Map<String, AggregatedFeedProcessorStatistics> statsToSend = null;
    try {
        // Gather Events and Stats to send Ops Manager
        // filter out the streaming feeds
        ensureStreamingFeedMetadata();
        eventsToSend = feedStatisticsMap.values().stream().flatMap(stats -> stats.getEventsToSend().stream().filter(event -> !FeedEventStatistics.getInstance().streamingFeedProcessorIdsList.contains(event.getFirstEventProcessorId()))).sorted(Comparator.comparing(ProvenanceEventRecordDTO::getEventTime).thenComparing(ProvenanceEventRecordDTO::getEventId)).collect(Collectors.toList());
        final String collectionId = UUID.randomUUID().toString();
        Map<String, Long> runningFlowsCount = new HashMap<>();
        for (FeedStatistics feedStatistics : feedStatisticsMap.values()) {
            if (feedStatistics.hasStats()) {
                if (statsToSend == null) {
                    statsToSend = new ConcurrentHashMap<>();
                }
                AggregatedFeedProcessorStatistics feedProcessorStatistics = statsToSend.computeIfAbsent(feedStatistics.getFeedProcessorId(), feedProcessorId -> new AggregatedFeedProcessorStatistics(feedStatistics.getFeedProcessorId(), collectionId, sendJmsTimeMillis));
                AggregatedProcessorStatistics processorStatistics = feedProcessorStatistics.getProcessorStats().computeIfAbsent(feedStatistics.getProcessorId(), processorId -> new AggregatedProcessorStatisticsV2(feedStatistics.getProcessorId(), null, collectionId));
                // accumulate the stats together into the processorStatistics object grouped by source connection id
                feedStatistics.getStats().stream().forEach(stats -> {
                    FeedProcessorStatisticsAggregator.getInstance().addStats1(processorStatistics.getStats(stats.getSourceConnectionIdentifier()), stats);
                });
            }
        }
        if ((eventsToSend != null && !eventsToSend.isEmpty()) || (statsToSend != null && !statsToSend.isEmpty())) {
            // send it off to jms on a different thread
            JmsSender jmsSender = new JmsSender(eventsToSend, statsToSend.values(), FeedEventStatistics.getInstance().getRunningFeedFlowsForFeed(statsToSend.keySet()));
            this.jmsService.submit(new JmsSenderConsumer(jmsSender));
        } else {
            // if we are empty but the runningFlows have changed, then send off as well
            if (FeedEventStatistics.getInstance().isFeedProcessorRunningFeedFlowsChanged()) {
                JmsSender jmsSender = new JmsSender(null, null, FeedEventStatistics.getInstance().getRunningFeedFlowsChanged());
                this.jmsService.submit(new JmsSenderConsumer(jmsSender));
            }
        }
    } finally {
        FeedEventStatistics.getInstance().markFeedProcessorRunningFeedFlowsUnchanged();
        feedStatisticsMap.values().stream().forEach(stats -> stats.clear());
        lock.unlock();
    }
}
Also used : ThreadFactoryBuilder(com.google.common.util.concurrent.ThreadFactoryBuilder) KyloFeedBatchStreamTypeJmsListener(com.thinkbiganalytics.nifi.provenance.jms.KyloFeedBatchStreamTypeJmsListener) ScheduledFuture(java.util.concurrent.ScheduledFuture) LoggerFactory(org.slf4j.LoggerFactory) AggregatedFeedProcessorStatistics(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatistics) HashMap(java.util.HashMap) ProvenanceEventRecord(org.apache.nifi.provenance.ProvenanceEventRecord) Map(java.util.Map) AggregatedProcessorStatistics(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedProcessorStatistics) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) AggregatedProcessorStatisticsV2(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedProcessorStatisticsV2) ThreadFactory(java.util.concurrent.ThreadFactory) ProvenanceEventRecordDTO(com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO) ExecutorService(java.util.concurrent.ExecutorService) Logger(org.slf4j.Logger) ReentrantLock(java.util.concurrent.locks.ReentrantLock) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) UUID(java.util.UUID) SpringApplicationContext(com.thinkbiganalytics.nifi.provenance.util.SpringApplicationContext) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) Lock(java.util.concurrent.locks.Lock) ProvenanceEventUtil(com.thinkbiganalytics.nifi.provenance.util.ProvenanceEventUtil) Comparator(java.util.Comparator) ProvenanceEventRecordDTO(com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO) HashMap(java.util.HashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) AggregatedFeedProcessorStatistics(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatistics) AggregatedProcessorStatistics(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedProcessorStatistics) AggregatedProcessorStatisticsV2(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedProcessorStatisticsV2)

Example 2 with AggregatedProcessorStatistics

use of com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedProcessorStatistics in project kylo by Teradata.

the class GroupedStatsUtil method groupStatsByProcessor.

/**
 * Group stats together by processor for a given feed
 *
 * @param feedName     feed name
 * @param groupedStats Map of processorIdentity to list of stats for that processor
 */
public static AggregatedFeedProcessorStatistics groupStatsByProcessor(String feedName, Map<GroupedStatsIdentity, List<GroupedStats>> groupedStats) {
    Long sendJmsTimeMillis = 3000L;
    String collectionId = UUID.randomUUID().toString();
    // Create a random id for the starting processor.
    // the starting feed processor id is not needed as we already know the feed name associated with these stats
    String startingProcessorId = UUID.randomUUID().toString();
    AggregatedFeedProcessorStatisticsV2 feedProcessorStatistics = new AggregatedFeedProcessorStatisticsV2(startingProcessorId, collectionId, sendJmsTimeMillis);
    feedProcessorStatistics.setFeedName(feedName);
    for (Map.Entry<GroupedStatsIdentity, List<GroupedStats>> stats : groupedStats.entrySet()) {
        String processorName = stats.getKey().getProcessorName();
        String processorId = stats.getKey().getProcessorId();
        Map<String, AggregatedProcessorStatistics> processorStatsMap = feedProcessorStatistics.getProcessorStats();
        if (!processorStatsMap.containsKey(processorName)) {
            processorStatsMap.put(processorName, new AggregatedProcessorStatisticsV2(processorId, processorName, collectionId));
        }
        AggregatedProcessorStatistics processorStatistics = processorStatsMap.get(processorName);
        for (GroupedStats s : stats.getValue()) {
            GroupedStatsUtil.addStats1(processorStatistics.getStats(GroupedStats.DEFAULT_SOURCE_CONNECTION_ID), s);
        }
    /*
            AggregatedProcessorStatistics
                processorStatistics =
                feedProcessorStatistics.getProcessorStats()
                    .computeIfAbsent(processorName, k -> new AggregatedProcessorStatisticsV2(processorId, k, collectionId));


            stats.getValue().stream().forEach( s ->  GroupedStatsUtil.add(processorStatistics.getStats(GroupedStats.DEFAULT_SOURCE_CONNECTION_ID), s));
            */
    }
    return feedProcessorStatistics;
}
Also used : GroupedStats(com.thinkbiganalytics.nifi.provenance.model.stats.GroupedStats) ArrayList(java.util.ArrayList) List(java.util.List) AggregatedProcessorStatistics(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedProcessorStatistics) GroupedStatsIdentity(com.thinkbiganalytics.nifi.provenance.model.stats.GroupedStatsIdentity) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Map(java.util.Map) AggregatedFeedProcessorStatisticsV2(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsV2) AggregatedProcessorStatisticsV2(com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedProcessorStatisticsV2)

Aggregations

AggregatedProcessorStatistics (com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedProcessorStatistics)2 AggregatedProcessorStatisticsV2 (com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedProcessorStatisticsV2)2 List (java.util.List)2 Map (java.util.Map)2 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 ThreadFactoryBuilder (com.google.common.util.concurrent.ThreadFactoryBuilder)1 KyloFeedBatchStreamTypeJmsListener (com.thinkbiganalytics.nifi.provenance.jms.KyloFeedBatchStreamTypeJmsListener)1 ProvenanceEventRecordDTO (com.thinkbiganalytics.nifi.provenance.model.ProvenanceEventRecordDTO)1 AggregatedFeedProcessorStatistics (com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatistics)1 AggregatedFeedProcessorStatisticsV2 (com.thinkbiganalytics.nifi.provenance.model.stats.AggregatedFeedProcessorStatisticsV2)1 GroupedStats (com.thinkbiganalytics.nifi.provenance.model.stats.GroupedStats)1 GroupedStatsIdentity (com.thinkbiganalytics.nifi.provenance.model.stats.GroupedStatsIdentity)1 ProvenanceEventUtil (com.thinkbiganalytics.nifi.provenance.util.ProvenanceEventUtil)1 SpringApplicationContext (com.thinkbiganalytics.nifi.provenance.util.SpringApplicationContext)1 ArrayList (java.util.ArrayList)1 Comparator (java.util.Comparator)1 HashMap (java.util.HashMap)1 UUID (java.util.UUID)1 ExecutorService (java.util.concurrent.ExecutorService)1 Executors (java.util.concurrent.Executors)1