Examples with GroupedStatsIdentity com.thinkbiganalytics.nifi.provenance.model.stats.GroupedStatsIdentity used on opensource projects

Example 1 with GroupedStatsIdentity

use of com.thinkbiganalytics.nifi.provenance.model.stats.GroupedStatsIdentity in project kylo by Teradata.

the class GroupedStatsUtil method gatherStats.

/**
 * Gather feed stats for a list of events
 */
public static AggregatedFeedProcessorStatisticsHolder gatherStats(final List<ProvenanceEventRecordDTO> events) {
    Map<String, Map<GroupedStatsIdentity, List<GroupedStats>>> feedStatsByProcessor = new ConcurrentHashMap<>();
    // events.stream().forEach(e -> {
    for (ProvenanceEventRecordDTO e : events) {
        if (!feedStatsByProcessor.containsKey(e.getFeedName())) {
            feedStatsByProcessor.put(e.getFeedName(), new ConcurrentHashMap<GroupedStatsIdentity, List<GroupedStats>>());
        }
        // feedStatsByProcessor.putIfAbsent(e.getFeedName(), );
        Map<GroupedStatsIdentity, List<GroupedStats>> feedStats = feedStatsByProcessor.get(e.getFeedName());
        GroupedStatsIdentity identity = new GroupedStatsIdentity(e.getComponentId(), e.getComponentName());
        if (!feedStats.containsKey(identity)) {
            feedStats.put(identity, new ArrayList<GroupedStats>());
        }
        // feedStats.putIfAbsent(identity, new ArrayList<>());
        List<GroupedStats> feedProcessorStats = feedStats.get(identity);
        // Add the new stats
        GroupedStats statsV2 = GroupedStatsUtil.add(new GroupedStatsV2(), e);
        feedProcessorStats.add(statsV2);
    }
    // );
    List<AggregatedFeedProcessorStatistics> statsList = new ArrayList<>();
    for (Map.Entry<String, Map<GroupedStatsIdentity, List<GroupedStats>>> feedStats : feedStatsByProcessor.entrySet()) {
        AggregatedFeedProcessorStatistics feedProcessorStatistics = GroupedStatsUtil.groupStatsByProcessor(feedStats.getKey(), feedStats.getValue());
        statsList.add(feedProcessorStatistics);
    }
    /* feedStatsByProcessor.entrySet().stream().forEach(feedStats -> {
            AggregatedFeedProcessorStatistics feedProcessorStatistics = GroupedStatsUtil.groupStatsByProcessor(feedStats.getKey(), feedStats.getValue());
            statsList.add(feedProcessorStatistics);
        });
        */
    AggregatedFeedProcessorStatisticsHolderV3 feedProcessorStatisticsHolderV3 = new AggregatedFeedProcessorStatisticsHolderV3();
    feedProcessorStatisticsHolderV3.setFeedStatistics(statsList);
    return feedProcessorStatisticsHolderV3;
}

Example 2 with GroupedStatsIdentity

use of com.thinkbiganalytics.nifi.provenance.model.stats.GroupedStatsIdentity in project kylo by Teradata.

the class GroupedStatsUtil method groupStatsByProcessor.

/**
 * Group stats together by processor for a given feed
 *
 * @param feedName     feed name
 * @param groupedStats Map of processorIdentity to list of stats for that processor
 */
public static AggregatedFeedProcessorStatistics groupStatsByProcessor(String feedName, Map<GroupedStatsIdentity, List<GroupedStats>> groupedStats) {
    Long sendJmsTimeMillis = 3000L;
    String collectionId = UUID.randomUUID().toString();
    // Create a random id for the starting processor.
    // the starting feed processor id is not needed as we already know the feed name associated with these stats
    String startingProcessorId = UUID.randomUUID().toString();
    AggregatedFeedProcessorStatisticsV2 feedProcessorStatistics = new AggregatedFeedProcessorStatisticsV2(startingProcessorId, collectionId, sendJmsTimeMillis);
    feedProcessorStatistics.setFeedName(feedName);
    for (Map.Entry<GroupedStatsIdentity, List<GroupedStats>> stats : groupedStats.entrySet()) {
        String processorName = stats.getKey().getProcessorName();
        String processorId = stats.getKey().getProcessorId();
        Map<String, AggregatedProcessorStatistics> processorStatsMap = feedProcessorStatistics.getProcessorStats();
        if (!processorStatsMap.containsKey(processorName)) {
            processorStatsMap.put(processorName, new AggregatedProcessorStatisticsV2(processorId, processorName, collectionId));
        }
        AggregatedProcessorStatistics processorStatistics = processorStatsMap.get(processorName);
        for (GroupedStats s : stats.getValue()) {
            GroupedStatsUtil.addStats1(processorStatistics.getStats(GroupedStats.DEFAULT_SOURCE_CONNECTION_ID), s);
        }
    /*
            AggregatedProcessorStatistics
                processorStatistics =
                feedProcessorStatistics.getProcessorStats()
                    .computeIfAbsent(processorName, k -> new AggregatedProcessorStatisticsV2(processorId, k, collectionId));


            stats.getValue().stream().forEach( s ->  GroupedStatsUtil.add(processorStatistics.getStats(GroupedStats.DEFAULT_SOURCE_CONNECTION_ID), s));
            */
    }
    return feedProcessorStatistics;
}