Examples with GroupedStatsV2 com.thinkbiganalytics.nifi.provenance.model.stats.GroupedStatsV2 used on opensource projects

Example 1 with GroupedStatsV2

use of com.thinkbiganalytics.nifi.provenance.model.stats.GroupedStatsV2 in project kylo by Teradata.

the class NifiStatsJmsReceiver method toSummaryStats.

private NifiFeedProcessorStats toSummaryStats(GroupedStats groupedStats) {
    NifiFeedProcessorStats nifiFeedProcessorStats = new JpaNifiFeedProcessorStats();
    nifiFeedProcessorStats.setTotalCount(groupedStats.getTotalCount());
    nifiFeedProcessorStats.setFlowFilesFinished(groupedStats.getFlowFilesFinished());
    nifiFeedProcessorStats.setFlowFilesStarted(groupedStats.getFlowFilesStarted());
    nifiFeedProcessorStats.setCollectionId(groupedStats.getGroupKey());
    nifiFeedProcessorStats.setBytesIn(groupedStats.getBytesIn());
    nifiFeedProcessorStats.setBytesOut(groupedStats.getBytesOut());
    nifiFeedProcessorStats.setDuration(groupedStats.getDuration());
    nifiFeedProcessorStats.setJobsFinished(groupedStats.getJobsFinished());
    nifiFeedProcessorStats.setJobsStarted(groupedStats.getJobsStarted());
    nifiFeedProcessorStats.setProcessorsFailed(groupedStats.getProcessorsFailed());
    nifiFeedProcessorStats.setCollectionTime(new DateTime(groupedStats.getTime()));
    nifiFeedProcessorStats.setMinEventTime(new DateTime(groupedStats.getMinTime()));
    nifiFeedProcessorStats.setMinEventTimeMillis(nifiFeedProcessorStats.getMinEventTime().getMillis());
    nifiFeedProcessorStats.setMaxEventTime(new DateTime(groupedStats.getMaxTime()));
    nifiFeedProcessorStats.setJobsFailed(groupedStats.getJobsFailed());
    nifiFeedProcessorStats.setSuccessfulJobDuration(groupedStats.getSuccessfulJobDuration());
    nifiFeedProcessorStats.setJobDuration(groupedStats.getJobDuration());
    nifiFeedProcessorStats.setMaxEventId(groupedStats.getMaxEventId());
    nifiFeedProcessorStats.setFailedCount(groupedStats.getProcessorsFailed());
    if (groupedStats instanceof GroupedStatsV2) {
        nifiFeedProcessorStats.setLatestFlowFileId(((GroupedStatsV2) groupedStats).getLatestFlowFileId());
    }
    if (provenanceEventFeedUtil.isFailure(groupedStats.getSourceConnectionIdentifier())) {
        nifiFeedProcessorStats.setFailedCount(groupedStats.getTotalCount() + groupedStats.getProcessorsFailed());
    }
    return nifiFeedProcessorStats;
}

Example 2 with GroupedStatsV2

use of com.thinkbiganalytics.nifi.provenance.model.stats.GroupedStatsV2 in project kylo by Teradata.

the class GroupedStatsUtil method gatherStats.

/**
 * Gather feed stats for a list of events
 */
public static AggregatedFeedProcessorStatisticsHolder gatherStats(final List<ProvenanceEventRecordDTO> events) {
    Map<String, Map<GroupedStatsIdentity, List<GroupedStats>>> feedStatsByProcessor = new ConcurrentHashMap<>();
    // events.stream().forEach(e -> {
    for (ProvenanceEventRecordDTO e : events) {
        if (!feedStatsByProcessor.containsKey(e.getFeedName())) {
            feedStatsByProcessor.put(e.getFeedName(), new ConcurrentHashMap<GroupedStatsIdentity, List<GroupedStats>>());
        }
        // feedStatsByProcessor.putIfAbsent(e.getFeedName(), );
        Map<GroupedStatsIdentity, List<GroupedStats>> feedStats = feedStatsByProcessor.get(e.getFeedName());
        GroupedStatsIdentity identity = new GroupedStatsIdentity(e.getComponentId(), e.getComponentName());
        if (!feedStats.containsKey(identity)) {
            feedStats.put(identity, new ArrayList<GroupedStats>());
        }
        // feedStats.putIfAbsent(identity, new ArrayList<>());
        List<GroupedStats> feedProcessorStats = feedStats.get(identity);
        // Add the new stats
        GroupedStats statsV2 = GroupedStatsUtil.add(new GroupedStatsV2(), e);
        feedProcessorStats.add(statsV2);
    }
    // );
    List<AggregatedFeedProcessorStatistics> statsList = new ArrayList<>();
    for (Map.Entry<String, Map<GroupedStatsIdentity, List<GroupedStats>>> feedStats : feedStatsByProcessor.entrySet()) {
        AggregatedFeedProcessorStatistics feedProcessorStatistics = GroupedStatsUtil.groupStatsByProcessor(feedStats.getKey(), feedStats.getValue());
        statsList.add(feedProcessorStatistics);
    }
    /* feedStatsByProcessor.entrySet().stream().forEach(feedStats -> {
            AggregatedFeedProcessorStatistics feedProcessorStatistics = GroupedStatsUtil.groupStatsByProcessor(feedStats.getKey(), feedStats.getValue());
            statsList.add(feedProcessorStatistics);
        });
        */
    AggregatedFeedProcessorStatisticsHolderV3 feedProcessorStatisticsHolderV3 = new AggregatedFeedProcessorStatisticsHolderV3();
    feedProcessorStatisticsHolderV3.setFeedStatistics(statsList);
    return feedProcessorStatisticsHolderV3;
}