Examples with NifiFeedProcessorStats com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedProcessorStats used on opensource projects

Example 1 with NifiFeedProcessorStats

use of com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedProcessorStats in project kylo by Teradata.

the class NifiStatsJmsReceiver method saveFeedStats.

/**
 * Save the running totals for the feed
 */
private Map<String, JpaNifiFeedStats> saveFeedStats(AggregatedFeedProcessorStatisticsHolderV2 holder, List<NifiFeedProcessorStats> summaryStats) {
    Map<String, JpaNifiFeedStats> feedStatsMap = new HashMap<>();
    if (summaryStats != null) {
        Map<String, Long> feedLatestTimestamp = summaryStats.stream().collect(Collectors.toMap(NifiFeedProcessorStats::getFeedName, stats -> stats.getMinEventTime().getMillis(), Long::max));
        feedLatestTimestamp.entrySet().stream().forEach(e -> {
            String feedName = e.getKey();
            Long timestamp = e.getValue();
            JpaNifiFeedStats stats = feedStatsMap.computeIfAbsent(feedName, name -> new JpaNifiFeedStats(feedName));
            OpsManagerFeed opsManagerFeed = provenanceEventFeedUtil.getFeed(feedName);
            if (opsManagerFeed != null) {
                stats.setFeedId(new JpaNifiFeedStats.OpsManagerFeedId(opsManagerFeed.getId().toString()));
            }
            stats.setLastActivityTimestamp(timestamp);
        });
    }
    if (holder.getProcessorIdRunningFlows() != null) {
        holder.getProcessorIdRunningFlows().entrySet().stream().forEach(e -> {
            String feedProcessorId = e.getKey();
            Long runningCount = e.getValue();
            // ensure not null
            String feedName = provenanceEventFeedUtil.getFeedName(feedProcessorId);
            if (StringUtils.isNotBlank(feedName)) {
                JpaNifiFeedStats stats = feedStatsMap.computeIfAbsent(feedName, name -> new JpaNifiFeedStats(feedName));
                OpsManagerFeed opsManagerFeed = provenanceEventFeedUtil.getFeed(feedName);
                if (opsManagerFeed != null) {
                    stats.setFeedId(new JpaNifiFeedStats.OpsManagerFeedId(opsManagerFeed.getId().toString()));
                    stats.setStream(opsManagerFeed.isStream());
                }
                stats.addRunningFeedFlows(runningCount);
                if (holder instanceof AggregatedFeedProcessorStatisticsHolderV3) {
                    stats.setTime(((AggregatedFeedProcessorStatisticsHolderV3) holder).getTimestamp());
                    if (stats.getLastActivityTimestamp() == null) {
                        stats.setLastActivityTimestamp(((AggregatedFeedProcessorStatisticsHolderV3) holder).getTimestamp());
                    }
                } else {
                    stats.setTime(DateTime.now().getMillis());
                }
                if (stats.getLastActivityTimestamp() == null) {
                    log.warn("The JpaNifiFeedStats.lastActivityTimestamp for the feed {} is NULL.  The JMS Class was: {}", feedName, holder.getClass().getSimpleName());
                }
            }
        });
    }
    // group stats to save together by feed name
    if (!feedStatsMap.isEmpty()) {
        // only save those that have changed
        List<NifiFeedStats> updatedStats = feedStatsMap.entrySet().stream().map(e -> e.getValue()).collect(Collectors.toList());
        // if the running flows are 0 and its streaming we should try back to see if this feed is running or not
        updatedStats.stream().filter(s -> s.isStream()).forEach(stats -> {
            latestStatsCache.put(stats.getFeedName(), (JpaNifiFeedStats) stats);
            if (stats.getRunningFeedFlows() == 0L) {
                batchJobExecutionProvider.markStreamingFeedAsStopped(stats.getFeedName());
            } else {
                batchJobExecutionProvider.markStreamingFeedAsStarted(stats.getFeedName());
            }
        });
        nifiFeedStatisticsProvider.saveLatestFeedStats(updatedStats);
    }
    return feedStatsMap;
}

Example 2 with NifiFeedProcessorStats

use of com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedProcessorStats in project kylo by Teradata.

the class NifiStatsJmsReceiver method toSummaryStats.

private NifiFeedProcessorStats toSummaryStats(GroupedStats groupedStats) {
    NifiFeedProcessorStats nifiFeedProcessorStats = new JpaNifiFeedProcessorStats();
    nifiFeedProcessorStats.setTotalCount(groupedStats.getTotalCount());
    nifiFeedProcessorStats.setFlowFilesFinished(groupedStats.getFlowFilesFinished());
    nifiFeedProcessorStats.setFlowFilesStarted(groupedStats.getFlowFilesStarted());
    nifiFeedProcessorStats.setCollectionId(groupedStats.getGroupKey());
    nifiFeedProcessorStats.setBytesIn(groupedStats.getBytesIn());
    nifiFeedProcessorStats.setBytesOut(groupedStats.getBytesOut());
    nifiFeedProcessorStats.setDuration(groupedStats.getDuration());
    nifiFeedProcessorStats.setJobsFinished(groupedStats.getJobsFinished());
    nifiFeedProcessorStats.setJobsStarted(groupedStats.getJobsStarted());
    nifiFeedProcessorStats.setProcessorsFailed(groupedStats.getProcessorsFailed());
    nifiFeedProcessorStats.setCollectionTime(new DateTime(groupedStats.getTime()));
    nifiFeedProcessorStats.setMinEventTime(new DateTime(groupedStats.getMinTime()));
    nifiFeedProcessorStats.setMinEventTimeMillis(nifiFeedProcessorStats.getMinEventTime().getMillis());
    nifiFeedProcessorStats.setMaxEventTime(new DateTime(groupedStats.getMaxTime()));
    nifiFeedProcessorStats.setJobsFailed(groupedStats.getJobsFailed());
    nifiFeedProcessorStats.setSuccessfulJobDuration(groupedStats.getSuccessfulJobDuration());
    nifiFeedProcessorStats.setJobDuration(groupedStats.getJobDuration());
    nifiFeedProcessorStats.setMaxEventId(groupedStats.getMaxEventId());
    nifiFeedProcessorStats.setFailedCount(groupedStats.getProcessorsFailed());
    if (groupedStats instanceof GroupedStatsV2) {
        nifiFeedProcessorStats.setLatestFlowFileId(((GroupedStatsV2) groupedStats).getLatestFlowFileId());
    }
    if (provenanceEventFeedUtil.isFailure(groupedStats.getSourceConnectionIdentifier())) {
        nifiFeedProcessorStats.setFailedCount(groupedStats.getTotalCount() + groupedStats.getProcessorsFailed());
    }
    return nifiFeedProcessorStats;
}

Example 3 with NifiFeedProcessorStats

use of com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedProcessorStats in project kylo by Teradata.

the class NifiStatsJmsReceiver method receiveTopic.

@JmsListener(id = JMS_LISTENER_ID, destination = Queues.PROVENANCE_EVENT_STATS_QUEUE, containerFactory = JmsConstants.QUEUE_LISTENER_CONTAINER_FACTORY)
public void receiveTopic(AggregatedFeedProcessorStatisticsHolder stats) {
    if (readyToProcess(stats)) {
        if (ensureValidRetryAttempt(stats)) {
            final List<AggregatedFeedProcessorStatistics> unregisteredEvents = new ArrayList<>();
            metadataAccess.commit(() -> {
                List<NifiFeedProcessorStats> summaryStats = createSummaryStats(stats, unregisteredEvents);
                List<JpaNifiFeedProcessorStats> failedStatsWithFlowFiles = new ArrayList<>();
                for (NifiFeedProcessorStats stat : summaryStats) {
                    NifiFeedProcessorStats savedStats = nifiEventStatisticsProvider.create(stat);
                    if (savedStats.getFailedCount() > 0L && savedStats.getLatestFlowFileId() != null) {
                        // offload the query to nifi and merge back in
                        failedStatsWithFlowFiles.add((JpaNifiFeedProcessorStats) savedStats);
                    }
                    ensureStreamingJobExecutionRecord(stat);
                }
                if (stats instanceof AggregatedFeedProcessorStatisticsHolderV2) {
                    saveFeedStats((AggregatedFeedProcessorStatisticsHolderV2) stats, summaryStats);
                }
                if (!failedStatsWithFlowFiles.isEmpty()) {
                    assignNiFiBulletinErrors(failedStatsWithFlowFiles);
                }
                return summaryStats;
            }, MetadataAccess.SERVICE);
            if (clusterService.isClustered() && !unregisteredEvents.isEmpty()) {
                // reprocess with delay
                if (retryProvenanceEventWithDelay != null) {
                    retryProvenanceEventWithDelay.delay(stats, unregisteredEvents);
                }
            }
        } else {
            // stop processing the events
            log.info("Unable find the feed in Ops Manager.  Not processing {} stats ", stats.getFeedStatistics().values().size());
        }
    } else {
        log.info("NiFi is not up yet.  Sending back to JMS for later dequeue ");
        throw new JmsProcessingException("Unable to process Statistics Events.  NiFi is either not up, or there is an error trying to populate the Kylo NiFi Flow Cache. ");
    }
}

Example 4 with NifiFeedProcessorStats

use of com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedProcessorStats in project kylo by Teradata.

the class FeedFailureService method findLatestJob.

/**
 * Find the latest Job, first looking for any failures between the last time this service ran, and now.
 *
 * @param feedName feed to check
 */
public LastFeedJob findLatestJob(String feedName) {
    LastFeedJob lastFeedJob = metadataAccess.read(() -> {
        LastFeedJob lastAssessedJob = lastAssessedFeedMap.getOrDefault(feedName, newEmptyFeedJob(DateTime.now()));
        LOG.debug("Feed failure service check.  LastAssessJob from map is {}", lastAssessedJob);
        DateTime lastAssessedTime = lastAssessedJob.getDateTime();
        if (isEmptyJob(lastAssessedJob)) {
            // attempt to get jobs since the app started
            lastAssessedTime = servicesApplicationStartupListener.getStartTime();
            lastAssessedJob.setDateTime(lastAssessedTime);
        }
        OpsManagerFeed feed = feedProvider.findByName(feedName);
        if (feed == null) {
            LOG.error("Feed Failure Service check Error!!!  Unable to find feed for:  {}", feedName);
            return newEmptyFeedJob(DateTime.now());
        }
        if (feed.isStream()) {
            List<NifiFeedProcessorStats> latestStats = nifiFeedProcessorStatisticsProvider.findLatestFinishedStatsSince(feedName, lastAssessedTime);
            LOG.debug("Streaming Feed failure check for {}.  Found {} stats", feedName, latestStats.size());
            Optional<NifiFeedProcessorStats> total = latestStats.stream().reduce((a, b) -> {
                a.setFailedCount(a.getFailedCount() + b.getFailedCount());
                if (b.getMinEventTime().isAfter(a.getMinEventTime())) {
                    a.setMinEventTime(b.getMinEventTime());
                }
                return a;
            });
            LastFeedJob lastJob = null;
            if (total.isPresent()) {
                NifiFeedProcessorStats stats = total.get();
                boolean success = stats.getFailedCount() == 0;
                lastJob = new LastFeedJob(feedName, stats.getMinEventTime(), success);
            } else {
                lastJob = new LastFeedJob(feedName, lastAssessedTime, true);
            }
            LOG.debug("{} stats for feed. Streaming Feed failure returning {}", total.isPresent() ? "Found" : "Did not find any", lastJob);
            return lastJob;
        } else {
            List<? extends BatchJobExecution> latestJobs = batchJobExecutionProvider.findLatestFinishedJobForFeedSince(feedName, lastAssessedTime);
            LOG.debug("Batch Feed failure check for {}.  Found {} jobs", feedName, latestJobs != null ? latestJobs.size() : 0);
            BatchJobExecution latestJob = latestJobs.stream().sorted(Comparator.comparing(BatchJobExecution::getEndTime).reversed()).filter(job -> FAILED.equals(job.getStatus())).findFirst().orElse(null);
            if (latestJob == null) {
                // find the last job if there are no failures
                latestJob = latestJobs.stream().sorted(Comparator.comparing(BatchJobExecution::getEndTime).reversed()).findFirst().orElse(null);
                // if the set doesnt have anything attempt to get the latest job
                if (latestJob == null) {
                    latestJob = batchJobExecutionProvider.findLatestFinishedJobForFeed(feedName);
                }
            }
            LastFeedJob lastJob = latestJob != null ? new LastFeedJob(feedName, latestJob.getEndTime(), !FAILED.equals(latestJob.getStatus()), latestJob.getJobExecutionId()) : newEmptyFeedJob(lastAssessedTime);
            LOG.debug("Batch Feed failure check returning {} for feed {}", lastJob, feedName);
            return lastJob;
        }
    }, MetadataAccess.SERVICE);
    lastAssessedFeedMap.put(feedName, lastFeedJob);
    return lastFeedJob;
}

Example 5 with NifiFeedProcessorStats

use of com.thinkbiganalytics.metadata.api.jobrepo.nifi.NifiFeedProcessorStats in project kylo by Teradata.

the class FeedFailureMetricAssessorTest method testBatchSuccess.

/**
 * Test batch jobs that come in as succes, success
 */
@Test
public void testBatchSuccess() throws ParseException {
    DateTime lastAssessedTime = DateTime.now();
    String feedName = "feed";
    List<NifiFeedProcessorStats> streamingStats = new ArrayList<>();
    boolean isStream = false;
    this.metric.setFeedName(feedName);
    when(feedProvider.findByName(feedName)).thenReturn(newOpsManagerFeed(feedName, isStream));
    // completed, failed, completed.
    // should fail
    DateTime startingEvent = DateTime.now().minusMinutes(5);
    List<? extends BatchJobExecution> batchJobs = new ArrayList<>();
    batchJobs.add(newBatchJob(feedName, startingEvent, false));
    batchJobs.add(newBatchJob(feedName, startingEvent.plusMinutes(2), false));
    Mockito.when(this.jobExecutionProvider.findLatestFinishedJobForFeedSince(Mockito.anyString(), Mockito.any(DateTime.class))).thenAnswer(x -> batchJobs);
    this.assessor.assess(metric, this.builder);
    verify(this.builder).result(AssessmentResult.SUCCESS);
}