Examples with OpsManagerFeed com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed used on opensource projects

Example 11 with OpsManagerFeed

use of com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed in project kylo by Teradata.

the class ProvenanceEventFeedUtil method enrichEventWithFeedInformation.

public ProvenanceEventRecordDTO enrichEventWithFeedInformation(ProvenanceEventRecordDTO event) {
    String feedName = null;
    if (StringUtils.isNotBlank(event.getFirstEventProcessorId())) {
        feedName = getFeedName(event.getFirstEventProcessorId());
    }
    if (StringUtils.isBlank(feedName) && StringUtils.isNotBlank(event.getFeedName())) {
        feedName = event.getFeedName();
    }
    // if we cant get the feed name check to see if the NiFi flow cache is updated... and wait for it to be updated before processing
    if (StringUtils.isBlank(feedName) && needsUpdateFromCluster() && StringUtils.isNotBlank(event.getFirstEventProcessorId())) {
        log.info("Unable to find the feed for processorId: {}.  Changes were detected from the cluster.  Refreshing the cache ...", event.getFirstEventProcessorId());
        nifiFlowCache.applyClusterUpdates();
        feedName = getFeedName(event.getFirstEventProcessorId());
        if (StringUtils.isNotBlank(feedName)) {
            log.info("Cache Refreshed.  Found the feed: {} ", feedName);
        } else {
            log.info("Cache Refreshed, but still unable to find the feed.  This event {} will not be processed ", event);
        }
    }
    // strip kylo versioning in the feed name in case it exists
    feedName = NifiTemplateNameUtil.parseVersionedProcessGroupName(feedName);
    String processGroupId = getFeedProcessGroupId(event.getFirstEventProcessorId());
    if (StringUtils.isBlank(processGroupId)) {
        processGroupId = event.getFeedProcessGroupId();
    }
    String processorName = getProcessorName(event.getComponentId());
    if (StringUtils.isBlank(processorName)) {
        processorName = event.getComponentName();
    }
    event.setFeedName(feedName);
    event.setFeedProcessGroupId(processGroupId);
    event.setComponentName(processorName);
    setProcessorFlowType(event);
    if (StringUtils.isNotBlank(feedName)) {
        OpsManagerFeed feed = opsManagerFeedProvider.findByNameWithoutAcl(feedName);
        if (feed != null && !OpsManagerFeed.NULL_FEED.equals(feed)) {
            event.setStream(feed.isStream());
        }
    }
    return event;
}

Example 12 with OpsManagerFeed

use of com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed in project kylo by Teradata.

the class ProvenanceEventReceiver method processEvent.

/**
 * process the event and persist it along with creating the Job and Step.  If there is a lock error it will retry until it hits the {@link #lockAcquisitionRetryAmount}
 *
 * @param event        a provenance event
 * @param retryAttempt the retry number.  If there is a lock error it will retry until it hits the {@link #lockAcquisitionRetryAmount}
 */
private void processEvent(ProvenanceEventRecordDTO event, int retryAttempt) {
    try {
        OpsManagerFeed feed = provenanceEventFeedUtil.getFeed(event);
        log.debug("Process {} for flowfile: {} and processorId: {} ", event, event.getJobFlowFileId(), event.getFirstEventProcessorId());
        // ensure the job is there
        BatchJobExecution jobExecution = findOrCreateJobExecution(event, feed);
        if (jobExecution != null) {
            batchJobExecutionProvider.updateFeedJobStartTime(jobExecution, feed);
        }
        if (jobExecution != null && !event.isStream()) {
            metadataAccess.commit(() -> receiveBatchEvent(jobExecution, event), MetadataAccess.SERVICE);
        }
        if (jobExecution != null && event.isFinalJobEvent()) {
            notifyJobFinished(jobExecution, event);
        }
    } catch (LockAcquisitionException lae) {
        if (retryAttempt < lockAcquisitionRetryAmount) {
            retryAttempt++;
            log.error("LockAcquisitionException found trying to process Event: {} .  Retry attempt # {} ", event, retryAttempt, lae);
            // wait and re attempt
            try {
                Thread.sleep(300L);
            } catch (InterruptedException var10) {
            }
            processEvent(event, retryAttempt);
        } else {
            log.error("LockAcquisitionException found.  Unsuccessful after retrying {} times.  This event {} will not be processed. ", retryAttempt, event, lae);
        }
    } catch (Exception e) {
        log.error("Error processing Event ", event, e);
    }
}

Example 13 with OpsManagerFeed

use of com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed in project kylo by Teradata.

the class ProvenanceEventReceiver method failedJob.

/**
 * Triggered for both Batch and Streaming Feed Jobs when the Job and any related Jobs (as a result of a Merge of other Jobs are complete but have a failure in the flow<br/> Example: <br/> Job
 * (FlowFile) 1,2,3 are all running<br/> Job 1,2,3 get Merged<br/> Job 1,2 finish<br/> Job 3 finishes <br/>
 *
 * This will fire when Job3 finishes indicating this entire flow is complete<br/>
 *
 * @param jobExecution the job
 * @param event        a provenance event
 */
private void failedJob(BatchJobExecution jobExecution, ProvenanceEventRecordDTO event) {
    if (queryForNiFiBulletins && !event.isStream()) {
        queryForNiFiErrorBulletins(event);
    }
    log.debug("Failed JOB for Event {} ", event);
    OpsManagerFeed feed = opsManagerFeedProvider.findByNameWithoutAcl(event.getFeedName());
    if (feed == null) {
        feed = jobExecution.getJobInstance().getFeed();
    }
    batchJobExecutionProvider.notifyFailure(jobExecution, feed, event.isStream(), null);
}

Example 14 with OpsManagerFeed

use of com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed in project kylo by Teradata.

the class ProvenanceEventReceiver method successfulJob.

/**
 * Triggered for both Batch and Streaming Feed Jobs when the Job and any related Jobs (as a result of a Merge of other Jobs are complete<br/> Example: <br/> Job (FlowFile) 1,2,3 are all
 * running<br/> Job 1,2,3 get Merged<br/> Job 1,2 finish<br/> Job 3 finishes <br/>
 *
 * This will fire when Job3 finishes indicating this entire flow is complete<br/>
 *
 * @param jobExecution the job
 * @param event        a provenance event
 */
private void successfulJob(BatchJobExecution jobExecution, ProvenanceEventRecordDTO event) {
    log.debug("Success JOB for Event {} ", event);
    // get teh feed from the cache first
    OpsManagerFeed feed = opsManagerFeedProvider.findByNameWithoutAcl(event.getFeedName());
    if (feed == null) {
        feed = jobExecution.getJobInstance().getFeed();
    }
    batchJobExecutionProvider.notifySuccess(jobExecution, feed, null);
}

Example 15 with OpsManagerFeed

use of com.thinkbiganalytics.metadata.api.feed.OpsManagerFeed in project kylo by Teradata.

the class FeedFailureService method findLatestJob.

/**
 * Find the latest Job, first looking for any failures between the last time this service ran, and now.
 *
 * @param feedName feed to check
 */
public LastFeedJob findLatestJob(String feedName) {
    LastFeedJob lastFeedJob = metadataAccess.read(() -> {
        LastFeedJob lastAssessedJob = lastAssessedFeedMap.getOrDefault(feedName, newEmptyFeedJob(DateTime.now()));
        LOG.debug("Feed failure service check.  LastAssessJob from map is {}", lastAssessedJob);
        DateTime lastAssessedTime = lastAssessedJob.getDateTime();
        if (isEmptyJob(lastAssessedJob)) {
            // attempt to get jobs since the app started
            lastAssessedTime = servicesApplicationStartupListener.getStartTime();
            lastAssessedJob.setDateTime(lastAssessedTime);
        }
        OpsManagerFeed feed = feedProvider.findByName(feedName);
        if (feed == null) {
            LOG.error("Feed Failure Service check Error!!!  Unable to find feed for:  {}", feedName);
            return newEmptyFeedJob(DateTime.now());
        }
        if (feed.isStream()) {
            List<NifiFeedProcessorStats> latestStats = nifiFeedProcessorStatisticsProvider.findLatestFinishedStatsSince(feedName, lastAssessedTime);
            LOG.debug("Streaming Feed failure check for {}.  Found {} stats", feedName, latestStats.size());
            Optional<NifiFeedProcessorStats> total = latestStats.stream().reduce((a, b) -> {
                a.setFailedCount(a.getFailedCount() + b.getFailedCount());
                if (b.getMinEventTime().isAfter(a.getMinEventTime())) {
                    a.setMinEventTime(b.getMinEventTime());
                }
                return a;
            });
            LastFeedJob lastJob = null;
            if (total.isPresent()) {
                NifiFeedProcessorStats stats = total.get();
                boolean success = stats.getFailedCount() == 0;
                lastJob = new LastFeedJob(feedName, stats.getMinEventTime(), success);
            } else {
                lastJob = new LastFeedJob(feedName, lastAssessedTime, true);
            }
            LOG.debug("{} stats for feed. Streaming Feed failure returning {}", total.isPresent() ? "Found" : "Did not find any", lastJob);
            return lastJob;
        } else {
            List<? extends BatchJobExecution> latestJobs = batchJobExecutionProvider.findLatestFinishedJobForFeedSince(feedName, lastAssessedTime);
            LOG.debug("Batch Feed failure check for {}.  Found {} jobs", feedName, latestJobs != null ? latestJobs.size() : 0);
            BatchJobExecution latestJob = latestJobs.stream().sorted(Comparator.comparing(BatchJobExecution::getEndTime).reversed()).filter(job -> FAILED.equals(job.getStatus())).findFirst().orElse(null);
            if (latestJob == null) {
                // find the last job if there are no failures
                latestJob = latestJobs.stream().sorted(Comparator.comparing(BatchJobExecution::getEndTime).reversed()).findFirst().orElse(null);
                // if the set doesnt have anything attempt to get the latest job
                if (latestJob == null) {
                    latestJob = batchJobExecutionProvider.findLatestFinishedJobForFeed(feedName);
                }
            }
            LastFeedJob lastJob = latestJob != null ? new LastFeedJob(feedName, latestJob.getEndTime(), !FAILED.equals(latestJob.getStatus()), latestJob.getJobExecutionId()) : newEmptyFeedJob(lastAssessedTime);
            LOG.debug("Batch Feed failure check returning {} for feed {}", lastJob, feedName);
            return lastJob;
        }
    }, MetadataAccess.SERVICE);
    lastAssessedFeedMap.put(feedName, lastFeedJob);
    return lastFeedJob;
}