Examples with JpaOpsManagerFeed com.thinkbiganalytics.metadata.jpa.feed.JpaOpsManagerFeed used on opensource projects

Example 1 with JpaOpsManagerFeed

use of com.thinkbiganalytics.metadata.jpa.feed.JpaOpsManagerFeed in project kylo by Teradata.

the class FeedOpsUpgradeAction method upgradeTo.

/* (non-Javadoc)
     * @see com.thinkbiganalytics.metadata.upgrade.UpgradeState#upgradeFrom(com.thinkbiganalytics.metadata.api.app.KyloVersion)
     */
@Override
public void upgradeTo(KyloVersion startingVersion) {
    log.info("Upgrading from version: " + startingVersion);
    for (Category category : categoryProvider.findAll()) {
        // Ensure each category has an allowedActions (gets create if not present.)
        category.getAllowedActions();
    }
    // get all feeds defined in feed manager
    List<Feed> domainFeeds = feedProvider.findAll();
    Map<String, Feed> feedManagerFeedMap = new HashMap<>();
    if (domainFeeds != null && !domainFeeds.isEmpty()) {
        List<OpsManagerFeed.ID> opsManagerFeedIds = new ArrayList<OpsManagerFeed.ID>();
        for (Feed feedManagerFeed : domainFeeds) {
            opsManagerFeedIds.add(opsManagerFeedProvider.resolveId(feedManagerFeed.getId().toString()));
            feedManagerFeedMap.put(feedManagerFeed.getId().toString(), feedManagerFeed);
            // Ensure each feed has an allowedActions (gets create if not present.)
            feedManagerFeed.getAllowedActions();
        }
        // find those that match
        List<? extends OpsManagerFeed> opsManagerFeeds = opsManagerFeedProvider.findByFeedIds(opsManagerFeedIds);
        if (opsManagerFeeds != null) {
            for (OpsManagerFeed opsManagerFeed : opsManagerFeeds) {
                feedManagerFeedMap.remove(opsManagerFeed.getId().toString());
            }
        }
        List<OpsManagerFeed> feedsToAdd = new ArrayList<>();
        for (Feed feed : feedManagerFeedMap.values()) {
            String fullName = FeedNameUtil.fullName(feed.getCategory().getSystemName(), feed.getName());
            OpsManagerFeed.ID opsManagerFeedId = opsManagerFeedProvider.resolveId(feed.getId().toString());
            OpsManagerFeed opsManagerFeed = new JpaOpsManagerFeed(opsManagerFeedId, fullName);
            feedsToAdd.add(opsManagerFeed);
        }
        log.info("Synchronizing Feeds from Feed Manager. About to insert {} feed ids/names into Operations Manager", feedsToAdd.size());
        opsManagerFeedProvider.save(feedsToAdd);
    }
}

Example 2 with JpaOpsManagerFeed

use of com.thinkbiganalytics.metadata.jpa.feed.JpaOpsManagerFeed in project kylo by Teradata.

the class JpaBatchJobExecutionProvider method updateJobType.

/**
 * Sets the Job Execution params to either Check Data or Feed Jobs
 */
private boolean updateJobType(BatchJobExecution jobExecution, ProvenanceEventRecordDTO event) {
    if (event.getUpdatedAttributes() != null && (event.getUpdatedAttributes().containsKey(NIFI_JOB_TYPE_PROPERTY) || event.getUpdatedAttributes().containsKey(NIFI_KYLO_JOB_TYPE_PROPERTY))) {
        String jobType = event.getUpdatedAttributes().get(NIFI_JOB_TYPE_PROPERTY);
        if (StringUtils.isBlank(jobType)) {
            jobType = event.getUpdatedAttributes().get(NIFI_KYLO_JOB_TYPE_PROPERTY);
        }
        String nifiCategory = event.getAttributeMap().get(NIFI_CATEGORY_PROPERTY);
        String nifiFeedName = event.getAttributeMap().get(NIFI_FEED_PROPERTY);
        String feedName = FeedNameUtil.fullName(nifiCategory, nifiFeedName);
        if (FeedConstants.PARAM_VALUE__JOB_TYPE_CHECK.equalsIgnoreCase(jobType)) {
            Set<JpaBatchJobExecutionParameter> updatedParams = ((JpaBatchJobExecution) jobExecution).setAsCheckDataJob(feedName);
            jobParametersRepository.save(updatedParams);
            // update feed type
            JpaOpsManagerFeed checkDataFeed = (JpaOpsManagerFeed) opsManagerFeedRepository.findByName(event.getFeedName());
            checkDataFeed.setFeedType(OpsManagerFeed.FeedType.CHECK);
            // relate to this feed
            JpaOpsManagerFeed feedToCheck = (JpaOpsManagerFeed) opsManagerFeedRepository.findByName(feedName);
            feedToCheck.getCheckDataFeeds().add(checkDataFeed);
            return true;
        }
    }
    return false;
}

Example 3 with JpaOpsManagerFeed

use of com.thinkbiganalytics.metadata.jpa.feed.JpaOpsManagerFeed in project kylo by Teradata.

the class JpaBatchJobExecutionProvider method getOrCreateBatchJobExecution.

private JpaBatchJobExecution getOrCreateBatchJobExecution(ProvenanceEventRecordDTO event, OpsManagerFeed feed) {
    JpaBatchJobExecution jobExecution = null;
    boolean isNew = false;
    try {
        jobExecution = jobExecutionRepository.findByFlowFile(event.getJobFlowFileId());
        if (jobExecution == null) {
            jobExecution = createNewJobExecution(event, feed);
            isNew = true;
        }
    } catch (OptimisticLockException e) {
        // read
        jobExecution = jobExecutionRepository.findByFlowFile(event.getJobFlowFileId());
    }
    // if the attrs coming in change the type to a CHECK job then update the entity
    boolean updatedJobType = updateJobType(jobExecution, event);
    boolean save = isNew || updatedJobType;
    if (event.isFinalJobEvent()) {
        finishJob(event, jobExecution);
        save = true;
    }
    // if the event is the start of the Job, but the job execution was created from another downstream event, ensure the start time and event are related correctly
    if (event.isStartOfJob() && !isNew) {
        jobExecution.getNifiEventJobExecution().setEventId(event.getEventId());
        DateTime startTime = jobExecution.getStartTime();
        if (event.getStartTime() != null) {
            DateTime eventStartTime = DateTimeUtil.convertToUTC(event.getStartTime());
            startTime = earliestTime(startTime, eventStartTime);
            jobExecution.setStartTime(startTime);
        } else {
            DateTime eventTime = DateTimeUtil.convertToUTC(event.getEventTime());
            startTime = earliestTime(startTime, eventTime);
            jobExecution.setStartTime(startTime);
        }
        // create the job params
        Map<String, Object> jobParameters = new HashMap<>();
        if (event.isStartOfJob() && event.getAttributeMap() != null) {
            jobParameters = new HashMap<>(event.getAttributeMap());
        } else {
            jobParameters = new HashMap<>();
        }
        this.jobParametersRepository.save(addJobParameters(jobExecution, jobParameters));
        save = true;
    }
    if (save) {
        jobExecution = (JpaBatchJobExecution) save(jobExecution);
        if (isNew) {
            log.info("Created new Job Execution with id of {} and starting event {} ", jobExecution.getJobExecutionId(), event);
        }
        if (updatedJobType) {
            // notify operations status
            if (feed instanceof JpaOpsManagerFeed) {
                ((JpaOpsManagerFeed) feed).setFeedType(OpsManagerFeed.FeedType.CHECK);
            }
            jobExecutionChangedNotifier.notifyDataConfidenceJob(jobExecution, feed, "Data Confidence Job detected ");
        }
    }
    return jobExecution;
}

Example 4 with JpaOpsManagerFeed

use of com.thinkbiganalytics.metadata.jpa.feed.JpaOpsManagerFeed in project kylo by Teradata.

the class FeedOpsUpgradeAction method upgradeTo.

/* (non-Javadoc)
     * @see com.thinkbiganalytics.metadata.upgrade.UpgradeState#upgradeFrom(com.thinkbiganalytics.metadata.api.app.KyloVersion)
     */
@Override
public void upgradeTo(KyloVersion startingVersion) {
    log.info("Upgrading from version: " + startingVersion);
    for (Category category : categoryProvider.findAll()) {
        // Ensure each category has an allowedActions (gets create if not present.)
        category.getAllowedActions();
    }
    // get all feeds defined in feed manager
    List<Feed> domainFeeds = feedProvider.findAll();
    Map<String, Feed> feedManagerFeedMap = new HashMap<>();
    if (domainFeeds != null && !domainFeeds.isEmpty()) {
        List<OpsManagerFeed.ID> opsManagerFeedIds = new ArrayList<OpsManagerFeed.ID>();
        for (Feed feedManagerFeed : domainFeeds) {
            opsManagerFeedIds.add(opsManagerFeedProvider.resolveId(feedManagerFeed.getId().toString()));
            feedManagerFeedMap.put(feedManagerFeed.getId().toString(), feedManagerFeed);
            // Ensure each feed has an allowedActions (gets create if not present.)
            feedManagerFeed.getAllowedActions();
        }
        // find those that match
        List<? extends OpsManagerFeed> opsManagerFeeds = opsManagerFeedProvider.findByFeedIds(opsManagerFeedIds);
        if (opsManagerFeeds != null) {
            for (OpsManagerFeed opsManagerFeed : opsManagerFeeds) {
                feedManagerFeedMap.remove(opsManagerFeed.getId().toString());
            }
        }
        List<OpsManagerFeed> feedsToAdd = new ArrayList<>();
        for (Feed feed : feedManagerFeedMap.values()) {
            String fullName = FeedNameUtil.fullName(feed.getCategory().getSystemName(), feed.getName());
            OpsManagerFeed.ID opsManagerFeedId = opsManagerFeedProvider.resolveId(feed.getId().toString());
            OpsManagerFeed opsManagerFeed = new JpaOpsManagerFeed(opsManagerFeedId, fullName);
            feedsToAdd.add(opsManagerFeed);
        }
        log.info("Synchronizing Feeds from Feed Manager. About to insert {} feed ids/names into Operations Manager", feedsToAdd.size());
        opsManagerFeedProvider.save(feedsToAdd);
    }
}