Examples with FeedOperation com.thinkbiganalytics.metadata.api.op.FeedOperation used on opensource projects

Example 1 with FeedOperation

use of com.thinkbiganalytics.metadata.api.op.FeedOperation in project kylo by Teradata.

the class JobRepoFeedOperationsProvider method findLatestCompleted.

@Override
public List<FeedOperation> findLatestCompleted(Feed.ID feedId) {
    return metadata.read(() -> {
        List<FeedOperation> operations = new ArrayList<>();
        Feed feed = this.feedProvider.getFeed(feedId);
        if (feed != null) {
            BatchJobExecution latestJobExecution = this.jobExecutionProvider.findLatestCompletedJobForFeed(feed.getQualifiedName());
            if (latestJobExecution != null) {
                LOG.debug("Latest completed job execution id {} ", latestJobExecution.getJobExecutionId());
                operations.add(createOperation(latestJobExecution));
            }
        }
        return operations;
    });
}

Example 2 with FeedOperation

use of com.thinkbiganalytics.metadata.api.op.FeedOperation in project kylo by Teradata.

the class FeedExecutedSinceFeedAssessor method assess.

@Override
public void assess(FeedExecutedSinceFeed metric, MetricAssessmentBuilder<Serializable> builder) {
    LOG.debug("Assessing metric {}", metric.getDescription());
    FeedProvider feedProvider = getFeedProvider();
    FeedOperationsProvider opsProvider = getFeedOperationsProvider();
    List<Feed> mainFeeds = feedProvider.getFeeds(feedProvider.feedCriteria().name(metric.getFeedName()).category(metric.getCategoryName()));
    LOG.debug("Main feeds {}", mainFeeds);
    List<Feed> triggeredFeeds = feedProvider.getFeeds(feedProvider.feedCriteria().name(metric.getSinceFeedName()).category(metric.getSinceCategoryName()));
    LOG.debug("Triggered feeds {}", triggeredFeeds);
    builder.metric(metric);
    if (!mainFeeds.isEmpty() && !triggeredFeeds.isEmpty()) {
        Feed mainFeed = mainFeeds.get(0);
        Feed triggeredFeed = triggeredFeeds.get(0);
        List<FeedOperation> mainFeedOps = opsProvider.findLatestCompleted(mainFeed.getId());
        List<FeedOperation> triggeredFeedOps = opsProvider.findLatest(triggeredFeed.getId());
        if (mainFeedOps.isEmpty()) {
            // If the feed we are checking has never run then it can't have run before the "since" feed.
            LOG.debug("Main feed ops is empty");
            builder.result(AssessmentResult.FAILURE).message("Main feed " + mainFeed.getName() + " has never executed.");
        } else {
            // If the "since" feed has never run then the tested feed has run before it.
            if (triggeredFeedOps.isEmpty()) {
                LOG.debug("Triggered feed ops is empty");
                builder.result(AssessmentResult.SUCCESS).message("Triggered feed " + triggeredFeed.getName() + " has never executed");
            } else {
                DateTime mainFeedStopTime = mainFeedOps.get(0).getStopTime();
                DateTime triggeredFeedStartTime = triggeredFeedOps.get(0).getStartTime();
                LOG.debug("Main feed stop time {}", mainFeedStopTime);
                LOG.debug("Triggered feed start time {}", triggeredFeedStartTime);
                if (mainFeedStopTime.isBefore(triggeredFeedStartTime)) {
                    LOG.debug("Main feed stop time is before triggered feed start time");
                    builder.result(AssessmentResult.FAILURE).message("Main feed " + mainFeed.getName() + " has not executed since triggered feed " + triggeredFeed.getName() + ": " + triggeredFeedStartTime);
                } else {
                    LOG.debug("Main feed stop time is after triggered feed start time");
                    boolean isMainFeedRunning = opsProvider.isFeedRunning(mainFeed.getId());
                    if (isMainFeedRunning) {
                        // todo whether to trigger the feed while the other one is already running should be a
                        // configuration parameter defined by the user
                        LOG.debug("Main feed is still running");
                        builder.result(AssessmentResult.SUCCESS).message("Triggered feed " + triggeredFeed.getName() + " has executed since feed " + mainFeed.getName() + ", but main feed " + mainFeed.getName() + " is still running");
                    } else {
                        LOG.debug("Main is not running");
                        builder.result(AssessmentResult.SUCCESS).message("Triggered feed " + triggeredFeed.getName() + " has executed since main feed " + mainFeed.getName() + ".");
                    }
                }
            }
        }
    } else {
        LOG.debug("Either triggered or main feed does not exist");
        builder.result(AssessmentResult.FAILURE).message("Either feed " + metric.getSinceCategoryAndFeedName() + " and/or feed " + metric.getSinceCategoryAndFeedName() + " does not exist.");
    }
}

Example 3 with FeedOperation

use of com.thinkbiganalytics.metadata.api.op.FeedOperation in project kylo by Teradata.

the class FeedExecutedSinceScheduleAssessor method assess.

@Override
public void assess(FeedExecutedSinceSchedule metric, MetricAssessmentBuilder<Serializable> builder) {
    Date prev = CronExpressionUtil.getPreviousFireTime(metric.getCronExpression(), 2);
    DateTime schedTime = new DateTime(prev);
    String feedName = metric.getFeedName();
    FeedCriteria crit = getFeedProvider().feedCriteria().name(feedName);
    List<Feed> feeds = getFeedProvider().getFeeds(crit);
    if (feeds.size() > 0) {
        Feed feed = feeds.get(0);
        List<FeedOperation> list = this.getFeedOperationsProvider().findLatestCompleted(feed.getId());
        if (!list.isEmpty()) {
            FeedOperation latest = list.get(0);
            if (latest.getStopTime().isAfter(schedTime)) {
                builder.result(AssessmentResult.SUCCESS).message("Feed " + feed.getName() + " has executed at least 1 operation since " + schedTime);
            } else {
                builder.result(AssessmentResult.FAILURE).message("Feed " + feed.getName() + " has not executed any data operations since " + schedTime);
            }
        }
    }
}

Example 4 with FeedOperation

use of com.thinkbiganalytics.metadata.api.op.FeedOperation in project kylo by Teradata.

the class FeedExecutedSinceFeedAssessorTest method setUpAssessor.

/**
 * @param triggeredFeedStartTime pass negative value for empty operations list
 * @param mainFeedStopTime       pass negative for empty operations list
 */
private FeedExecutedSinceFeedAssessor setUpAssessor(int triggeredFeedStartTime, boolean isTriggeredFeedRunning, int mainFeedStopTime, boolean isMainFeedRunning) {
    FeedCriteria dummyCriteria = mock(FeedCriteria.class);
    when(dummyCriteria.name(Mockito.anyString())).thenReturn(dummyCriteria);
    when(dummyCriteria.category(Mockito.anyString())).thenReturn(dummyCriteria);
    FeedProvider feedProvider = mock(FeedProvider.class);
    when(feedProvider.feedCriteria()).thenReturn(dummyCriteria);
    List<Feed> triggeredFeeds = new ArrayList<>();
    Feed triggeredFeed = mock(Feed.class);
    Feed.ID triggeredFeedId = mock(Feed.ID.class);
    when(triggeredFeed.getId()).thenReturn(triggeredFeedId);
    triggeredFeeds.add(triggeredFeed);
    List<Feed> mainFeeds = new ArrayList<>();
    Feed mainFeed = mock(Feed.class);
    Feed.ID mainFeedId = mock(Feed.ID.class);
    when(mainFeed.getId()).thenReturn(mainFeedId);
    mainFeeds.add(mainFeed);
    when(feedProvider.getFeeds(dummyCriteria)).thenReturn(mainFeeds, triggeredFeeds);
    FeedOperationsProvider opsProvider = mock(FeedOperationsProvider.class);
    List<FeedOperation> triggeredFeedOps = new ArrayList<>();
    if (triggeredFeedStartTime > 0) {
        FeedOperation triggeredOp = mock(FeedOperation.class);
        when(triggeredOp.getStartTime()).thenReturn(new DateTime(triggeredFeedStartTime));
        triggeredFeedOps.add(triggeredOp);
    }
    List<FeedOperation> mainFeedOps = new ArrayList<>();
    if (mainFeedStopTime > 0) {
        FeedOperation mainFeedOp = mock(FeedOperation.class);
        when(mainFeedOp.getStopTime()).thenReturn(new DateTime(mainFeedStopTime));
        mainFeedOps.add(mainFeedOp);
    }
    when(opsProvider.findLatestCompleted(mainFeedId)).thenReturn(mainFeedOps);
    when(opsProvider.findLatest(triggeredFeedId)).thenReturn(triggeredFeedOps);
    when(opsProvider.isFeedRunning(mainFeedId)).thenReturn(isMainFeedRunning);
    when(opsProvider.isFeedRunning(triggeredFeedId)).thenReturn(isTriggeredFeedRunning);
    return new FeedExecutedSinceFeedAssessor() {

        @Override
        protected FeedProvider getFeedProvider() {
            return feedProvider;
        }

        @Override
        protected FeedOperationsProvider getFeedOperationsProvider() {
            return opsProvider;
        }
    };
}

Example 5 with FeedOperation

use of com.thinkbiganalytics.metadata.api.op.FeedOperation in project kylo by Teradata.

the class JobRepoFeedOperationsProvider method findLatest.

@Override
public List<FeedOperation> findLatest(Feed.ID feedId) {
    return metadata.read(() -> {
        List<FeedOperation> operations = new ArrayList<>();
        Feed feed = this.feedProvider.getFeed(feedId);
        if (feed != null) {
            BatchJobExecution latestJobExecution = this.jobExecutionProvider.findLatestJobForFeed(feed.getQualifiedName());
            if (latestJobExecution != null) {
                operations.add(createOperation(latestJobExecution));
            }
        }
        return operations;
    });
}