Examples with BatchJobExecution com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution used on opensource projects

Example 1 with BatchJobExecution

use of com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution in project kylo by Teradata.

the class JpaBatchJobExecutionProvider method markStreamingFeedAsStopped.

public void markStreamingFeedAsStopped(String feed) {
    BatchJobExecution jobExecution = findLatestJobForFeed(feed);
    if (jobExecution != null && !jobExecution.getStatus().equals(BatchJobExecution.JobStatus.STOPPED)) {
        log.info("Stopping Streaming feed job {} for Feed {} ", jobExecution.getJobExecutionId(), feed);
        jobExecution.setStatus(BatchJobExecution.JobStatus.STOPPED);
        jobExecution.setExitCode(ExecutionConstants.ExitCode.COMPLETED);
        ((JpaBatchJobExecution) jobExecution).setLastUpdated(DateTimeUtil.getNowUTCTime());
        jobExecution.setEndTime(DateTimeUtil.getNowUTCTime());
        save(jobExecution);
        // update the cache
        latestStreamingJobByFeedName.put(feed, jobExecution);
    }
}

Example 2 with BatchJobExecution

use of com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution in project kylo by Teradata.

the class OpsFeedManagerFeedProvider method updateStreamingFlag.

/**
 * Sets the stream flag for the list of feeds
 *
 * @param feedNames the feed names to update
 * @param isStream  true if stream/ false if not
 */
public void updateStreamingFlag(Set<String> feedNames, boolean isStream) {
    List<JpaOpsManagerFeed> feeds = (List<JpaOpsManagerFeed>) findByFeedNames(feedNames, false);
    if (feeds != null) {
        for (JpaOpsManagerFeed feed : feeds) {
            // if we move from a stream to a batch we need to stop/complete the running stream job
            if (feed.isStream() && !isStream) {
                BatchJobExecution jobExecution = batchJobExecutionProvider.findLatestJobForFeed(feed.getName());
                if (jobExecution != null && !jobExecution.isFinished()) {
                    jobExecution.setStatus(BatchJobExecution.JobStatus.STOPPED);
                    jobExecution.setExitCode(ExecutionConstants.ExitCode.COMPLETED);
                    jobExecution.setEndTime(DateTime.now());
                    jobExecution = batchJobExecutionProvider.save(jobExecution);
                    batchJobExecutionProvider.notifyStopped(jobExecution, feed, null);
                    // notify stream to batch for feed
                    batchJobExecutionProvider.notifyStreamToBatch(jobExecution, feed);
                }
            } else if (!feed.isStream() && isStream) {
                // if we move from a batch to a stream we need to complete any jobs that are running.
                batchJobExecutionProvider.findRunningJobsForFeed(feed.getName()).stream().forEach(jobExecution -> {
                    jobExecution.setExitCode(ExecutionConstants.ExitCode.STOPPED);
                    jobExecution.setEndTime(DateTime.now());
                    jobExecution.setExitMessage("Stopping and Abandoning the Job.  The job was running while the feed/template changed from a batch to a stream");
                    jobExecution.setStatus(BatchJobExecution.JobStatus.ABANDONED);
                    batchJobExecutionProvider.save(jobExecution);
                    log.info("Stopping and Abandoning the Job {} for feed {}.  The job was running while the feed/template changed from a batch to a stream", jobExecution.getJobExecutionId(), feed.getName());
                    batchJobExecutionProvider.notifyFailure(jobExecution, feed, false, null);
                    // notify batch to stream for feed
                    batchJobExecutionProvider.notifyBatchToStream(jobExecution, feed);
                });
            }
            feed.setStream(isStream);
        }
        save(feeds);
    }
}

Example 3 with BatchJobExecution

use of com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution in project kylo by Teradata.

the class OpsFeedManagerFeedProvider method getLastActiveTimeStamp.

@Override
public DateTime getLastActiveTimeStamp(String feedName) {
    DateTime lastFeedTime = null;
    OpsManagerFeed feed = this.findByName(feedName);
    if (feed != null && feed.isStream()) {
        NifiFeedStats feedStats = metadataAccess.read(() -> nifiFeedStatisticsProvider.findLatestStatsForFeed(feedName));
        if (feedStats != null && feedStats.getLastActivityTimestamp() != null) {
            lastFeedTime = new DateTime(feedStats.getLastActivityTimestamp());
        } else {
            log.warn("feedStats.getLastActivityTimestamp is null for streaming feed {} ", feedName);
        }
    } else {
        BatchJobExecution jobExecution = metadataAccess.read(() -> batchJobExecutionProvider.findLatestCompletedJobForFeed(feedName));
        if (jobExecution != null) {
            lastFeedTime = jobExecution.getEndTime();
        }
    }
    return lastFeedTime;
}

Example 4 with BatchJobExecution

use of com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution in project kylo by Teradata.

the class FeedFailureService method findLatestJob.

/**
 * Find the latest Job, first looking for any failures between the last time this service ran, and now.
 *
 * @param feedName feed to check
 */
public LastFeedJob findLatestJob(String feedName) {
    LastFeedJob lastFeedJob = metadataAccess.read(() -> {
        LastFeedJob lastAssessedJob = lastAssessedFeedMap.getOrDefault(feedName, newEmptyFeedJob(DateTime.now()));
        LOG.debug("Feed failure service check.  LastAssessJob from map is {}", lastAssessedJob);
        DateTime lastAssessedTime = lastAssessedJob.getDateTime();
        if (isEmptyJob(lastAssessedJob)) {
            // attempt to get jobs since the app started
            lastAssessedTime = servicesApplicationStartupListener.getStartTime();
            lastAssessedJob.setDateTime(lastAssessedTime);
        }
        OpsManagerFeed feed = feedProvider.findByName(feedName);
        if (feed == null) {
            LOG.error("Feed Failure Service check Error!!!  Unable to find feed for:  {}", feedName);
            return newEmptyFeedJob(DateTime.now());
        }
        if (feed.isStream()) {
            List<NifiFeedProcessorStats> latestStats = nifiFeedProcessorStatisticsProvider.findLatestFinishedStatsSince(feedName, lastAssessedTime);
            LOG.debug("Streaming Feed failure check for {}.  Found {} stats", feedName, latestStats.size());
            Optional<NifiFeedProcessorStats> total = latestStats.stream().reduce((a, b) -> {
                a.setFailedCount(a.getFailedCount() + b.getFailedCount());
                if (b.getMinEventTime().isAfter(a.getMinEventTime())) {
                    a.setMinEventTime(b.getMinEventTime());
                }
                return a;
            });
            LastFeedJob lastJob = null;
            if (total.isPresent()) {
                NifiFeedProcessorStats stats = total.get();
                boolean success = stats.getFailedCount() == 0;
                lastJob = new LastFeedJob(feedName, stats.getMinEventTime(), success);
            } else {
                lastJob = new LastFeedJob(feedName, lastAssessedTime, true);
            }
            LOG.debug("{} stats for feed. Streaming Feed failure returning {}", total.isPresent() ? "Found" : "Did not find any", lastJob);
            return lastJob;
        } else {
            List<? extends BatchJobExecution> latestJobs = batchJobExecutionProvider.findLatestFinishedJobForFeedSince(feedName, lastAssessedTime);
            LOG.debug("Batch Feed failure check for {}.  Found {} jobs", feedName, latestJobs != null ? latestJobs.size() : 0);
            BatchJobExecution latestJob = latestJobs.stream().sorted(Comparator.comparing(BatchJobExecution::getEndTime).reversed()).filter(job -> FAILED.equals(job.getStatus())).findFirst().orElse(null);
            if (latestJob == null) {
                // find the last job if there are no failures
                latestJob = latestJobs.stream().sorted(Comparator.comparing(BatchJobExecution::getEndTime).reversed()).findFirst().orElse(null);
                // if the set doesnt have anything attempt to get the latest job
                if (latestJob == null) {
                    latestJob = batchJobExecutionProvider.findLatestFinishedJobForFeed(feedName);
                }
            }
            LastFeedJob lastJob = latestJob != null ? new LastFeedJob(feedName, latestJob.getEndTime(), !FAILED.equals(latestJob.getStatus()), latestJob.getJobExecutionId()) : newEmptyFeedJob(lastAssessedTime);
            LOG.debug("Batch Feed failure check returning {} for feed {}", lastJob, feedName);
            return lastJob;
        }
    }, MetadataAccess.SERVICE);
    lastAssessedFeedMap.put(feedName, lastFeedJob);
    return lastFeedJob;
}

Example 5 with BatchJobExecution

use of com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution in project kylo by Teradata.

the class DefaultJobService method restartJobExecution.

@Override
public Long restartJobExecution(ReplayJobExecution replayJobExecution) throws JobExecutionException {
    SavepointReplayJobExecution savepointReplayJobExecution = ((SavepointReplayJobExecution) replayJobExecution);
    return metadataAccess.commit(() -> {
        BatchJobExecution jobExecution = this.jobExecutionProvider.findByJobExecutionId(savepointReplayJobExecution.getJobExecutionId(), false);
        if (jobExecution != null) {
            ((JpaBatchJobExecution) jobExecution).markAsRunning();
            // trigger the jms message
            SavepointReplayEvent event = new SavepointReplayEvent();
            event.setJobExecutionId(savepointReplayJobExecution.getJobExecutionId());
            event.setFlowfileId(savepointReplayJobExecution.getFlowFileId());
            event.setAction(SavepointReplayEvent.Action.RETRY);
            savepointReplayJmsEventService.triggerSavepoint(event);
            return jobExecution.getJobExecutionId();
        }
        return null;
    });
}