Examples with BatchJobExecution com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution used on opensource projects

Example 11 with BatchJobExecution

use of com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution in project kylo by Teradata.

the class ProvenanceEventReceiver method queryForNiFiErrorBulletins.

/**
 * Make a REST call to NiFi and query for the NiFi Bulletins that have a flowfile id matching for this job execution and write the bulletin message to the {@link
 * BatchJobExecution#setExitMessage(String)}
 *
 * @param event a provenance event
 */
private void queryForNiFiErrorBulletins(ProvenanceEventRecordDTO event) {
    try {
        metadataAccess.commit(() -> {
            // query for nifi logs
            List<String> relatedFlowFiles = batchJobExecutionProvider.findRelatedFlowFiles(event.getFlowFileUuid());
            if (relatedFlowFiles == null) {
                relatedFlowFiles = new ArrayList<>();
            }
            if (relatedFlowFiles.isEmpty()) {
                relatedFlowFiles.add(event.getFlowFileUuid());
            }
            log.debug("Failed Job {}/{}. Found {} related flow files. ", event.getEventId(), event.getFlowFileUuid(), relatedFlowFiles.size());
            List<BulletinDTO> bulletinDTOS = nifiBulletinExceptionExtractor.getErrorBulletinsForFlowFiles(relatedFlowFiles);
            if (bulletinDTOS != null && !bulletinDTOS.isEmpty()) {
                // write them back to the job
                BatchJobExecution jobExecution = batchJobExecutionProvider.findJobExecution(event);
                if (jobExecution != null) {
                    String msg = jobExecution.getExitMessage() != null ? jobExecution.getExitMessage() + "\n" : "";
                    msg += "NiFi exceptions: \n" + bulletinDTOS.stream().map(bulletinDTO -> bulletinDTO.getMessage()).collect(Collectors.joining("\n"));
                    jobExecution.setExitMessage(msg);
                    this.batchJobExecutionProvider.save(jobExecution);
                }
            }
        }, MetadataAccess.SERVICE);
    } catch (Exception e) {
        log.error("Unable to query NiFi and save exception bulletins for job failure eventid/flowfile : {} / {}. Exception Message:  {}", event.getEventId(), event.getFlowFileUuid(), e.getMessage(), e);
    }
}

Example 12 with BatchJobExecution

use of com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution in project kylo by Teradata.

the class FeedFailureService method findLatestJob.

/**
 * Find the latest Job, first looking for any failures between the last time this service ran, and now.
 *
 * @param feedName feed to check
 */
public LastFeedJob findLatestJob(String feedName) {
    LastFeedJob lastFeedJob = metadataAccess.read(() -> {
        LastFeedJob lastAssessedJob = lastAssessedFeedMap.getOrDefault(feedName, newEmptyFeedJob(DateTime.now()));
        LOG.debug("Feed failure service check.  LastAssessJob from map is {}", lastAssessedJob);
        DateTime lastAssessedTime = lastAssessedJob.getDateTime();
        if (isEmptyJob(lastAssessedJob)) {
            // attempt to get jobs since the app started
            lastAssessedTime = servicesApplicationStartupListener.getStartTime();
            lastAssessedJob.setDateTime(lastAssessedTime);
        }
        OpsManagerFeed feed = feedProvider.findByName(feedName);
        if (feed == null) {
            LOG.error("Feed Failure Service check Error!!!  Unable to find feed for:  {}", feedName);
            return newEmptyFeedJob(DateTime.now());
        }
        if (feed.isStream()) {
            List<NifiFeedProcessorStats> latestStats = nifiFeedProcessorStatisticsProvider.findLatestFinishedStatsSince(feedName, lastAssessedTime);
            LOG.debug("Streaming Feed failure check for {}.  Found {} stats", feedName, latestStats.size());
            Optional<NifiFeedProcessorStats> total = latestStats.stream().reduce((a, b) -> {
                a.setFailedCount(a.getFailedCount() + b.getFailedCount());
                if (b.getMinEventTime().isAfter(a.getMinEventTime())) {
                    a.setMinEventTime(b.getMinEventTime());
                }
                return a;
            });
            LastFeedJob lastJob = null;
            if (total.isPresent()) {
                NifiFeedProcessorStats stats = total.get();
                boolean success = stats.getFailedCount() == 0;
                lastJob = new LastFeedJob(feedName, stats.getMinEventTime(), success);
            } else {
                lastJob = new LastFeedJob(feedName, lastAssessedTime, true);
            }
            LOG.debug("{} stats for feed. Streaming Feed failure returning {}", total.isPresent() ? "Found" : "Did not find any", lastJob);
            return lastJob;
        } else {
            List<? extends BatchJobExecution> latestJobs = batchJobExecutionProvider.findLatestFinishedJobForFeedSince(feedName, lastAssessedTime);
            LOG.debug("Batch Feed failure check for {}.  Found {} jobs", feedName, latestJobs != null ? latestJobs.size() : 0);
            BatchJobExecution latestJob = latestJobs.stream().sorted(Comparator.comparing(BatchJobExecution::getEndTime).reversed()).filter(job -> FAILED.equals(job.getStatus())).findFirst().orElse(null);
            if (latestJob == null) {
                // find the last job if there are no failures
                latestJob = latestJobs.stream().sorted(Comparator.comparing(BatchJobExecution::getEndTime).reversed()).findFirst().orElse(null);
                // if the set doesnt have anything attempt to get the latest job
                if (latestJob == null) {
                    latestJob = batchJobExecutionProvider.findLatestFinishedJobForFeed(feedName);
                }
            }
            LastFeedJob lastJob = latestJob != null ? new LastFeedJob(feedName, latestJob.getEndTime(), !FAILED.equals(latestJob.getStatus()), latestJob.getJobExecutionId()) : newEmptyFeedJob(lastAssessedTime);
            LOG.debug("Batch Feed failure check returning {} for feed {}", lastJob, feedName);
            return lastJob;
        }
    }, MetadataAccess.SERVICE);
    lastAssessedFeedMap.put(feedName, lastFeedJob);
    return lastFeedJob;
}

Example 13 with BatchJobExecution

use of com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution in project kylo by Teradata.

the class JobsRestController method getJob.

@GET
@Path("/{executionId}")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation("Gets the specified job.")
@ApiResponses({ @ApiResponse(code = 200, message = "Returns the job.", response = ExecutedJob.class), @ApiResponse(code = 400, message = "The executionId is not a valid integer.", response = RestResponseStatus.class) })
public ExecutedJob getJob(@PathParam("executionId") String executionId, @QueryParam(value = "includeSteps") @DefaultValue("false") boolean includeSteps) {
    this.accessController.checkPermission(AccessController.SERVICES, OperationsAccessControl.ACCESS_OPS);
    return metadataAccess.read(() -> {
        ExecutedJob executedJob = null;
        BatchJobExecution jobExecution = jobExecutionProvider.findByJobExecutionId(Long.parseLong(executionId), false);
        if (jobExecution != null) {
            if (includeSteps) {
                executedJob = JobModelTransform.executedJob(jobExecution);
            } else {
                executedJob = JobModelTransform.executedJobSimple(jobExecution);
            }
        }
        return executedJob;
    });
}

Example 14 with BatchJobExecution

use of com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution in project kylo by Teradata.

the class JpaBatchJobExecutionProvider method abandonJob.

@Override
public BatchJobExecution abandonJob(Long executionId) {
    BatchJobExecution execution = findByJobExecutionId(executionId, false);
    if (execution != null && !execution.getStatus().equals(BatchJobExecution.JobStatus.ABANDONED)) {
        if (execution.getStartTime() == null) {
            execution.setStartTime(DateTimeUtil.getNowUTCTime());
        }
        execution.setStatus(BatchJobExecution.JobStatus.ABANDONED);
        if (execution.getEndTime() == null) {
            execution.setEndTime(DateTimeUtil.getNowUTCTime());
        }
        String abandonMessage = "Job manually abandoned @ " + DateTimeUtil.getNowFormattedWithTimeZone();
        String msg = execution.getExitMessage() != null ? execution.getExitMessage() + "\n" : "";
        msg += abandonMessage;
        execution.setExitMessage(msg);
        // also stop any running steps??
        // find the feed associated with the job
        OpsManagerFeed feed = execution.getJobInstance().getFeed();
        save(execution);
        jobExecutionChangedNotifier.notifyAbandoned(execution, feed, null);
        // clear the associated alert
        String alertId = execution.getJobExecutionContextAsMap().get(BatchJobExecutionProvider.KYLO_ALERT_ID_PROPERTY);
        if (StringUtils.isNotBlank(alertId)) {
            provider.respondTo(provider.resolve(alertId), (alert1, response) -> response.handle(abandonMessage));
        }
    }
    return execution;
}

Example 15 with BatchJobExecution

use of com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution in project kylo by Teradata.

the class JpaBatchJobExecutionProvider method getOrCreateStreamJobExecution.

private JpaBatchJobExecution getOrCreateStreamJobExecution(ProvenanceEventRecordDTO event, OpsManagerFeed feed) {
    JpaBatchJobExecution jobExecution = null;
    boolean isNew = false;
    try {
        BatchJobExecution latestJobExecution = latestStreamingJobByFeedName.get(event.getFeedName());
        if (latestJobExecution == null) {
            latestJobExecution = findLatestJobForFeed(event.getFeedName());
        } else {
            if (clusterService.isClustered()) {
                latestJobExecution = jobExecutionRepository.findOne(latestJobExecution.getJobExecutionId());
            }
        }
        if (latestJobExecution == null || (latestJobExecution != null && !latestJobExecution.isStream())) {
            // If the latest Job is not set to be a Stream and its still running we need to fail it and create the new streaming job.
            if (latestJobExecution != null && !latestJobExecution.isFinished()) {
                ProvenanceEventRecordDTO tempFailedEvent = new ProvenanceEventRecordDTO();
                tempFailedEvent.setFeedName(event.getFeedName());
                tempFailedEvent.setAttributeMap(new HashMap<>());
                tempFailedEvent.setIsFailure(true);
                tempFailedEvent.setDetails("Failed Running Batch event as this Feed has now become a Stream");
                finishJob(tempFailedEvent, (JpaBatchJobExecution) latestJobExecution);
                latestJobExecution.setExitMessage("Failed Running Batch event as this Feed has now become a Stream");
                save(latestJobExecution);
            }
            jobExecution = createNewJobExecution(event, feed);
            jobExecution.setStream(true);
            latestStreamingJobByFeedName.put(event.getFeedName(), jobExecution);
            log.info("Created new Streaming Job Execution with id of {} and starting event {} ", jobExecution.getJobExecutionId(), event);
        } else {
            jobExecution = (JpaBatchJobExecution) latestJobExecution;
        }
        if (jobExecution != null) {
            latestStreamingJobByFeedName.put(event.getFeedName(), jobExecution);
        }
    } catch (OptimisticLockException e) {
        // read
        jobExecution = (JpaBatchJobExecution) findLatestJobForFeed(event.getFeedName());
    }
    boolean save = isNew;
    if (!jobExecution.isStream()) {
        jobExecution.setStream(true);
        save = true;
    }
    if (save) {
        save(jobExecution);
    }
    return jobExecution;
}