Examples with BatchJobExecution com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution used on opensource projects

Example 6 with BatchJobExecution

use of com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution in project kylo by Teradata.

the class DefaultJobService method failJobExecution.

@Override
public void failJobExecution(Long executionId) {
    metadataAccess.commit(() -> {
        BatchJobExecution execution = this.jobExecutionProvider.findByJobExecutionId(executionId, false);
        if (execution != null && !execution.isFailed()) {
            Set<BatchStepExecution> steps = execution.getStepExecutions();
            if (steps != null) {
                for (BatchStepExecution step : steps) {
                    if (!step.isFinished()) {
                        step.setStatus(BatchStepExecution.StepStatus.FAILED);
                        step.setExitCode(ExecutionConstants.ExitCode.FAILED);
                        String msg = step.getExitMessage() != null ? step.getExitMessage() + "\n" : "";
                        msg += "Step manually failed @ " + DateTimeUtil.getNowFormattedWithTimeZone();
                        step.setExitMessage(msg);
                        execution.setExitMessage(msg);
                    }
                }
            }
            if (execution.getStartTime() == null) {
                execution.setStartTime(DateTimeUtil.getNowUTCTime());
            }
            execution.setStatus(BatchJobExecution.JobStatus.FAILED);
            if (execution.getEndTime() == null) {
                execution.setEndTime(DateTimeUtil.getNowUTCTime());
            }
            String msg = execution.getExitMessage() != null ? execution.getExitMessage() + "\n" : "";
            msg += "Job manually failed @ " + DateTimeUtil.getNowFormattedWithTimeZone();
            execution.setExitMessage(msg);
            OpsManagerFeed feed = execution.getJobInstance().getFeed();
            this.jobExecutionProvider.save(execution);
            this.jobExecutionProvider.notifyFailure(execution, feed, false, "Job manually failed @ " + DateTimeUtil.getNowFormattedWithTimeZone());
        }
        return execution;
    });
}

Example 7 with BatchJobExecution

use of com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution in project kylo by Teradata.

the class DefaultJobService method restartJobExecution.

@Override
public Long restartJobExecution(ReplayJobExecution replayJobExecution) throws JobExecutionException {
    SavepointReplayJobExecution savepointReplayJobExecution = ((SavepointReplayJobExecution) replayJobExecution);
    return metadataAccess.commit(() -> {
        BatchJobExecution jobExecution = this.jobExecutionProvider.findByJobExecutionId(savepointReplayJobExecution.getJobExecutionId(), false);
        if (jobExecution != null) {
            ((JpaBatchJobExecution) jobExecution).markAsRunning();
            // trigger the jms message
            SavepointReplayEvent event = new SavepointReplayEvent();
            event.setJobExecutionId(savepointReplayJobExecution.getJobExecutionId());
            event.setFlowfileId(savepointReplayJobExecution.getFlowFileId());
            event.setAction(SavepointReplayEvent.Action.RETRY);
            savepointReplayJmsEventService.triggerSavepoint(event);
            return jobExecution.getJobExecutionId();
        }
        return null;
    });
}

Example 8 with BatchJobExecution

use of com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution in project kylo by Teradata.

the class ProvenanceEventReceiver method processEvent.

/**
 * process the event and persist it along with creating the Job and Step.  If there is a lock error it will retry until it hits the {@link #lockAcquisitionRetryAmount}
 *
 * @param event        a provenance event
 * @param retryAttempt the retry number.  If there is a lock error it will retry until it hits the {@link #lockAcquisitionRetryAmount}
 */
private void processEvent(ProvenanceEventRecordDTO event, int retryAttempt) {
    try {
        OpsManagerFeed feed = provenanceEventFeedUtil.getFeed(event);
        log.debug("Process {} for flowfile: {} and processorId: {} ", event, event.getJobFlowFileId(), event.getFirstEventProcessorId());
        // ensure the job is there
        BatchJobExecution jobExecution = findOrCreateJobExecution(event, feed);
        if (jobExecution != null) {
            batchJobExecutionProvider.updateFeedJobStartTime(jobExecution, feed);
        }
        if (jobExecution != null && !event.isStream()) {
            metadataAccess.commit(() -> receiveBatchEvent(jobExecution, event), MetadataAccess.SERVICE);
        }
        if (jobExecution != null && event.isFinalJobEvent()) {
            notifyJobFinished(jobExecution, event);
        }
    } catch (LockAcquisitionException lae) {
        if (retryAttempt < lockAcquisitionRetryAmount) {
            retryAttempt++;
            log.error("LockAcquisitionException found trying to process Event: {} .  Retry attempt # {} ", event, retryAttempt, lae);
            // wait and re attempt
            try {
                Thread.sleep(300L);
            } catch (InterruptedException var10) {
            }
            processEvent(event, retryAttempt);
        } else {
            log.error("LockAcquisitionException found.  Unsuccessful after retrying {} times.  This event {} will not be processed. ", retryAttempt, event, lae);
        }
    } catch (Exception e) {
        log.error("Error processing Event ", event, e);
    }
}

Example 9 with BatchJobExecution

use of com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution in project kylo by Teradata.

the class ProvenanceEventReceiver method findOrCreateJobExecution.

/**
 * Try to get JobExecution for the given event.
 * This will lock on the events JobFlowFileId when looking for the jobExecution
 * @param event the event to process
 * @return the JobExeuction related to this Event
 * @throws InterruptedException
 */
private BatchJobExecution findOrCreateJobExecution(ProvenanceEventRecordDTO event, OpsManagerFeed feed) throws InterruptedException {
    BatchJobExecution jobExecution = null;
    Lock lock = ProvenanceEventJobExecutionLockManager.getLock(event.getJobFlowFileId());
    String key = event.getJobFlowFileId() + "-" + event.getFlowFileUuid() + "-" + event.getEventId();
    findJobExecutionAttempts.putIfAbsent(key, new AtomicInteger(0));
    try {
        if (lock.tryLock(2L, TimeUnit.SECONDS)) {
            try {
                jobExecution = metadataAccess.commit(() -> batchJobExecutionProvider.getOrCreateJobExecution(event, feed), MetadataAccess.SERVICE);
            } finally {
                findJobExecutionAttempts.remove(key);
                ProvenanceEventJobExecutionLockManager.releaseLock(event.getJobFlowFileId());
            }
        } else {
            int retries = findJobExecutionAttempts.get(key).incrementAndGet();
            log.warn("unable to acquire lock for {} while looking for JobExecution.  Retry attempt: {} ", event.getJobFlowFileId(), retries);
            // try three times before giving up
            if (retries < 3) {
                findOrCreateJobExecution(event, feed);
            } else {
                // give up
                findJobExecutionAttempts.remove(key);
                log.error("Unable to acquire the Lock to get/create the JobExecution for this event {}. This event will not be processed", event);
            }
        }
    } catch (InterruptedException e) {
        throw e;
    }
    return jobExecution;
}

Example 10 with BatchJobExecution

use of com.thinkbiganalytics.metadata.api.jobrepo.job.BatchJobExecution in project kylo by Teradata.

the class ProvenanceEventReceiver method receiveBatchEvent.

/**
 * Process this record and record the Job and steps
 *
 * @param jobExecution the job execution
 * @param event        a provenance event
 * @return a persisted nifi event object
 */
private void receiveBatchEvent(BatchJobExecution jobExecution, ProvenanceEventRecordDTO event) {
    // NifiEvent nifiEvent = null;
    log.debug("Received ProvenanceEvent {}.  is ending flowfile:{}", event, event.isEndingFlowFileEvent());
    // query it again
    jobExecution = batchJobExecutionProvider.findByJobExecutionId(jobExecution.getJobExecutionId(), true);
    BatchJobExecution job = batchJobExecutionProvider.save(jobExecution, event);
    if (job == null) {
        log.error(" Detected a Batch event, but could not find related Job record. for event: {}  is end of Job: {}.  is ending flowfile:{}, ", event, event.isEndingFlowFileEvent(), event.isEndingFlowFileEvent());
    }
}