use of org.pentaho.di.core.gui.JobTracker in project pentaho-kettle by pentaho.
the class Job method execute.
/**
* Execute a job entry recursively and move to the next job entry automatically.<br>
* Uses a back-tracking algorithm.<br>
*
* @param nr
* @param prev_result
* @param jobEntryCopy
* @param previous
* @param reason
* @return
* @throws KettleException
*/
private Result execute(final int nr, Result prev_result, final JobEntryCopy jobEntryCopy, JobEntryCopy previous, String reason) throws KettleException {
Result res = null;
if (stopped.get()) {
res = new Result(nr);
res.stopped = true;
return res;
}
// if we didn't have a previous result, create one, otherwise, copy the content...
//
final Result newResult;
Result prevResult = null;
if (prev_result != null) {
prevResult = prev_result.clone();
} else {
prevResult = new Result();
}
JobExecutionExtension extension = new JobExecutionExtension(this, prevResult, jobEntryCopy, true);
ExtensionPointHandler.callExtensionPoint(log, KettleExtensionPoint.JobBeforeJobEntryExecution.id, extension);
jobMeta.disposeEmbeddedMetastoreProvider();
if (jobMeta.getMetastoreLocatorOsgi() != null) {
jobMeta.setEmbeddedMetastoreProviderKey(jobMeta.getMetastoreLocatorOsgi().setEmbeddedMetastore(jobMeta.getEmbeddedMetaStore()));
}
if (extension.result != null) {
prevResult = extension.result;
}
if (!extension.executeEntry) {
newResult = prevResult;
} else {
if (log.isDetailed()) {
log.logDetailed("exec(" + nr + ", " + (prev_result != null ? prev_result.getNrErrors() : 0) + ", " + (jobEntryCopy != null ? jobEntryCopy.toString() : "null") + ")");
}
// Which entry is next?
JobEntryInterface jobEntryInterface = jobEntryCopy.getEntry();
jobEntryInterface.getLogChannel().setLogLevel(logLevel);
// Track the fact that we are going to launch the next job entry...
JobEntryResult jerBefore = new JobEntryResult(null, null, BaseMessages.getString(PKG, "Job.Comment.JobStarted"), reason, jobEntryCopy.getName(), jobEntryCopy.getNr(), environmentSubstitute(jobEntryCopy.getEntry().getFilename()));
jobTracker.addJobTracker(new JobTracker(jobMeta, jerBefore));
ClassLoader cl = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(jobEntryInterface.getClass().getClassLoader());
// Execute this entry...
JobEntryInterface cloneJei = (JobEntryInterface) jobEntryInterface.clone();
((VariableSpace) cloneJei).copyVariablesFrom(this);
cloneJei.setRepository(rep);
if (rep != null) {
cloneJei.setMetaStore(rep.getMetaStore());
}
cloneJei.setParentJob(this);
cloneJei.setParentJobMeta(this.getJobMeta());
final long start = System.currentTimeMillis();
cloneJei.getLogChannel().logDetailed("Starting job entry");
for (JobEntryListener jobEntryListener : jobEntryListeners) {
jobEntryListener.beforeExecution(this, jobEntryCopy, cloneJei);
}
if (interactive) {
if (jobEntryCopy.isTransformation()) {
getActiveJobEntryTransformations().put(jobEntryCopy, (JobEntryTrans) cloneJei);
}
if (jobEntryCopy.isJob()) {
getActiveJobEntryJobs().put(jobEntryCopy, (JobEntryJob) cloneJei);
}
}
log.snap(Metrics.METRIC_JOBENTRY_START, cloneJei.toString());
newResult = cloneJei.execute(prevResult, nr);
log.snap(Metrics.METRIC_JOBENTRY_STOP, cloneJei.toString());
final long end = System.currentTimeMillis();
if (interactive) {
if (jobEntryCopy.isTransformation()) {
getActiveJobEntryTransformations().remove(jobEntryCopy);
}
if (jobEntryCopy.isJob()) {
getActiveJobEntryJobs().remove(jobEntryCopy);
}
}
if (cloneJei instanceof JobEntryTrans) {
String throughput = newResult.getReadWriteThroughput((int) ((end - start) / 1000));
if (throughput != null) {
log.logMinimal(throughput);
}
}
for (JobEntryListener jobEntryListener : jobEntryListeners) {
jobEntryListener.afterExecution(this, jobEntryCopy, cloneJei, newResult);
}
Thread.currentThread().setContextClassLoader(cl);
addErrors((int) newResult.getNrErrors());
// Also capture the logging text after the execution...
//
LoggingBuffer loggingBuffer = KettleLogStore.getAppender();
StringBuffer logTextBuffer = loggingBuffer.getBuffer(cloneJei.getLogChannel().getLogChannelId(), false);
newResult.setLogText(logTextBuffer.toString() + newResult.getLogText());
// Save this result as well...
//
JobEntryResult jerAfter = new JobEntryResult(newResult, cloneJei.getLogChannel().getLogChannelId(), BaseMessages.getString(PKG, "Job.Comment.JobFinished"), null, jobEntryCopy.getName(), jobEntryCopy.getNr(), environmentSubstitute(jobEntryCopy.getEntry().getFilename()));
jobTracker.addJobTracker(new JobTracker(jobMeta, jerAfter));
synchronized (jobEntryResults) {
jobEntryResults.add(jerAfter);
//
if (maxJobEntriesLogged > 0) {
while (jobEntryResults.size() > maxJobEntriesLogged) {
// Remove the oldest.
jobEntryResults.removeFirst();
}
}
}
}
extension = new JobExecutionExtension(this, prevResult, jobEntryCopy, extension.executeEntry);
ExtensionPointHandler.callExtensionPoint(log, KettleExtensionPoint.JobAfterJobEntryExecution.id, extension);
// Try all next job entries.
//
// Keep track of all the threads we fired in case of parallel execution...
// Keep track of the results of these executions too.
//
final List<Thread> threads = new ArrayList<Thread>();
// next 2 lists is being modified concurrently so must be synchronized for this case.
final Queue<Result> threadResults = new ConcurrentLinkedQueue<Result>();
final Queue<KettleException> threadExceptions = new ConcurrentLinkedQueue<KettleException>();
final List<JobEntryCopy> threadEntries = new ArrayList<JobEntryCopy>();
// Launch only those where the hop indicates true or false
//
int nrNext = jobMeta.findNrNextJobEntries(jobEntryCopy);
for (int i = 0; i < nrNext && !isStopped(); i++) {
// The next entry is...
final JobEntryCopy nextEntry = jobMeta.findNextJobEntry(jobEntryCopy, i);
// See if we need to execute this...
final JobHopMeta hi = jobMeta.findJobHop(jobEntryCopy, nextEntry);
// The next comment...
final String nextComment;
if (hi.isUnconditional()) {
nextComment = BaseMessages.getString(PKG, "Job.Comment.FollowedUnconditional");
} else {
if (newResult.getResult()) {
nextComment = BaseMessages.getString(PKG, "Job.Comment.FollowedSuccess");
} else {
nextComment = BaseMessages.getString(PKG, "Job.Comment.FollowedFailure");
}
}
//
if (hi.isUnconditional() || (jobEntryCopy.evaluates() && (!(hi.getEvaluation() ^ newResult.getResult())))) {
// Start this next step!
if (log.isBasic()) {
log.logBasic(BaseMessages.getString(PKG, "Job.Log.StartingEntry", nextEntry.getName()));
}
// When an evaluation is executed the errors e.g. should not be reset.
if (nextEntry.resetErrorsBeforeExecution()) {
newResult.setNrErrors(0);
}
//
if (jobEntryCopy.isLaunchingInParallel()) {
threadEntries.add(nextEntry);
Runnable runnable = new Runnable() {
public void run() {
try {
Result threadResult = execute(nr + 1, newResult, nextEntry, jobEntryCopy, nextComment);
threadResults.add(threadResult);
} catch (Throwable e) {
log.logError(Const.getStackTracker(e));
threadExceptions.add(new KettleException(BaseMessages.getString(PKG, "Job.Log.UnexpectedError", nextEntry.toString()), e));
Result threadResult = new Result();
threadResult.setResult(false);
threadResult.setNrErrors(1L);
threadResults.add(threadResult);
}
}
};
Thread thread = new Thread(runnable);
threads.add(thread);
thread.start();
if (log.isBasic()) {
log.logBasic(BaseMessages.getString(PKG, "Job.Log.LaunchedJobEntryInParallel", nextEntry.getName()));
}
} else {
try {
// Same as before: blocks until it's done
//
res = execute(nr + 1, newResult, nextEntry, jobEntryCopy, nextComment);
} catch (Throwable e) {
log.logError(Const.getStackTracker(e));
throw new KettleException(BaseMessages.getString(PKG, "Job.Log.UnexpectedError", nextEntry.toString()), e);
}
if (log.isBasic()) {
log.logBasic(BaseMessages.getString(PKG, "Job.Log.FinishedJobEntry", nextEntry.getName(), res.getResult() + ""));
}
}
}
}
//
if (jobEntryCopy.isLaunchingInParallel()) {
for (int i = 0; i < threads.size(); i++) {
Thread thread = threads.get(i);
JobEntryCopy nextEntry = threadEntries.get(i);
try {
thread.join();
} catch (InterruptedException e) {
log.logError(jobMeta.toString(), BaseMessages.getString(PKG, "Job.Log.UnexpectedErrorWhileWaitingForJobEntry", nextEntry.getName()));
threadExceptions.add(new KettleException(BaseMessages.getString(PKG, "Job.Log.UnexpectedErrorWhileWaitingForJobEntry", nextEntry.getName()), e));
}
}
// if(log.isBasic()) log.logBasic(BaseMessages.getString(PKG,
// "Job.Log.FinishedJobEntry",startpoint.getName(),res.getResult()+""));
}
// In this case, return the previous result.
if (res == null) {
res = prevResult;
}
//
if (threadExceptions.size() > 0) {
res.setResult(false);
res.setNrErrors(threadExceptions.size());
for (KettleException e : threadExceptions) {
log.logError(jobMeta.toString(), e.getMessage(), e);
}
//
throw threadExceptions.poll();
}
//
for (Result threadResult : threadResults) {
res.add(threadResult);
}
//
if (res.getNrErrors() > 0) {
res.setResult(false);
}
return res;
}
use of org.pentaho.di.core.gui.JobTracker in project pentaho-kettle by pentaho.
the class JobEntryLogTable method getLogRecord.
/**
* This method calculates all the values that are required
*
* @param id
* the id to use or -1 if no id is needed
* @param status
* the log status to use
* @param subject
* the object to log
* @param parent
* the parent to which the object belongs
*/
public RowMetaAndData getLogRecord(LogStatus status, Object subject, Object parent) {
if (subject == null || subject instanceof JobEntryCopy) {
JobEntryCopy jobEntryCopy = (JobEntryCopy) subject;
Job parentJob = (Job) parent;
RowMetaAndData row = new RowMetaAndData();
for (LogTableField field : fields) {
if (field.isEnabled()) {
Object value = null;
if (subject != null) {
JobEntryInterface jobEntry = jobEntryCopy.getEntry();
JobTracker jobTracker = parentJob.getJobTracker();
JobTracker entryTracker = jobTracker.findJobTracker(jobEntryCopy);
JobEntryResult jobEntryResult = null;
if (entryTracker != null) {
jobEntryResult = entryTracker.getJobEntryResult();
}
Result result = null;
if (jobEntryResult != null) {
result = jobEntryResult.getResult();
}
switch(ID.valueOf(field.getId())) {
case ID_BATCH:
value = new Long(parentJob.getBatchId());
break;
case CHANNEL_ID:
value = jobEntry.getLogChannel().getLogChannelId();
break;
case LOG_DATE:
value = new Date();
break;
case JOBNAME:
value = parentJob.getJobname();
break;
case JOBENTRYNAME:
value = jobEntry.getName();
break;
case LINES_READ:
value = new Long(result != null ? result.getNrLinesRead() : 0);
break;
case LINES_WRITTEN:
value = new Long(result != null ? result.getNrLinesWritten() : 0);
break;
case LINES_UPDATED:
value = new Long(result != null ? result.getNrLinesUpdated() : 0);
break;
case LINES_INPUT:
value = new Long(result != null ? result.getNrLinesInput() : 0);
break;
case LINES_OUTPUT:
value = new Long(result != null ? result.getNrLinesOutput() : 0);
break;
case LINES_REJECTED:
value = new Long(result != null ? result.getNrLinesRejected() : 0);
break;
case ERRORS:
value = new Long(result != null ? result.getNrErrors() : 0);
break;
case RESULT:
value = new Boolean(result != null ? result.getResult() : false);
break;
case NR_RESULT_FILES:
value = new Long(result != null && result.getResultFiles() != null ? result.getResultFiles().size() : 0);
break;
case NR_RESULT_ROWS:
value = new Long(result != null && result.getRows() != null ? result.getRows().size() : 0);
break;
case LOG_FIELD:
if (result != null) {
value = result.getLogText();
}
break;
case COPY_NR:
value = new Long(jobEntryCopy.getNr());
break;
default:
break;
}
}
row.addValue(field.getFieldName(), field.getDataType(), value);
row.getRowMeta().getValueMeta(row.size() - 1).setLength(field.getLength());
}
}
return row;
} else {
return null;
}
}
use of org.pentaho.di.core.gui.JobTracker in project pentaho-kettle by pentaho.
the class JobTrackerExecutionIT method testJobTracker.
@Test
public void testJobTracker() throws UnknownParamException, KettleXMLException, URISyntaxException, IOException {
if (res.setAsVariable != null) {
System.getProperties().setProperty(DatabaseLogExceptionFactory.KETTLE_GLOBAL_PROP_NAME, res.setAsVariable.toString());
}
try {
Job job = new Job(null, getJobMeta(res.fileName));
job.setLogLevel(LogLevel.BASIC);
job.start();
job.waitUntilFinished();
// this simulates - Spoon 'Job Metrics' tab attempt to refresh:
JobTracker tracker = job.getJobTracker();
List<JobTracker> trackers = tracker.getJobTrackers();
Assert.assertEquals("Job trackers count is correct: " + res.assertMessage, res.jobTrackerStatus.length, trackers.size());
for (int i = 0; i < res.jobTrackerStatus.length; i++) {
JobTracker record = trackers.get(i);
Boolean actual;
JobEntryResult jer = record.getJobEntryResult();
// don't look into nested JobTrackers
if (jer == null) {
actual = null;
} else {
actual = record.getJobEntryResult().getResult() == null ? null : Boolean.valueOf(record.getJobEntryResult().getResult().getResult());
}
Assert.assertEquals(res.assertMessage + ": " + i, res.jobTrackerStatus[i], actual);
}
} finally {
System.getProperties().remove(DatabaseLogExceptionFactory.KETTLE_GLOBAL_PROP_NAME);
}
}
Aggregations