Example 1 with WorkflowTracker
use of org.apache.hop.core.gui.WorkflowTracker in project hop by apache.
the class ActionMail method execute.
@Override
public Result execute(Result result, int nr) {
File masterZipfile = null;
// Send an e-mail...
// create some properties and get the default Session
Properties props = new Properties();
if (Utils.isEmpty(server)) {
logError(BaseMessages.getString(PKG, "ActionMail.Error.HostNotSpecified"));
result.setNrErrors(1L);
result.setResult(false);
return result;
}
String protocol = "smtp";
if (usingSecureAuthentication) {
if (secureConnectionType.equals("TLS")) {
// Allow TLS authentication
props.put("mail.smtp.starttls.enable", "true");
} else if (secureConnectionType.equals("TLS 1.2")) {
// Allow TLS 1.2 authentication
props.put("mail.smtp.starttls.enable", "true");
props.put("mail.smtp.ssl.protocols", "TLSv1.2");
} else {
protocol = "smtps";
// required to get rid of a SSL exception :
// nested exception is:
// javax.net.ssl.SSLException: Unsupported record version Unknown
props.put("mail.smtps.quitwait", "false");
}
}
props.put("mail." + protocol + ".host", resolve(server));
if (!Utils.isEmpty(port)) {
props.put("mail." + protocol + ".port", resolve(port));
}
if (log.isDebug()) {
props.put("mail.debug", "true");
}
if (usingAuthentication) {
props.put("mail." + protocol + ".auth", "true");
}
Session session = Session.getInstance(props);
session.setDebug(log.isDebug());
try {
// create a message
Message msg = new MimeMessage(session);
// set message priority
if (usePriority) {
String priorityInt = "1";
if (priority.equals("low")) {
priorityInt = "3";
}
if (priority.equals("normal")) {
priorityInt = "2";
}
// (String)int between 1= high and 3 = low.
msg.setHeader("X-Priority", priorityInt);
msg.setHeader("Importance", importance);
// seems to be needed for MS Outlook.
// where it returns a string of high /normal /low.
msg.setHeader("Sensitivity", sensitivity);
// Possible values are normal, personal, private, company-confidential
}
// Set Mail sender (From)
String senderAddress = resolve(replyAddress);
if (!Utils.isEmpty(senderAddress)) {
String senderName = resolve(replyName);
if (!Utils.isEmpty(senderName)) {
senderAddress = senderName + '<' + senderAddress + '>';
}
msg.setFrom(new InternetAddress(senderAddress));
} else {
throw new MessagingException(BaseMessages.getString(PKG, "ActionMail.Error.ReplyEmailNotFilled"));
}
// set Reply to addresses
String replyToAddress = resolve(replyToAddresses);
if (!Utils.isEmpty(replyToAddress)) {
// Split the mail-address: variables separated
String[] replyAddressList = resolve(replyToAddress).split(" ");
InternetAddress[] address = new InternetAddress[replyAddressList.length];
for (int i = 0; i < replyAddressList.length; i++) {
address[i] = new InternetAddress(replyAddressList[i]);
}
msg.setReplyTo(address);
}
// Split the mail-address: variables separated
String[] destinations = resolve(destination).split(" ");
InternetAddress[] address = new InternetAddress[destinations.length];
for (int i = 0; i < destinations.length; i++) {
address[i] = new InternetAddress(destinations[i]);
}
msg.setRecipients(Message.RecipientType.TO, address);
String realCC = resolve(getDestinationCc());
if (!Utils.isEmpty(realCC)) {
// Split the mail-address Cc: variables separated
String[] destinationsCc = realCC.split(" ");
InternetAddress[] addressCc = new InternetAddress[destinationsCc.length];
for (int i = 0; i < destinationsCc.length; i++) {
addressCc[i] = new InternetAddress(destinationsCc[i]);
}
msg.setRecipients(Message.RecipientType.CC, addressCc);
}
String realBCc = resolve(getDestinationBCc());
if (!Utils.isEmpty(realBCc)) {
// Split the mail-address BCc: variables separated
String[] destinationsBCc = realBCc.split(" ");
InternetAddress[] addressBCc = new InternetAddress[destinationsBCc.length];
for (int i = 0; i < destinationsBCc.length; i++) {
addressBCc[i] = new InternetAddress(destinationsBCc[i]);
}
msg.setRecipients(Message.RecipientType.BCC, addressBCc);
}
String realSubject = resolve(subject);
if (!Utils.isEmpty(realSubject)) {
msg.setSubject(realSubject);
}
msg.setSentDate(new Date());
StringBuilder messageText = new StringBuilder();
String endRow = isUseHTML() ? "<br>" : Const.CR;
String realComment = resolve(comment);
if (!Utils.isEmpty(realComment)) {
messageText.append(realComment).append(Const.CR).append(Const.CR);
}
if (!onlySendComment) {
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.Workflow")).append(endRow);
messageText.append("-----").append(endRow);
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.JobName") + " : ").append(parentWorkflow.getWorkflowMeta().getName()).append(endRow);
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.Action") + " : ").append(getName()).append(endRow);
messageText.append(Const.CR);
}
if (includeDate) {
messageText.append(endRow).append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.MsgDate") + ": ").append(XmlHandler.date2string(new Date())).append(endRow).append(endRow);
}
if (!onlySendComment && result != null) {
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.PreviousResult") + ":").append(endRow);
messageText.append("-----------------").append(endRow);
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.ActionNr") + " : ").append(result.getEntryNr()).append(endRow);
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.Errors") + " : ").append(result.getNrErrors()).append(endRow);
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.LinesRead") + " : ").append(result.getNrLinesRead()).append(endRow);
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.LinesWritten") + " : ").append(result.getNrLinesWritten()).append(endRow);
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.LinesInput") + " : ").append(result.getNrLinesInput()).append(endRow);
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.LinesOutput") + " : ").append(result.getNrLinesOutput()).append(endRow);
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.LinesUpdated") + " : ").append(result.getNrLinesUpdated()).append(endRow);
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.LinesRejected") + " : ").append(result.getNrLinesRejected()).append(endRow);
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.Status") + " : ").append(result.getExitStatus()).append(endRow);
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.Result") + " : ").append(result.getResult()).append(endRow);
messageText.append(endRow);
}
if (!onlySendComment && (!Utils.isEmpty(resolve(contactPerson)) || !Utils.isEmpty(resolve(contactPhone)))) {
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.ContactInfo") + " :").append(endRow);
messageText.append("---------------------").append(endRow);
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.PersonToContact") + " : ").append(resolve(contactPerson)).append(endRow);
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.Tel") + " : ").append(resolve(contactPhone)).append(endRow);
messageText.append(endRow);
}
// Include the path to this action...
if (!onlySendComment) {
WorkflowTracker workflowTracker = parentWorkflow.getWorkflowTracker();
if (workflowTracker != null) {
messageText.append(BaseMessages.getString(PKG, "ActionMail.Log.Comment.PathToJobentry") + ":").append(endRow);
messageText.append("------------------------").append(endRow);
addBacktracking(workflowTracker, messageText);
if (isUseHTML()) {
messageText.replace(0, messageText.length(), messageText.toString().replace(Const.CR, endRow));
}
}
}
MimeMultipart parts = new MimeMultipart();
// put the text in the
MimeBodyPart part1 = new MimeBodyPart();
// Attached files counter
int nrattachedFiles = 0;
if (useHTML) {
if (!Utils.isEmpty(getEncoding())) {
part1.setContent(messageText.toString(), "text/html; " + "charset=" + getEncoding());
} else {
part1.setContent(messageText.toString(), "text/html; " + "charset=ISO-8859-1");
}
} else {
part1.setText(messageText.toString());
}
parts.addBodyPart(part1);
if (includingFiles && result != null) {
List<ResultFile> resultFiles = result.getResultFilesList();
if (resultFiles != null && !resultFiles.isEmpty()) {
if (!zipFiles) {
//
for (ResultFile resultFile : resultFiles) {
FileObject file = resultFile.getFile();
if (file != null && file.exists()) {
boolean found = false;
for (int i = 0; i < fileType.length; i++) {
if (fileType[i] == resultFile.getType()) {
found = true;
}
}
if (found) {
// create a data source
MimeBodyPart files = new MimeBodyPart();
URLDataSource fds = new URLDataSource(file.getURL());
// get a data IHandler to manipulate this file type
files.setDataHandler(new DataHandler(fds));
// include the file in the data source
files.setFileName(file.getName().getBaseName());
// insist on base64 to preserve line endings
files.addHeader("Content-Transfer-Encoding", "base64");
// add the part with the file in the BodyPart()
parts.addBodyPart(files);
nrattachedFiles++;
logBasic("Added file '" + fds.getName() + "' to the mail message.");
}
}
}
} else {
// create a single ZIP archive of all files
masterZipfile = new File(System.getProperty("java.io.tmpdir") + Const.FILE_SEPARATOR + resolve(zipFilename));
ZipOutputStream zipOutputStream = null;
try {
zipOutputStream = new ZipOutputStream(new FileOutputStream(masterZipfile));
for (ResultFile resultFile : resultFiles) {
boolean found = false;
for (int i = 0; i < fileType.length; i++) {
if (fileType[i] == resultFile.getType()) {
found = true;
}
}
if (found) {
FileObject file = resultFile.getFile();
ZipEntry zipEntry = new ZipEntry(file.getName().getBaseName());
zipOutputStream.putNextEntry(zipEntry);
// Now put the content of this file into this archive...
BufferedInputStream inputStream = new BufferedInputStream(HopVfs.getInputStream(file));
try {
int c;
while ((c = inputStream.read()) >= 0) {
zipOutputStream.write(c);
}
} finally {
inputStream.close();
}
zipOutputStream.closeEntry();
nrattachedFiles++;
logBasic("Added file '" + file.getName().getURI() + "' to the mail message in a zip archive.");
}
}
} catch (Exception e) {
logError("Error zipping attachement files into file [" + masterZipfile.getPath() + "] : " + e.toString());
logError(Const.getStackTracker(e));
result.setNrErrors(1);
} finally {
if (zipOutputStream != null) {
try {
zipOutputStream.finish();
zipOutputStream.close();
} catch (IOException e) {
logError("Unable to close attachement zip file archive : " + e.toString());
logError(Const.getStackTracker(e));
result.setNrErrors(1);
}
}
}
// Now attach the master zip file to the message.
if (result.getNrErrors() == 0) {
// create a data source
MimeBodyPart files = new MimeBodyPart();
FileDataSource fds = new FileDataSource(masterZipfile);
// get a data IHandler to manipulate this file type
files.setDataHandler(new DataHandler(fds));
// include the file in the data source
files.setFileName(fds.getName());
// add the part with the file in the BodyPart()
parts.addBodyPart(files);
}
}
}
}
int nrEmbeddedImages = 0;
if (embeddedimages != null && embeddedimages.length > 0) {
FileObject imageFile = null;
for (int i = 0; i < embeddedimages.length; i++) {
String realImageFile = resolve(embeddedimages[i]);
String realcontenID = resolve(contentids[i]);
if (messageText.indexOf("cid:" + realcontenID) < 0) {
if (log.isDebug()) {
log.logDebug("Image [" + realImageFile + "] is not used in message body!");
}
} else {
try {
boolean found = false;
imageFile = HopVfs.getFileObject(realImageFile);
if (imageFile.exists() && imageFile.getType() == FileType.FILE) {
found = true;
} else {
log.logError("We can not find [" + realImageFile + "] or it is not a file");
}
if (found) {
// Create part for the image
MimeBodyPart messageBodyPart = new MimeBodyPart();
// Load the image
URLDataSource fds = new URLDataSource(imageFile.getURL());
messageBodyPart.setDataHandler(new DataHandler(fds));
// Setting the header
messageBodyPart.setHeader("Content-ID", "<" + realcontenID + ">");
// Add part to multi-part
parts.addBodyPart(messageBodyPart);
nrEmbeddedImages++;
log.logBasic("Image '" + fds.getName() + "' was embedded in message.");
}
} catch (Exception e) {
log.logError("Error embedding image [" + realImageFile + "] in message : " + e.toString());
log.logError(Const.getStackTracker(e));
result.setNrErrors(1);
} finally {
if (imageFile != null) {
try {
imageFile.close();
} catch (Exception e) {
/* Ignore */
}
}
}
}
}
}
if (nrEmbeddedImages > 0 && nrattachedFiles == 0) {
// If we need to embedd images...
// We need to create a "multipart/related" message.
// otherwise image will appear as attached file
parts.setSubType("related");
}
// put all parts together
msg.setContent(parts);
Transport transport = null;
try {
transport = session.getTransport(protocol);
String authPass = getPassword(authenticationPassword);
if (usingAuthentication) {
if (!Utils.isEmpty(port)) {
transport.connect(resolve(Const.NVL(server, "")), Integer.parseInt(resolve(Const.NVL(port, ""))), resolve(Const.NVL(authenticationUser, "")), authPass);
} else {
transport.connect(resolve(Const.NVL(server, "")), resolve(Const.NVL(authenticationUser, "")), authPass);
}
} else {
transport.connect();
}
transport.sendMessage(msg, msg.getAllRecipients());
} finally {
if (transport != null) {
transport.close();
}
}
} catch (IOException e) {
logError("Problem while sending message: " + e.toString());
result.setNrErrors(1);
} catch (MessagingException mex) {
logError("Problem while sending message: " + mex.toString());
result.setNrErrors(1);
Exception ex = mex;
do {
if (ex instanceof SendFailedException) {
SendFailedException sfex = (SendFailedException) ex;
Address[] invalid = sfex.getInvalidAddresses();
if (invalid != null) {
logError(" ** Invalid Addresses");
for (int i = 0; i < invalid.length; i++) {
logError(" " + invalid[i]);
result.setNrErrors(1);
}
}
Address[] validUnsent = sfex.getValidUnsentAddresses();
if (validUnsent != null) {
logError(" ** ValidUnsent Addresses");
for (int i = 0; i < validUnsent.length; i++) {
logError(" " + validUnsent[i]);
result.setNrErrors(1);
}
}
Address[] validSent = sfex.getValidSentAddresses();
if (validSent != null) {
for (int i = 0; i < validSent.length; i++) {
logError(" " + validSent[i]);
result.setNrErrors(1);
}
}
}
if (ex instanceof MessagingException) {
ex = ((MessagingException) ex).getNextException();
} else {
ex = null;
}
} while (ex != null);
} finally {
if (masterZipfile != null && masterZipfile.exists()) {
masterZipfile.delete();
}
}
if (result.getNrErrors() > 0) {
result.setResult(false);
} else {
result.setResult(true);
}
return result;
}
Also used :
InternetAddress(javax.mail.internet.InternetAddress)
MimeMessage(javax.mail.internet.MimeMessage)
ZipEntry(java.util.zip.ZipEntry)
DataHandler(javax.activation.DataHandler)
Properties(java.util.Properties)
MimeMessage(javax.mail.internet.MimeMessage)
MimeMultipart(javax.mail.internet.MimeMultipart)
BufferedInputStream(java.io.BufferedInputStream)
FileDataSource(javax.activation.FileDataSource)
FileObject(org.apache.commons.vfs2.FileObject)
URLDataSource(javax.activation.URLDataSource)
IOException(java.io.IOException)
ResultFile(org.apache.hop.core.ResultFile)
Date(java.util.Date)
HopException(org.apache.hop.core.exception.HopException)
HopXmlException(org.apache.hop.core.exception.HopXmlException)
IOException(java.io.IOException)
ZipOutputStream(java.util.zip.ZipOutputStream)
FileOutputStream(java.io.FileOutputStream)
WorkflowTracker(org.apache.hop.core.gui.WorkflowTracker)
MimeBodyPart(javax.mail.internet.MimeBodyPart)
ResultFile(org.apache.hop.core.ResultFile)
File(java.io.File)
Example 2 with WorkflowTracker
use of org.apache.hop.core.gui.WorkflowTracker in project hop by apache.
the class RemoteWorkflowEngine method startExecution.
@Override
public Result startExecution() {
try {
executionStartDate = new Date();
// Create a new log channel when we start the action
// It's only now that we use it
//
logChannel = new LogChannel(workflowMeta, parentLoggingObject, gatheringMetrics);
loggingObject = new LoggingObject(this);
logLevel = logChannel.getLogLevel();
workflowTracker = new WorkflowTracker(workflowMeta);
if (previousResult == null) {
result = new Result();
} else {
result = previousResult;
}
IWorkflowEngineRunConfiguration engineRunConfiguration = workflowRunConfiguration.getEngineRunConfiguration();
if (!(engineRunConfiguration instanceof RemoteWorkflowRunConfiguration)) {
throw new HopException("The remote workflow engine expects a remote workflow configuration");
}
remoteWorkflowRunConfiguration = (RemoteWorkflowRunConfiguration) workflowRunConfiguration.getEngineRunConfiguration();
String hopServerName = resolve(remoteWorkflowRunConfiguration.getHopServerName());
if (StringUtils.isEmpty(hopServerName)) {
throw new HopException("No remote Hop server was specified to run the workflow on");
}
String remoteRunConfigurationName = remoteWorkflowRunConfiguration.getRunConfigurationName();
if (StringUtils.isEmpty(remoteRunConfigurationName)) {
throw new HopException("No run configuration was specified to the remote workflow with");
}
if (metadataProvider == null) {
throw new HopException("The remote workflow engine didn't receive a metadata to load hop server '" + hopServerName + "'");
}
logChannel.logBasic("Executing this workflow using the Remote Workflow Engine with run configuration '" + workflowRunConfiguration.getName() + "'");
serverPollDelay = Const.toLong(resolve(remoteWorkflowRunConfiguration.getServerPollDelay()), 1000L);
serverPollInterval = Const.toLong(resolve(remoteWorkflowRunConfiguration.getServerPollInterval()), 2000L);
hopServer = metadataProvider.getSerializer(HopServer.class).load(hopServerName);
if (hopServer == null) {
throw new HopException("Hop server '" + hopServerName + "' could not be found");
}
WorkflowExecutionConfiguration workflowExecutionConfiguration = new WorkflowExecutionConfiguration();
workflowExecutionConfiguration.setRunConfiguration(remoteRunConfigurationName);
if (logLevel != null) {
workflowExecutionConfiguration.setLogLevel(logLevel);
}
if (previousResult != null) {
// This contains result rows, files, ...
//
workflowExecutionConfiguration.setPreviousResult(previousResult);
}
workflowExecutionConfiguration.setGatheringMetrics(gatheringMetrics);
// Pass all variables to the parameters map...
//
Map<String, String> parametersMap = workflowExecutionConfiguration.getParametersMap();
for (String variableName : this.getVariableNames()) {
String variableValue = this.getVariable(variableName);
if (variableName != null && variableValue != null) {
parametersMap.put(variableName, variableValue);
}
}
sendToHopServer(this, workflowMeta, workflowExecutionConfiguration, metadataProvider);
fireWorkflowStartedListeners();
initialized = true;
monitorRemoteWorkflowUntilFinished();
fireWorkflowFinishListeners();
executionEndDate = new Date();
} catch (Exception e) {
logChannel.logError("Error starting workflow", e);
result.setNrErrors(result.getNrErrors() + 1);
try {
fireWorkflowFinishListeners();
} catch (Exception ex) {
logChannel.logError("Error executing workflow finished listeners", ex);
result.setNrErrors(result.getNrErrors() + 1);
}
}
return result;
}
Also used :
IWorkflowEngineRunConfiguration(org.apache.hop.workflow.config.IWorkflowEngineRunConfiguration)
HopException(org.apache.hop.core.exception.HopException)
WorkflowTracker(org.apache.hop.core.gui.WorkflowTracker)
HopException(org.apache.hop.core.exception.HopException)
WebResult(org.apache.hop.www.WebResult)
Result(org.apache.hop.core.Result)
Example 3 with WorkflowTracker
use of org.apache.hop.core.gui.WorkflowTracker in project hop by apache.
the class Workflow method executeFromStart.
/**
* Execute a action recursively and move to the next action automatically.<br>
* Uses a back-tracking algorithm.<br>
*
* @param nr
* @param previousResult
* @param actionMeta
* @param previous
* @param reason
* @return
* @throws HopException
*/
private Result executeFromStart(final int nr, Result previousResult, final ActionMeta actionMeta, ActionMeta previous, String reason) throws HopException {
Result res = null;
if (isStopped()) {
res = new Result(nr);
res.stopped = true;
return res;
}
// if we didn't have a previous result, create one, otherwise, copy the content...
//
final Result newResult;
Result prevResult = null;
if (previousResult != null) {
prevResult = previousResult.clone();
} else {
prevResult = new Result();
}
WorkflowExecutionExtension extension = new WorkflowExecutionExtension(this, prevResult, actionMeta, true);
ExtensionPointHandler.callExtensionPoint(log, this, HopExtensionPoint.WorkflowBeforeActionExecution.id, extension);
if (extension.result != null) {
prevResult = extension.result;
}
if (!extension.executeAction) {
newResult = prevResult;
} else {
if (log.isDetailed()) {
log.logDetailed("exec(" + nr + ", " + (prevResult != null ? prevResult.getNrErrors() : 0) + ", " + (actionMeta != null ? actionMeta.toString() : "null") + ")");
}
// Which entry is next?
IAction action = actionMeta.getAction();
action.getLogChannel().setLogLevel(logLevel);
// Track the fact that we are going to launch the next action...
ActionResult jerBefore = new ActionResult(null, null, BaseMessages.getString(PKG, "Workflow.Comment.WorkflowStarted"), reason, actionMeta.getName(), resolve(actionMeta.getAction().getFilename()));
workflowTracker.addWorkflowTracker(new WorkflowTracker(workflowMeta, jerBefore));
ClassLoader cl = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(action.getClass().getClassLoader());
// Execute this entry...
IAction cloneJei = (IAction) action.clone();
cloneJei.copyFrom(this);
cloneJei.getLogChannel().setLogLevel(getLogLevel());
cloneJei.setMetadataProvider(metadataProvider);
cloneJei.setParentWorkflow(this);
cloneJei.setParentWorkflowMeta(this.getWorkflowMeta());
final long start = System.currentTimeMillis();
cloneJei.getLogChannel().logDetailed("Starting action");
for (IActionListener actionListener : actionListeners) {
actionListener.beforeExecution(this, actionMeta, cloneJei);
}
if (interactive) {
getActiveActions().add(actionMeta.clone());
}
log.snap(Metrics.METRIC_ACTION_START, cloneJei.toString());
newResult = cloneJei.execute(prevResult, nr);
log.snap(Metrics.METRIC_ACTION_STOP, cloneJei.toString());
if (interactive) {
getActiveActions().remove(actionMeta);
}
for (IActionListener actionListener : actionListeners) {
actionListener.afterExecution(this, actionMeta, cloneJei, newResult);
}
Thread.currentThread().setContextClassLoader(cl);
addErrors((int) newResult.getNrErrors());
// Also capture the logging text after the execution...
//
LoggingBuffer loggingBuffer = HopLogStore.getAppender();
StringBuffer logTextBuffer = loggingBuffer.getBuffer(cloneJei.getLogChannel().getLogChannelId(), false);
newResult.setLogText(logTextBuffer.toString() + newResult.getLogText());
// Save this result as well...
//
ActionResult jerAfter = new ActionResult(newResult, cloneJei.getLogChannel().getLogChannelId(), BaseMessages.getString(PKG, "Workflow.Comment.WorkflowFinished"), null, actionMeta.getName(), resolve(actionMeta.getAction().getFilename()));
workflowTracker.addWorkflowTracker(new WorkflowTracker(workflowMeta, jerAfter));
synchronized (actionResults) {
actionResults.add(jerAfter);
//
if (maxActionsLogged > 0) {
while (actionResults.size() > maxActionsLogged) {
// Remove the oldest.
actionResults.removeFirst();
}
}
}
}
extension = new WorkflowExecutionExtension(this, prevResult, actionMeta, extension.executeAction);
ExtensionPointHandler.callExtensionPoint(log, this, HopExtensionPoint.WorkflowAfterActionExecution.id, extension);
// Try all next actions.
//
// Keep track of all the threads we fired in case of parallel execution...
// Keep track of the results of these executions too.
//
final List<Thread> threads = new ArrayList<>();
// next 2 lists is being modified concurrently so must be synchronized for this case.
final Queue<Result> threadResults = new ConcurrentLinkedQueue<>();
final Queue<HopException> threadExceptions = new ConcurrentLinkedQueue<>();
final List<ActionMeta> threadActions = new ArrayList<>();
// Launch only those where the hop indicates true or false
//
int nrNext = workflowMeta.findNrNextActions(actionMeta);
for (int i = 0; i < nrNext && !isStopped(); i++) {
// The next entry is...
final ActionMeta nextAction = workflowMeta.findNextAction(actionMeta, i);
// See if we need to execute this...
final WorkflowHopMeta hi = workflowMeta.findWorkflowHop(actionMeta, nextAction);
// The next comment...
final String nextComment;
if (hi.isUnconditional()) {
nextComment = BaseMessages.getString(PKG, "Workflow.Comment.FollowedUnconditional");
} else {
if (newResult.getResult()) {
nextComment = BaseMessages.getString(PKG, "Workflow.Comment.FollowedSuccess");
} else {
nextComment = BaseMessages.getString(PKG, "Workflow.Comment.FollowedFailure");
}
}
//
if (hi.isUnconditional() || (actionMeta.isEvaluation() && (!(hi.getEvaluation() ^ newResult.getResult())))) {
// Start this next transform!
if (log.isBasic()) {
log.logBasic(BaseMessages.getString(PKG, "Workflow.Log.StartingAction", nextAction.getName()));
}
// When an evaluation is executed the errors e.g. should not be reset.
if (nextAction.resetErrorsBeforeExecution()) {
newResult.setNrErrors(0);
}
//
if (actionMeta.isLaunchingInParallel()) {
threadActions.add(nextAction);
Runnable runnable = () -> {
try {
Result threadResult = executeFromStart(nr + 1, newResult, nextAction, actionMeta, nextComment);
threadResults.add(threadResult);
} catch (Throwable e) {
log.logError(Const.getStackTracker(e));
threadExceptions.add(new HopException(BaseMessages.getString(PKG, "Workflow.Log.UnexpectedError", nextAction.toString()), e));
Result threadResult = new Result();
threadResult.setResult(false);
threadResult.setNrErrors(1L);
threadResults.add(threadResult);
}
};
Thread thread = new Thread(runnable);
threads.add(thread);
thread.start();
if (log.isBasic()) {
log.logBasic(BaseMessages.getString(PKG, "Workflow.Log.LaunchedActionInParallel", nextAction.getName()));
}
} else {
try {
// Same as before: blocks until it's done
//
res = executeFromStart(nr + 1, newResult, nextAction, actionMeta, nextComment);
} catch (Throwable e) {
log.logError(Const.getStackTracker(e));
throw new HopException(BaseMessages.getString(PKG, "Workflow.Log.UnexpectedError", nextAction.toString()), e);
}
if (log.isBasic()) {
log.logBasic(BaseMessages.getString(PKG, "Workflow.Log.FinishedAction", nextAction.getName(), res.getResult() + ""));
}
}
}
}
//
if (actionMeta.isLaunchingInParallel()) {
for (int i = 0; i < threads.size(); i++) {
Thread thread = threads.get(i);
ActionMeta nextAction = threadActions.get(i);
try {
thread.join();
} catch (InterruptedException e) {
log.logError(workflowMeta.toString(), BaseMessages.getString(PKG, "Workflow.Log.UnexpectedErrorWhileWaitingForAction", nextAction.getName()));
threadExceptions.add(new HopException(BaseMessages.getString(PKG, "Workflow.Log.UnexpectedErrorWhileWaitingForAction", nextAction.getName()), e));
}
}
}
// In this case, return the previous result.
if (res == null) {
res = prevResult;
}
//
if (threadExceptions.size() > 0) {
res.setResult(false);
res.setNrErrors(threadExceptions.size());
for (HopException e : threadExceptions) {
log.logError(workflowMeta.toString(), e.getMessage(), e);
}
//
throw threadExceptions.poll();
}
//
for (Result threadResult : threadResults) {
res.add(threadResult);
}
//
if (res.getNrErrors() > 0) {
res.setResult(false);
}
return res;
}
Also used :
ActionMeta(org.apache.hop.workflow.action.ActionMeta)
IAction(org.apache.hop.workflow.action.IAction)
HopException(org.apache.hop.core.exception.HopException)
HopExtensionPoint(org.apache.hop.core.extension.HopExtensionPoint)
WorkflowTracker(org.apache.hop.core.gui.WorkflowTracker)
ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue)
Example 4 with WorkflowTracker
use of org.apache.hop.core.gui.WorkflowTracker in project hop by apache.
the class Workflow method init.
private void init() {
status = new AtomicInteger();
workflowStartedListeners = Collections.synchronizedList(new ArrayList<>());
workflowFinishedListeners = Collections.synchronizedList(new ArrayList<>());
actionListeners = new ArrayList<>();
// this map is being modified concurrently and must be thread-safe
activeActions = Collections.synchronizedSet(new HashSet<>());
extensionDataMap = new HashMap<>();
workflowTracker = new WorkflowTracker(workflowMeta);
synchronized (actionResults) {
actionResults.clear();
}
errors = new AtomicInteger(0);
maxActionsLogged = Const.toInt(EnvUtil.getSystemProperty(Const.HOP_MAX_ACTIONS_LOGGED), 1000);
result = null;
startActionMeta = null;
startActionResult = null;
initializingVariablesOnStart = true;
}
Also used :
AtomicInteger(java.util.concurrent.atomic.AtomicInteger)
WorkflowTracker(org.apache.hop.core.gui.WorkflowTracker)
Example 5 with WorkflowTracker
use of org.apache.hop.core.gui.WorkflowTracker in project hop by apache.
the class Workflow method executeFromStart.
/**
* Execute a workflow without previous results. This is an action point (not recursive)<br>
* <br>
*
* @return the result of the execution
* @throws HopException
*/
private Result executeFromStart() throws HopException {
try {
log.snap(Metrics.METRIC_WORKFLOW_START);
setFinished(false);
setStopped(false);
HopEnvironment.setExecutionInformation(this);
log.logBasic(BaseMessages.getString(PKG, "Workflow.Comment.WorkflowStarted"));
ExtensionPointHandler.callExtensionPoint(log, this, HopExtensionPoint.WorkflowStart.id, this);
// Start the tracking...
ActionResult jerStart = new ActionResult(null, null, BaseMessages.getString(PKG, "Workflow.Comment.WorkflowStarted"), BaseMessages.getString(PKG, "Workflow.Reason.Started"), null, null);
workflowTracker.addWorkflowTracker(new WorkflowTracker(workflowMeta, jerStart));
setActive(true);
// Where do we start?
ActionMeta startpoint;
// synchronize this to a parent workflow if needed.
//
Object syncObject = this;
if (parentWorkflow != null) {
// parallel execution in a workflow
syncObject = parentWorkflow;
}
synchronized (syncObject) {
beginProcessing();
}
Result res = null;
if (startActionMeta == null) {
startpoint = workflowMeta.findStart();
} else {
startpoint = startActionMeta;
res = startActionResult;
}
if (startpoint == null) {
throw new HopWorkflowException(BaseMessages.getString(PKG, "Workflow.Log.CounldNotFindStartingPoint"));
}
ActionResult jerEnd = null;
if (startpoint.isStart()) {
// Perform optional looping in the special Start action...
//
boolean isFirst = true;
// Use a result obj coming from input otherwise init an empty Result object
Result inputRes = null;
if (result != null) {
inputRes = result;
} else {
inputRes = new Result();
}
// Perhaps there is already a list of input rows available?
if (getSourceRows() != null) {
inputRes.setRows(getSourceRows());
}
ActionStart jes = (ActionStart) startpoint.getAction();
while ((jes.isRepeat() || isFirst) && !isStopped()) {
isFirst = false;
res = executeFromStart(0, inputRes, startpoint, null, BaseMessages.getString(PKG, "Workflow.Reason.Started"));
}
jerEnd = new ActionResult(res, jes.getLogChannelId(), BaseMessages.getString(PKG, "Workflow.Comment.WorkflowFinished"), BaseMessages.getString(PKG, "Workflow.Reason.Finished"), null, null);
} else {
res = executeFromStart(0, res, startpoint, null, BaseMessages.getString(PKG, "Workflow.Reason.Started"));
jerEnd = new ActionResult(res, startpoint.getAction().getLogChannel().getLogChannelId(), BaseMessages.getString(PKG, "Workflow.Comment.WorkflowFinished"), BaseMessages.getString(PKG, "Workflow.Reason.Finished"), null, null);
}
// Save this result...
workflowTracker.addWorkflowTracker(new WorkflowTracker(workflowMeta, jerEnd));
log.logBasic(BaseMessages.getString(PKG, "Workflow.Comment.WorkflowFinished"));
setActive(false);
if (!isStopped()) {
setFinished(true);
}
return res;
} finally {
log.snap(Metrics.METRIC_WORKFLOW_STOP);
}
}
Also used :
ActionMeta(org.apache.hop.workflow.action.ActionMeta)
HopWorkflowException(org.apache.hop.core.exception.HopWorkflowException)
WorkflowTracker(org.apache.hop.core.gui.WorkflowTracker)
FileObject(org.apache.commons.vfs2.FileObject)
ActionStart(org.apache.hop.workflow.actions.start.ActionStart)
Aggregations
WorkflowTracker (org.apache.hop.core.gui.WorkflowTracker)7
HopException (org.apache.hop.core.exception.HopException)3
FileObject (org.apache.commons.vfs2.FileObject)2
ActionMeta (org.apache.hop.workflow.action.ActionMeta)2
BufferedInputStream (java.io.BufferedInputStream)1
File (java.io.File)1
FileOutputStream (java.io.FileOutputStream)1
IOException (java.io.IOException)1
Date (java.util.Date)1
Properties (java.util.Properties)1
ConcurrentLinkedQueue (java.util.concurrent.ConcurrentLinkedQueue)1
AtomicInteger (java.util.concurrent.atomic.AtomicInteger)1
ZipEntry (java.util.zip.ZipEntry)1
ZipOutputStream (java.util.zip.ZipOutputStream)1
DataHandler (javax.activation.DataHandler)1
FileDataSource (javax.activation.FileDataSource)1
URLDataSource (javax.activation.URLDataSource)1
InternetAddress (javax.mail.internet.InternetAddress)1
MimeBodyPart (javax.mail.internet.MimeBodyPart)1
MimeMessage (javax.mail.internet.MimeMessage)1
