Example 1 with ActionMeta
use of org.apache.hop.workflow.action.ActionMeta in project hop by apache.
the class WorkflowActionEvalTableContentTest method setUp.
@Before
public void setUp() throws Exception {
MockDriver.registerInstance();
IWorkflowEngine<WorkflowMeta> workflow = new LocalWorkflowEngine(new WorkflowMeta());
action = new ActionEvalTableContent();
workflow.getWorkflowMeta().addAction(new ActionMeta(action));
action.setParentWorkflow(workflow);
workflow.setStopped(false);
DatabaseMeta dbMeta = new DatabaseMeta();
dbMeta.setDatabaseType("mock-db");
action.setDatabase(dbMeta);
action.setVariable(Const.HOP_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_WORKFLOW_ACTIONS, "N");
}
Also used :
ActionMeta(org.apache.hop.workflow.action.ActionMeta)
LocalWorkflowEngine(org.apache.hop.workflow.engines.local.LocalWorkflowEngine)
DatabaseMeta(org.apache.hop.core.database.DatabaseMeta)
BaseDatabaseMeta(org.apache.hop.core.database.BaseDatabaseMeta)
WorkflowMeta(org.apache.hop.workflow.WorkflowMeta)
Example 2 with ActionMeta
use of org.apache.hop.workflow.action.ActionMeta in project hop by apache.
the class WorkflowActionFilesExistTest method setUp.
@Before
public void setUp() throws Exception {
workflow = new LocalWorkflowEngine(new WorkflowMeta());
action = new ActionFilesExist();
workflow.getWorkflowMeta().addAction(new ActionMeta(action));
action.setParentWorkflow(workflow);
WorkflowMeta mockWorkflowMeta = mock(WorkflowMeta.class);
action.setParentWorkflowMeta(mockWorkflowMeta);
workflow.setStopped(false);
existingFile1 = TestUtils.createRamFile(getClass().getSimpleName() + "/existingFile1.ext", action);
existingFile2 = TestUtils.createRamFile(getClass().getSimpleName() + "/existingFile2.ext", action);
}
Also used :
ActionMeta(org.apache.hop.workflow.action.ActionMeta)
LocalWorkflowEngine(org.apache.hop.workflow.engines.local.LocalWorkflowEngine)
WorkflowMeta(org.apache.hop.workflow.WorkflowMeta)
Before(org.junit.Before)
Example 3 with ActionMeta
use of org.apache.hop.workflow.action.ActionMeta in project hop by apache.
the class AddWorkflowServlet method doGet.
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
if (isJettyMode() && !request.getRequestURI().startsWith(CONTEXT_PATH)) {
return;
}
if (log.isDebug()) {
logDebug("Addition of workflow requested");
}
boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml"));
PrintWriter out = response.getWriter();
// read from the client
BufferedReader in = request.getReader();
if (log.isDetailed()) {
logDetailed("Encoding: " + request.getCharacterEncoding());
}
if (useXML) {
response.setContentType("text/xml");
out.print(XmlHandler.getXmlHeader());
} else {
response.setContentType("text/html");
out.println("<HTML>");
out.println("<HEAD><TITLE>Add workflow</TITLE></HEAD>");
out.println("<BODY>");
}
response.setStatus(HttpServletResponse.SC_OK);
try {
// First read the complete pipeline in memory from the request
int c;
StringBuilder xml = new StringBuilder();
while ((c = in.read()) != -1) {
xml.append((char) c);
}
// Parse the XML, create a workflow configuration
//
WorkflowConfiguration workflowConfiguration = WorkflowConfiguration.fromXml(xml.toString(), variables);
IHopMetadataProvider metadataProvider = new MultiMetadataProvider(variables, getServerConfig().getMetadataProvider(), workflowConfiguration.getMetadataProvider());
WorkflowMeta workflowMeta = workflowConfiguration.getWorkflowMeta();
WorkflowExecutionConfiguration workflowExecutionConfiguration = workflowConfiguration.getWorkflowExecutionConfiguration();
String serverObjectId = UUID.randomUUID().toString();
SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.HOP_SERVER, null);
servletLoggingObject.setContainerObjectId(serverObjectId);
servletLoggingObject.setLogLevel(workflowExecutionConfiguration.getLogLevel());
// Create the workflow and store in the list...
//
String runConfigurationName = workflowExecutionConfiguration.getRunConfiguration();
final IWorkflowEngine<WorkflowMeta> workflow = WorkflowEngineFactory.createWorkflowEngine(variables, runConfigurationName, metadataProvider, workflowMeta, servletLoggingObject);
// Setting variables
//
workflow.initializeFrom(null);
workflow.getWorkflowMeta().setInternalHopVariables(workflow);
workflow.setVariables(workflowConfiguration.getWorkflowExecutionConfiguration().getVariablesMap());
// Also copy the parameters over...
//
workflow.copyParametersFromDefinitions(workflowMeta);
workflow.clearParameterValues();
String[] parameterNames = workflow.listParameters();
for (int idx = 0; idx < parameterNames.length; idx++) {
// Grab the parameter value set in the action
//
String thisValue = workflowExecutionConfiguration.getParametersMap().get(parameterNames[idx]);
if (!Utils.isEmpty(thisValue)) {
// Set the value as specified by the user in the action
//
workflow.setParameterValue(parameterNames[idx], thisValue);
}
}
workflow.activateParameters(workflow);
// Check if there is a starting point specified.
String startActionName = workflowExecutionConfiguration.getStartActionName();
if (startActionName != null && !startActionName.isEmpty()) {
ActionMeta startActionMeta = workflowMeta.findAction(startActionName);
workflow.setStartActionMeta(startActionMeta);
}
getWorkflowMap().addWorkflow(workflow.getWorkflowName(), serverObjectId, workflow, workflowConfiguration);
String message = "Workflow '" + workflow.getWorkflowName() + "' was added to the list with id " + serverObjectId;
if (useXML) {
out.println(new WebResult(WebResult.STRING_OK, message, serverObjectId));
} else {
out.println("<H1>" + message + "</H1>");
out.println("<p><a href=\"" + convertContextPath(GetWorkflowStatusServlet.CONTEXT_PATH) + "?name=" + workflow.getWorkflowName() + "&id=" + serverObjectId + "\">Go to the workflow status page</a><p>");
}
} catch (Exception ex) {
if (useXML) {
out.println(new WebResult(WebResult.STRING_ERROR, Const.getStackTracker(ex)));
} else {
out.println("<p>");
out.println("<pre>");
ex.printStackTrace(out);
out.println("</pre>");
}
}
if (!useXML) {
out.println("<p>");
out.println("</BODY>");
out.println("</HTML>");
}
}
Also used :
SimpleLoggingObject(org.apache.hop.core.logging.SimpleLoggingObject)
WorkflowExecutionConfiguration(org.apache.hop.workflow.WorkflowExecutionConfiguration)
ServletException(javax.servlet.ServletException)
IOException(java.io.IOException)
MultiMetadataProvider(org.apache.hop.metadata.serializer.multi.MultiMetadataProvider)
WorkflowMeta(org.apache.hop.workflow.WorkflowMeta)
WorkflowConfiguration(org.apache.hop.workflow.WorkflowConfiguration)
ActionMeta(org.apache.hop.workflow.action.ActionMeta)
BufferedReader(java.io.BufferedReader)
IHopMetadataProvider(org.apache.hop.metadata.api.IHopMetadataProvider)
PrintWriter(java.io.PrintWriter)
Example 4 with ActionMeta
use of org.apache.hop.workflow.action.ActionMeta in project hop by apache.
the class Workflow method executeFromStart.
/**
* Execute a action recursively and move to the next action automatically.<br>
* Uses a back-tracking algorithm.<br>
*
* @param nr
* @param previousResult
* @param actionMeta
* @param previous
* @param reason
* @return
* @throws HopException
*/
private Result executeFromStart(final int nr, Result previousResult, final ActionMeta actionMeta, ActionMeta previous, String reason) throws HopException {
Result res = null;
if (isStopped()) {
res = new Result(nr);
res.stopped = true;
return res;
}
// if we didn't have a previous result, create one, otherwise, copy the content...
//
final Result newResult;
Result prevResult = null;
if (previousResult != null) {
prevResult = previousResult.clone();
} else {
prevResult = new Result();
}
WorkflowExecutionExtension extension = new WorkflowExecutionExtension(this, prevResult, actionMeta, true);
ExtensionPointHandler.callExtensionPoint(log, this, HopExtensionPoint.WorkflowBeforeActionExecution.id, extension);
if (extension.result != null) {
prevResult = extension.result;
}
if (!extension.executeAction) {
newResult = prevResult;
} else {
if (log.isDetailed()) {
log.logDetailed("exec(" + nr + ", " + (prevResult != null ? prevResult.getNrErrors() : 0) + ", " + (actionMeta != null ? actionMeta.toString() : "null") + ")");
}
// Which entry is next?
IAction action = actionMeta.getAction();
action.getLogChannel().setLogLevel(logLevel);
// Track the fact that we are going to launch the next action...
ActionResult jerBefore = new ActionResult(null, null, BaseMessages.getString(PKG, "Workflow.Comment.WorkflowStarted"), reason, actionMeta.getName(), resolve(actionMeta.getAction().getFilename()));
workflowTracker.addWorkflowTracker(new WorkflowTracker(workflowMeta, jerBefore));
ClassLoader cl = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(action.getClass().getClassLoader());
// Execute this entry...
IAction cloneJei = (IAction) action.clone();
cloneJei.copyFrom(this);
cloneJei.getLogChannel().setLogLevel(getLogLevel());
cloneJei.setMetadataProvider(metadataProvider);
cloneJei.setParentWorkflow(this);
cloneJei.setParentWorkflowMeta(this.getWorkflowMeta());
final long start = System.currentTimeMillis();
cloneJei.getLogChannel().logDetailed("Starting action");
for (IActionListener actionListener : actionListeners) {
actionListener.beforeExecution(this, actionMeta, cloneJei);
}
if (interactive) {
getActiveActions().add(actionMeta.clone());
}
log.snap(Metrics.METRIC_ACTION_START, cloneJei.toString());
newResult = cloneJei.execute(prevResult, nr);
log.snap(Metrics.METRIC_ACTION_STOP, cloneJei.toString());
if (interactive) {
getActiveActions().remove(actionMeta);
}
for (IActionListener actionListener : actionListeners) {
actionListener.afterExecution(this, actionMeta, cloneJei, newResult);
}
Thread.currentThread().setContextClassLoader(cl);
addErrors((int) newResult.getNrErrors());
// Also capture the logging text after the execution...
//
LoggingBuffer loggingBuffer = HopLogStore.getAppender();
StringBuffer logTextBuffer = loggingBuffer.getBuffer(cloneJei.getLogChannel().getLogChannelId(), false);
newResult.setLogText(logTextBuffer.toString() + newResult.getLogText());
// Save this result as well...
//
ActionResult jerAfter = new ActionResult(newResult, cloneJei.getLogChannel().getLogChannelId(), BaseMessages.getString(PKG, "Workflow.Comment.WorkflowFinished"), null, actionMeta.getName(), resolve(actionMeta.getAction().getFilename()));
workflowTracker.addWorkflowTracker(new WorkflowTracker(workflowMeta, jerAfter));
synchronized (actionResults) {
actionResults.add(jerAfter);
//
if (maxActionsLogged > 0) {
while (actionResults.size() > maxActionsLogged) {
// Remove the oldest.
actionResults.removeFirst();
}
}
}
}
extension = new WorkflowExecutionExtension(this, prevResult, actionMeta, extension.executeAction);
ExtensionPointHandler.callExtensionPoint(log, this, HopExtensionPoint.WorkflowAfterActionExecution.id, extension);
// Try all next actions.
//
// Keep track of all the threads we fired in case of parallel execution...
// Keep track of the results of these executions too.
//
final List<Thread> threads = new ArrayList<>();
// next 2 lists is being modified concurrently so must be synchronized for this case.
final Queue<Result> threadResults = new ConcurrentLinkedQueue<>();
final Queue<HopException> threadExceptions = new ConcurrentLinkedQueue<>();
final List<ActionMeta> threadActions = new ArrayList<>();
// Launch only those where the hop indicates true or false
//
int nrNext = workflowMeta.findNrNextActions(actionMeta);
for (int i = 0; i < nrNext && !isStopped(); i++) {
// The next entry is...
final ActionMeta nextAction = workflowMeta.findNextAction(actionMeta, i);
// See if we need to execute this...
final WorkflowHopMeta hi = workflowMeta.findWorkflowHop(actionMeta, nextAction);
// The next comment...
final String nextComment;
if (hi.isUnconditional()) {
nextComment = BaseMessages.getString(PKG, "Workflow.Comment.FollowedUnconditional");
} else {
if (newResult.getResult()) {
nextComment = BaseMessages.getString(PKG, "Workflow.Comment.FollowedSuccess");
} else {
nextComment = BaseMessages.getString(PKG, "Workflow.Comment.FollowedFailure");
}
}
//
if (hi.isUnconditional() || (actionMeta.isEvaluation() && (!(hi.getEvaluation() ^ newResult.getResult())))) {
// Start this next transform!
if (log.isBasic()) {
log.logBasic(BaseMessages.getString(PKG, "Workflow.Log.StartingAction", nextAction.getName()));
}
// When an evaluation is executed the errors e.g. should not be reset.
if (nextAction.resetErrorsBeforeExecution()) {
newResult.setNrErrors(0);
}
//
if (actionMeta.isLaunchingInParallel()) {
threadActions.add(nextAction);
Runnable runnable = () -> {
try {
Result threadResult = executeFromStart(nr + 1, newResult, nextAction, actionMeta, nextComment);
threadResults.add(threadResult);
} catch (Throwable e) {
log.logError(Const.getStackTracker(e));
threadExceptions.add(new HopException(BaseMessages.getString(PKG, "Workflow.Log.UnexpectedError", nextAction.toString()), e));
Result threadResult = new Result();
threadResult.setResult(false);
threadResult.setNrErrors(1L);
threadResults.add(threadResult);
}
};
Thread thread = new Thread(runnable);
threads.add(thread);
thread.start();
if (log.isBasic()) {
log.logBasic(BaseMessages.getString(PKG, "Workflow.Log.LaunchedActionInParallel", nextAction.getName()));
}
} else {
try {
// Same as before: blocks until it's done
//
res = executeFromStart(nr + 1, newResult, nextAction, actionMeta, nextComment);
} catch (Throwable e) {
log.logError(Const.getStackTracker(e));
throw new HopException(BaseMessages.getString(PKG, "Workflow.Log.UnexpectedError", nextAction.toString()), e);
}
if (log.isBasic()) {
log.logBasic(BaseMessages.getString(PKG, "Workflow.Log.FinishedAction", nextAction.getName(), res.getResult() + ""));
}
}
}
}
//
if (actionMeta.isLaunchingInParallel()) {
for (int i = 0; i < threads.size(); i++) {
Thread thread = threads.get(i);
ActionMeta nextAction = threadActions.get(i);
try {
thread.join();
} catch (InterruptedException e) {
log.logError(workflowMeta.toString(), BaseMessages.getString(PKG, "Workflow.Log.UnexpectedErrorWhileWaitingForAction", nextAction.getName()));
threadExceptions.add(new HopException(BaseMessages.getString(PKG, "Workflow.Log.UnexpectedErrorWhileWaitingForAction", nextAction.getName()), e));
}
}
}
// In this case, return the previous result.
if (res == null) {
res = prevResult;
}
//
if (threadExceptions.size() > 0) {
res.setResult(false);
res.setNrErrors(threadExceptions.size());
for (HopException e : threadExceptions) {
log.logError(workflowMeta.toString(), e.getMessage(), e);
}
//
throw threadExceptions.poll();
}
//
for (Result threadResult : threadResults) {
res.add(threadResult);
}
//
if (res.getNrErrors() > 0) {
res.setResult(false);
}
return res;
}
Also used :
ActionMeta(org.apache.hop.workflow.action.ActionMeta)
IAction(org.apache.hop.workflow.action.IAction)
HopException(org.apache.hop.core.exception.HopException)
HopExtensionPoint(org.apache.hop.core.extension.HopExtensionPoint)
WorkflowTracker(org.apache.hop.core.gui.WorkflowTracker)
ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue)
Example 5 with ActionMeta
use of org.apache.hop.workflow.action.ActionMeta in project hop by apache.
the class Workflow method executeFromStart.
/**
* Execute a workflow without previous results. This is an action point (not recursive)<br>
* <br>
*
* @return the result of the execution
* @throws HopException
*/
private Result executeFromStart() throws HopException {
try {
log.snap(Metrics.METRIC_WORKFLOW_START);
setFinished(false);
setStopped(false);
HopEnvironment.setExecutionInformation(this);
log.logBasic(BaseMessages.getString(PKG, "Workflow.Comment.WorkflowStarted"));
ExtensionPointHandler.callExtensionPoint(log, this, HopExtensionPoint.WorkflowStart.id, this);
// Start the tracking...
ActionResult jerStart = new ActionResult(null, null, BaseMessages.getString(PKG, "Workflow.Comment.WorkflowStarted"), BaseMessages.getString(PKG, "Workflow.Reason.Started"), null, null);
workflowTracker.addWorkflowTracker(new WorkflowTracker(workflowMeta, jerStart));
setActive(true);
// Where do we start?
ActionMeta startpoint;
// synchronize this to a parent workflow if needed.
//
Object syncObject = this;
if (parentWorkflow != null) {
// parallel execution in a workflow
syncObject = parentWorkflow;
}
synchronized (syncObject) {
beginProcessing();
}
Result res = null;
if (startActionMeta == null) {
startpoint = workflowMeta.findStart();
} else {
startpoint = startActionMeta;
res = startActionResult;
}
if (startpoint == null) {
throw new HopWorkflowException(BaseMessages.getString(PKG, "Workflow.Log.CounldNotFindStartingPoint"));
}
ActionResult jerEnd = null;
if (startpoint.isStart()) {
// Perform optional looping in the special Start action...
//
boolean isFirst = true;
// Use a result obj coming from input otherwise init an empty Result object
Result inputRes = null;
if (result != null) {
inputRes = result;
} else {
inputRes = new Result();
}
// Perhaps there is already a list of input rows available?
if (getSourceRows() != null) {
inputRes.setRows(getSourceRows());
}
ActionStart jes = (ActionStart) startpoint.getAction();
while ((jes.isRepeat() || isFirst) && !isStopped()) {
isFirst = false;
res = executeFromStart(0, inputRes, startpoint, null, BaseMessages.getString(PKG, "Workflow.Reason.Started"));
}
jerEnd = new ActionResult(res, jes.getLogChannelId(), BaseMessages.getString(PKG, "Workflow.Comment.WorkflowFinished"), BaseMessages.getString(PKG, "Workflow.Reason.Finished"), null, null);
} else {
res = executeFromStart(0, res, startpoint, null, BaseMessages.getString(PKG, "Workflow.Reason.Started"));
jerEnd = new ActionResult(res, startpoint.getAction().getLogChannel().getLogChannelId(), BaseMessages.getString(PKG, "Workflow.Comment.WorkflowFinished"), BaseMessages.getString(PKG, "Workflow.Reason.Finished"), null, null);
}
// Save this result...
workflowTracker.addWorkflowTracker(new WorkflowTracker(workflowMeta, jerEnd));
log.logBasic(BaseMessages.getString(PKG, "Workflow.Comment.WorkflowFinished"));
setActive(false);
if (!isStopped()) {
setFinished(true);
}
return res;
} finally {
log.snap(Metrics.METRIC_WORKFLOW_STOP);
}
}
Also used :
ActionMeta(org.apache.hop.workflow.action.ActionMeta)
HopWorkflowException(org.apache.hop.core.exception.HopWorkflowException)
WorkflowTracker(org.apache.hop.core.gui.WorkflowTracker)
FileObject(org.apache.commons.vfs2.FileObject)
ActionStart(org.apache.hop.workflow.actions.start.ActionStart)
Aggregations
ActionMeta (org.apache.hop.workflow.action.ActionMeta)92
HopExtensionPoint (org.apache.hop.core.extension.HopExtensionPoint)34
Point (org.apache.hop.core.gui.Point)28
HopException (org.apache.hop.core.exception.HopException)23
WorkflowMeta (org.apache.hop.workflow.WorkflowMeta)21
IAction (org.apache.hop.workflow.action.IAction)12
ErrorDialog (org.apache.hop.ui.core.dialog.ErrorDialog)11
GuiContextAction (org.apache.hop.core.action.GuiContextAction)9
NotePadMeta (org.apache.hop.core.NotePadMeta)8
Test (org.junit.Test)7
WorkflowHopMeta (org.apache.hop.workflow.WorkflowHopMeta)6
IVariables (org.apache.hop.core.variables.IVariables)5
IHopMetadataProvider (org.apache.hop.metadata.api.IHopMetadataProvider)5
EnterSelectionDialog (org.apache.hop.ui.core.dialog.EnterSelectionDialog)5
HopGui (org.apache.hop.ui.hopgui.HopGui)5
LocalWorkflowEngine (org.apache.hop.workflow.engines.local.LocalWorkflowEngine)5
ArrayList (java.util.ArrayList)4
ExtensionPoint (org.apache.hop.core.extension.ExtensionPoint)4
IExtensionPoint (org.apache.hop.core.extension.IExtensionPoint)4
HashMap (java.util.HashMap)3
