Examples with IWorkflowEngine org.apache.hop.workflow.engine.IWorkflowEngine used on opensource projects

Example 1 with IWorkflowEngine

use of org.apache.hop.workflow.engine.IWorkflowEngine in project hop by apache.

the class WorkflowRunConfigurationMetadataObjectFactory method createObject.

@Override
public Object createObject(String id, Object parentObject) throws HopException {
    PluginRegistry registry = PluginRegistry.getInstance();
    IPlugin plugin = registry.findPluginWithId(WorkflowEnginePluginType.class, id);
    if (plugin == null) {
        throw new HopException("Unable to find the plugin in the context of a pipeline engine plugin for id: " + id);
    }
    try {
        // We don't return the engine but the corresponding engine configuration
        // 
        IWorkflowEngine engine = registry.loadClass(plugin, IWorkflowEngine.class);
        IWorkflowEngineRunConfiguration engineRunConfiguration = engine.createDefaultWorkflowEngineRunConfiguration();
        engineRunConfiguration.setEnginePluginId(plugin.getIds()[0]);
        engineRunConfiguration.setEnginePluginName(plugin.getName());
        if (parentObject != null && (parentObject instanceof IVariables)) {
            engineRunConfiguration.initializeFrom((IVariables) parentObject);
        }
        return engineRunConfiguration;
    } catch (HopPluginException e) {
        throw new HopException("Unable to load the workflow engine plugin class with plugin id: " + id, e);
    }
}

Example 2 with IWorkflowEngine

use of org.apache.hop.workflow.engine.IWorkflowEngine in project hop by apache.

the class LocalWorkflowEngine method startExecution.

@Override
public Result startExecution() {
    if (!(workflowRunConfiguration.getEngineRunConfiguration() instanceof LocalWorkflowRunConfiguration)) {
        log.logError("Error starting workflow", new HopException("A local workflow execution expects a local workflow configuration, not an instance of class " + workflowRunConfiguration.getEngineRunConfiguration().getClass().getName()));
        result = new Result();
        result.setNrErrors(1L);
        return result;
    }
    LocalWorkflowRunConfiguration config = (LocalWorkflowRunConfiguration) workflowRunConfiguration.getEngineRunConfiguration();
    // See if we need to enable transactions...
    // 
    IExtensionData parentExtensionData = getParentPipeline();
    if (parentExtensionData == null) {
        parentExtensionData = getParentWorkflow();
    }
    String connectionGroup = null;
    if (parentExtensionData != null && parentExtensionData.getExtensionDataMap() != null) {
        connectionGroup = (String) parentExtensionData.getExtensionDataMap().get(Const.CONNECTION_GROUP);
    }
    // 
    if (config.isTransactional() && connectionGroup == null) {
        // Store a value in the parent...
        // 
        connectionGroup = getWorkflowMeta().getName() + " - " + UUID.randomUUID();
        // We also need to commit/rollback at the end of this workflow...
        // 
        addWorkflowFinishedListener(workflow -> {
            String group = (String) workflow.getExtensionDataMap().get(Const.CONNECTION_GROUP);
            List<Database> databases = DatabaseConnectionMap.getInstance().getDatabases(group);
            Result result = workflow.getResult();
            for (Database database : databases) {
                // 
                try {
                    if (result.getResult() && !result.isStopped() && result.getNrErrors() == 0) {
                        try {
                            database.commit(true);
                            workflow.getLogChannel().logBasic("All transactions of database connection '" + database.getDatabaseMeta().getName() + "' were committed at the end of the workflow!");
                        } catch (HopDatabaseException e) {
                            workflow.getLogChannel().logError("Error committing database connection " + database.getDatabaseMeta().getName(), e);
                            result.setNrErrors(result.getNrErrors() + 1);
                        }
                    } else {
                        // Error? Rollback!
                        try {
                            database.rollback(true);
                            workflow.getLogChannel().logBasic("All transactions of database connection '" + database.getDatabaseMeta().getName() + "' were rolled back at the end of the workflow!");
                        } catch (HopDatabaseException e) {
                            workflow.getLogChannel().logError("Error rolling back database connection " + database.getDatabaseMeta().getName(), e);
                            result.setNrErrors(result.getNrErrors() + 1);
                        }
                    }
                } finally {
                    // Always close connection!
                    try {
                        database.closeConnectionOnly();
                        workflow.getLogChannel().logDebug("Database connection '" + database.getDatabaseMeta().getName() + "' closed successfully!");
                    } catch (HopDatabaseException hde) {
                        workflow.getLogChannel().logError("Error disconnecting from database - closeConnectionOnly failed:" + Const.CR + hde.getMessage());
                        workflow.getLogChannel().logError(Const.getStackTracker(hde));
                    }
                    // Definitely remove the connection reference the connections map
                    DatabaseConnectionMap.getInstance().removeConnection(group, null, database);
                }
            }
        });
    }
    // 
    if (connectionGroup != null && getExtensionDataMap() != null) {
        // Set the connection group for this workflow
        // 
        getExtensionDataMap().put(Const.CONNECTION_GROUP, connectionGroup);
    }
    // Pass down the value of the connection group value to actions before they're executed...
    // 
    addActionListener(new IActionListener() {

        @Override
        public void beforeExecution(IWorkflowEngine workflow, ActionMeta actionMeta, IAction action) {
            String connectionGroup = (String) workflow.getExtensionDataMap().get(Const.CONNECTION_GROUP);
            if (connectionGroup != null) {
                action.getExtensionDataMap().put(Const.CONNECTION_GROUP, connectionGroup);
            }
        }

        @Override
        public void afterExecution(IWorkflowEngine workflow, ActionMeta actionMeta, IAction action, Result result) {
        // Nothing
        }
    });
    return super.startExecution();
}

Example 3 with IWorkflowEngine

use of org.apache.hop.workflow.engine.IWorkflowEngine in project hop by apache.

the class WorkflowStartCheckProjectExtensionPoint method callExtensionPoint.

@Override
public void callExtensionPoint(ILogChannel log, IVariables variables, Object object) throws HopException {
    if (!(object instanceof IWorkflowEngine)) {
        return;
    }
    IWorkflowEngine workflow = (IWorkflowEngine) object;
    String filename = workflow.getFilename();
    try {
        ProjectsUtil.validateFileInProject(log, filename, workflow);
    } catch (Exception e) {
        throw new HopException("Validation error against workflow '" + filename + "' in active project", e);
    }
}

Example 4 with IWorkflowEngine

use of org.apache.hop.workflow.engine.IWorkflowEngine in project hop by apache.

the class WorkflowLoggingExtensionPoint method logEndOfWorkflow.

private void logEndOfWorkflow(final ILogChannel log, final Session session, final NeoConnection connection, final IWorkflowEngine<WorkflowMeta> workflow) throws HopException {
    log.logDetailed("Logging execution end of workflow to Neo4j connection : " + connection.getName());
    final WorkflowMeta workflowMeta = workflow.getWorkflowMeta();
    synchronized (session) {
        session.writeTransaction((TransactionWork<Void>) transaction -> {
            try {
                ILogChannel channel = workflow.getLogChannel();
                Result workflowResult = workflow.getResult();
                String workflowLogChannelId = workflow.getLogChannelId();
                String workflowLoggingText = HopLogStore.getAppender().getBuffer(workflowLogChannelId, true).toString();
                Date endDate = new Date();
                workflow.getExtensionDataMap().put(WORKFLOW_END_DATE, new Date());
                Date startDate = (Date) workflow.getExtensionDataMap().get(WORKFLOW_START_DATE);
                Map<String, Object> workflowPars = new HashMap<>();
                workflowPars.put("workflowName", workflowMeta.getName());
                workflowPars.put("type", EXECUTION_TYPE_WORKFLOW);
                workflowPars.put("id", channel.getLogChannelId());
                workflowPars.put("executionEnd", new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss").format(endDate));
                workflowPars.put("durationMs", endDate.getTime() - startDate.getTime());
                workflowPars.put("errors", workflowResult.getNrErrors());
                workflowPars.put("linesInput", workflowResult.getNrLinesInput());
                workflowPars.put("linesOutput", workflowResult.getNrLinesOutput());
                workflowPars.put("linesRead", workflowResult.getNrLinesRead());
                workflowPars.put("linesWritten", workflowResult.getNrLinesWritten());
                workflowPars.put("linesRejected", workflowResult.getNrLinesRejected());
                workflowPars.put("loggingText", workflowLoggingText);
                workflowPars.put("result", workflowResult.getResult());
                workflowPars.put("nrResultRows", workflowResult.getRows().size());
                workflowPars.put("nrResultFiles", workflowResult.getResultFilesList().size());
                StringBuilder execCypher = new StringBuilder();
                execCypher.append("MERGE (e:Execution { name : $workflowName, type : $type, id : $id } ) ");
                execCypher.append("SET ");
                execCypher.append("  e.executionEnd = $executionEnd ");
                execCypher.append(", e.durationMs = $durationMs ");
                execCypher.append(", e.errors = $errors ");
                execCypher.append(", e.linesInput = $linesInput ");
                execCypher.append(", e.linesOutput = $linesOutput ");
                execCypher.append(", e.linesRead = $linesRead ");
                execCypher.append(", e.linesWritten = $linesWritten ");
                execCypher.append(", e.linesRejected = $linesRejected ");
                execCypher.append(", e.loggingText = $loggingText ");
                execCypher.append(", e.result = $result ");
                execCypher.append(", e.nrResultRows = $nrResultRows ");
                execCypher.append(", e.nrResultFiles = $nrResultFiles ");
                transaction.run(execCypher.toString(), workflowPars);
                StringBuilder relCypher = new StringBuilder();
                relCypher.append("MATCH (w:Workflow { name : $workflowName } ) ");
                relCypher.append("MATCH (e:Execution { name : $workflowName, type : $type, id : $id } ) ");
                relCypher.append("MERGE (e)-[r:EXECUTION_OF_WORKFLOW]->(w) ");
                transaction.run(relCypher.toString(), workflowPars);
                List<ActionResult> actionResults = workflow.getActionResults();
                for (ActionResult actionResult : actionResults) {
                    String actionLogChannelId = actionResult.getLogChannelId();
                    String transformLoggingText = HopLogStore.getAppender().getBuffer(actionLogChannelId, true).toString();
                    Result result = actionResult.getResult();
                    Map<String, Object> actionPars = new HashMap<>();
                    actionPars.put("workflowName", workflowMeta.getName());
                    actionPars.put("name", actionResult.getActionName());
                    actionPars.put("type", EXECUTION_TYPE_ACTION);
                    actionPars.put("id", actionLogChannelId);
                    actionPars.put("workflowId", workflowLogChannelId);
                    actionPars.put("comment", actionResult.getComment());
                    actionPars.put("reason", actionResult.getReason());
                    actionPars.put("loggingText", transformLoggingText);
                    actionPars.put("errors", result.getNrErrors());
                    actionPars.put("linesRead", result.getNrLinesRead());
                    actionPars.put("linesWritten", result.getNrLinesWritten());
                    actionPars.put("linesInput", result.getNrLinesInput());
                    actionPars.put("linesOutput", result.getNrLinesOutput());
                    actionPars.put("linesRejected", result.getNrLinesRejected());
                    StringBuilder actionExecCypher = new StringBuilder();
                    actionExecCypher.append("MERGE (e:Execution { name : $name, type : $type, id : $id } ) ");
                    actionExecCypher.append("SET ");
                    actionExecCypher.append("  e.workflowId = $workflowId ");
                    actionExecCypher.append(", e.loggingText = $loggingText ");
                    actionExecCypher.append(", e.comment = $comment ");
                    actionExecCypher.append(", e.reason = $reason ");
                    actionExecCypher.append(", e.linesRead = $linesRead ");
                    actionExecCypher.append(", e.linesWritten = $linesWritten ");
                    actionExecCypher.append(", e.linesInput = $linesInput ");
                    actionExecCypher.append(", e.linesOutput = $linesOutput ");
                    actionExecCypher.append(", e.linesRejected = $linesRejected ");
                    transaction.run(actionExecCypher.toString(), actionPars);
                    StringBuilder actionRelCypher = new StringBuilder();
                    actionRelCypher.append("MATCH (a:Action { workflowName : $workflowName, name : $name } ) ");
                    actionRelCypher.append("MATCH (e:Execution { name : $name, type : $type, id : $id } ) ");
                    actionRelCypher.append("MERGE (e)-[r:EXECUTION_OF_ACTION]->(a) ");
                    transaction.run(actionRelCypher.toString(), actionPars);
                }
                transaction.commit();
            } catch (Exception e) {
                log.logError("Error logging workflow end", e);
                transaction.rollback();
            }
            return null;
        });
    }
}

Example 5 with IWorkflowEngine

use of org.apache.hop.workflow.engine.IWorkflowEngine in project hop by apache.

the class WorkflowLoggingExtensionPoint method callExtensionPoint.

@Override
public void callExtensionPoint(ILogChannel log, IVariables variables, IWorkflowEngine<WorkflowMeta> workflow) throws HopException {
    // 
    if (!LoggingCore.isEnabled(workflow)) {
        return;
    }
    // Keep the start date
    // 
    workflow.getExtensionDataMap().put(WORKFLOW_START_DATE, new Date());
    String connectionName = workflow.getVariable(Defaults.VARIABLE_NEO4J_LOGGING_CONNECTION);
    try {
        final NeoConnection connection = LoggingCore.getConnection(workflow.getMetadataProvider(), workflow);
        if (connection == null) {
            log.logBasic("Warning! Unable to find Neo4j connection to log to : " + connectionName);
            return;
        }
        log.logDetailed("Logging workflow information to Neo4j connection : " + connection.getName());
        final Driver driver = connection.getDriver(log, variables);
        final Session session = connection.getSession(log, driver, variables);
        logWorkflowMetadata(log, session, connection, workflow);
        logStartOfWorkflow(log, session, connection, workflow);
        workflow.addWorkflowFinishedListener(new IExecutionFinishedListener<IWorkflowEngine<WorkflowMeta>>() {

            @Override
            public void finished(IWorkflowEngine<WorkflowMeta> workflowMetaIWorkflowEngine) throws HopException {
                logEndOfWorkflow(log, session, connection, workflow);
                // 
                if (workflow.getParentWorkflow() == null && workflow.getParentPipeline() == null) {
                    String logChannelId = workflow.getLogChannelId();
                    List<LoggingHierarchy> loggingHierarchy = LoggingCore.getLoggingHierarchy(logChannelId);
                    logHierarchy(log, session, connection, loggingHierarchy, logChannelId);
                }
                // 
                if (session != null) {
                    session.close();
                }
                if (driver != null) {
                    driver.close();
                }
            }
        });
    } catch (Exception e) {
        // Let's not kill the workflow just yet, just log the error
        // otherwise: throw new HopException(...)
        // 
        log.logError("Error logging to Neo4j:", e);
    }
}