Examples with WorkflowConfiguration org.apache.hop.workflow.WorkflowConfiguration used on opensource projects

Example 1 with WorkflowConfiguration

use of org.apache.hop.workflow.WorkflowConfiguration in project hop by apache.

the class AddExportServlet method doGet.

@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (isJettyMode() && !request.getRequestURI().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug("Addition of export requested");
    }
    PrintWriter out = response.getWriter();
    // read from the client
    InputStream in = request.getInputStream();
    if (log.isDetailed()) {
        logDetailed("Encoding: " + request.getCharacterEncoding());
    }
    boolean isWorkflow = TYPE_WORKFLOW.equalsIgnoreCase(request.getParameter(PARAMETER_TYPE));
    // the resource to load
    String load = request.getParameter(PARAMETER_LOAD);
    response.setContentType("text/xml");
    out.print(XmlHandler.getXmlHeader());
    response.setStatus(HttpServletResponse.SC_OK);
    OutputStream outputStream = null;
    try {
        FileObject tempFile = HopVfs.createTempFile("export", ".zip", System.getProperty("java.io.tmpdir"));
        outputStream = HopVfs.getOutputStream(tempFile, false);
        // Pass the input directly to a temporary file
        // 
        int c;
        while ((c = in.read()) != -1) {
            outputStream.write(c);
        }
        outputStream.flush();
        outputStream.close();
        // don't close it twice
        outputStream = null;
        String archiveUrl = tempFile.getName().toString();
        String fileUrl = null;
        String serverObjectId = null;
        SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.HOP_SERVER, null);
        // 
        if (!Utils.isEmpty(load)) {
            String metaStoreJson = RegisterPackageServlet.getMetaStoreJsonFromZIP("zip:" + archiveUrl + "!metadata.json");
            SerializableMetadataProvider metadataProvider = new SerializableMetadataProvider(metaStoreJson);
            fileUrl = "zip:" + archiveUrl + "!" + load;
            if (isWorkflow) {
                // Open the workflow from inside the ZIP archive
                // 
                HopVfs.getFileObject(fileUrl);
                WorkflowMeta workflowMeta = new WorkflowMeta(fileUrl);
                // Also read the execution configuration information
                // 
                String configUrl = "zip:" + archiveUrl + "!" + Workflow.CONFIGURATION_IN_EXPORT_FILENAME;
                Document configDoc = XmlHandler.loadXmlFile(configUrl);
                WorkflowExecutionConfiguration workflowExecutionConfiguration = new WorkflowExecutionConfiguration(XmlHandler.getSubNode(configDoc, WorkflowExecutionConfiguration.XML_TAG));
                serverObjectId = UUID.randomUUID().toString();
                servletLoggingObject.setContainerObjectId(serverObjectId);
                servletLoggingObject.setLogLevel(workflowExecutionConfiguration.getLogLevel());
                String runConfigurationName = workflowExecutionConfiguration.getRunConfiguration();
                // Inflate the metadata and simply store it into the workflow metadata
                // 
                workflowMeta.setMetadataProvider(metadataProvider);
                final IWorkflowEngine<WorkflowMeta> workflow = WorkflowEngineFactory.createWorkflowEngine(variables, runConfigurationName, metadataProvider, workflowMeta, servletLoggingObject);
                // store it all in the map...
                // 
                getWorkflowMap().addWorkflow(workflow.getWorkflowName(), serverObjectId, workflow, new WorkflowConfiguration(workflowMeta, workflowExecutionConfiguration, metadataProvider));
                // Apply the execution configuration...
                // 
                log.setLogLevel(workflowExecutionConfiguration.getLogLevel());
            } else {
                // Read the execution configuration information
                // 
                String configUrl = "zip:" + archiveUrl + "!" + Pipeline.CONFIGURATION_IN_EXPORT_FILENAME;
                Document configDoc = XmlHandler.loadXmlFile(configUrl);
                PipelineExecutionConfiguration executionConfiguration = new PipelineExecutionConfiguration(XmlHandler.getSubNode(configDoc, PipelineExecutionConfiguration.XML_TAG));
                // Open the pipeline from inside the ZIP archive
                // 
                PipelineMeta pipelineMeta = new PipelineMeta(fileUrl, metadataProvider, true, Variables.getADefaultVariableSpace());
                serverObjectId = UUID.randomUUID().toString();
                servletLoggingObject.setContainerObjectId(serverObjectId);
                servletLoggingObject.setLogLevel(executionConfiguration.getLogLevel());
                String runConfigurationName = executionConfiguration.getRunConfiguration();
                IPipelineEngine<PipelineMeta> pipeline = PipelineEngineFactory.createPipelineEngine(variables, runConfigurationName, metadataProvider, pipelineMeta);
                pipeline.setParent(servletLoggingObject);
                // store it all in the map...
                // 
                getPipelineMap().addPipeline(pipeline.getPipelineMeta().getName(), serverObjectId, pipeline, new PipelineConfiguration(pipelineMeta, executionConfiguration, metadataProvider));
            }
        } else {
            fileUrl = archiveUrl;
        }
        out.println(new WebResult(WebResult.STRING_OK, fileUrl, serverObjectId));
    } catch (Exception ex) {
        out.println(new WebResult(WebResult.STRING_ERROR, Const.getStackTracker(ex)));
    } finally {
        if (outputStream != null) {
            outputStream.close();
        }
    }
}

Example 2 with WorkflowConfiguration

use of org.apache.hop.workflow.WorkflowConfiguration in project hop by apache.

the class AddWorkflowServlet method doGet.

@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (isJettyMode() && !request.getRequestURI().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug("Addition of workflow requested");
    }
    boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml"));
    PrintWriter out = response.getWriter();
    // read from the client
    BufferedReader in = request.getReader();
    if (log.isDetailed()) {
        logDetailed("Encoding: " + request.getCharacterEncoding());
    }
    if (useXML) {
        response.setContentType("text/xml");
        out.print(XmlHandler.getXmlHeader());
    } else {
        response.setContentType("text/html");
        out.println("<HTML>");
        out.println("<HEAD><TITLE>Add workflow</TITLE></HEAD>");
        out.println("<BODY>");
    }
    response.setStatus(HttpServletResponse.SC_OK);
    try {
        // First read the complete pipeline in memory from the request
        int c;
        StringBuilder xml = new StringBuilder();
        while ((c = in.read()) != -1) {
            xml.append((char) c);
        }
        // Parse the XML, create a workflow configuration
        // 
        WorkflowConfiguration workflowConfiguration = WorkflowConfiguration.fromXml(xml.toString(), variables);
        IHopMetadataProvider metadataProvider = new MultiMetadataProvider(variables, getServerConfig().getMetadataProvider(), workflowConfiguration.getMetadataProvider());
        WorkflowMeta workflowMeta = workflowConfiguration.getWorkflowMeta();
        WorkflowExecutionConfiguration workflowExecutionConfiguration = workflowConfiguration.getWorkflowExecutionConfiguration();
        String serverObjectId = UUID.randomUUID().toString();
        SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.HOP_SERVER, null);
        servletLoggingObject.setContainerObjectId(serverObjectId);
        servletLoggingObject.setLogLevel(workflowExecutionConfiguration.getLogLevel());
        // Create the workflow and store in the list...
        // 
        String runConfigurationName = workflowExecutionConfiguration.getRunConfiguration();
        final IWorkflowEngine<WorkflowMeta> workflow = WorkflowEngineFactory.createWorkflowEngine(variables, runConfigurationName, metadataProvider, workflowMeta, servletLoggingObject);
        // Setting variables
        // 
        workflow.initializeFrom(null);
        workflow.getWorkflowMeta().setInternalHopVariables(workflow);
        workflow.setVariables(workflowConfiguration.getWorkflowExecutionConfiguration().getVariablesMap());
        // Also copy the parameters over...
        // 
        workflow.copyParametersFromDefinitions(workflowMeta);
        workflow.clearParameterValues();
        String[] parameterNames = workflow.listParameters();
        for (int idx = 0; idx < parameterNames.length; idx++) {
            // Grab the parameter value set in the action
            // 
            String thisValue = workflowExecutionConfiguration.getParametersMap().get(parameterNames[idx]);
            if (!Utils.isEmpty(thisValue)) {
                // Set the value as specified by the user in the action
                // 
                workflow.setParameterValue(parameterNames[idx], thisValue);
            }
        }
        workflow.activateParameters(workflow);
        // Check if there is a starting point specified.
        String startActionName = workflowExecutionConfiguration.getStartActionName();
        if (startActionName != null && !startActionName.isEmpty()) {
            ActionMeta startActionMeta = workflowMeta.findAction(startActionName);
            workflow.setStartActionMeta(startActionMeta);
        }
        getWorkflowMap().addWorkflow(workflow.getWorkflowName(), serverObjectId, workflow, workflowConfiguration);
        String message = "Workflow '" + workflow.getWorkflowName() + "' was added to the list with id " + serverObjectId;
        if (useXML) {
            out.println(new WebResult(WebResult.STRING_OK, message, serverObjectId));
        } else {
            out.println("<H1>" + message + "</H1>");
            out.println("<p><a href=\"" + convertContextPath(GetWorkflowStatusServlet.CONTEXT_PATH) + "?name=" + workflow.getWorkflowName() + "&id=" + serverObjectId + "\">Go to the workflow status page</a><p>");
        }
    } catch (Exception ex) {
        if (useXML) {
            out.println(new WebResult(WebResult.STRING_ERROR, Const.getStackTracker(ex)));
        } else {
            out.println("<p>");
            out.println("<pre>");
            ex.printStackTrace(out);
            out.println("</pre>");
        }
    }
    if (!useXML) {
        out.println("<p>");
        out.println("</BODY>");
        out.println("</HTML>");
    }
}

Example 3 with WorkflowConfiguration

use of org.apache.hop.workflow.WorkflowConfiguration in project hop by apache.

the class StartWorkflowServlet method doGet.

@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug(BaseMessages.getString(PKG, "StartWorkflowServlet.Log.StartWorkflowRequested"));
    }
    String workflowName = request.getParameter("name");
    String id = request.getParameter("id");
    boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml"));
    response.setStatus(HttpServletResponse.SC_OK);
    PrintWriter out = response.getWriter();
    if (useXML) {
        response.setContentType("text/xml");
        response.setCharacterEncoding(Const.XML_ENCODING);
        out.print(XmlHandler.getXmlHeader(Const.XML_ENCODING));
    } else {
        response.setContentType("text/html;charset=UTF-8");
        out.println("<HTML>");
        out.println("<HEAD>");
        out.println("<TITLE>Start workflow</TITLE>");
        out.println("<META http-equiv=\"Refresh\" content=\"2;url=" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(workflowName, "UTF-8") + "\">");
        out.println("<META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">");
        out.println("</HEAD>");
        out.println("<BODY>");
    }
    try {
        // ID is optional...
        // 
        IWorkflowEngine<WorkflowMeta> workflow = getWorkflowMap().findWorkflow(workflowName, id);
        if (workflow != null) {
            // 
            if (workflow.isInitialized() && !workflow.isActive()) {
                // 
                synchronized (this) {
                    WorkflowConfiguration workflowConfiguration = getWorkflowMap().getConfiguration(workflowName);
                    IHopMetadataProvider metadataProvider = new MultiMetadataProvider(variables, getServerConfig().getMetadataProvider(), workflowConfiguration.getMetadataProvider());
                    // This new workflow execution engine instance needs a new container ID...
                    // 
                    String containerId = UUID.randomUUID().toString();
                    SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.HOP_SERVER, null);
                    servletLoggingObject.setContainerObjectId(containerId);
                    String runConfigurationName = workflowConfiguration.getWorkflowExecutionConfiguration().getRunConfiguration();
                    IWorkflowEngine<WorkflowMeta> newWorkflow = WorkflowEngineFactory.createWorkflowEngine(variables, runConfigurationName, metadataProvider, workflow.getWorkflowMeta(), servletLoggingObject);
                    newWorkflow.setLogLevel(workflow.getLogLevel());
                    // Variables are set in AddWorkflowServlet
                    // 
                    newWorkflow.activateParameters(newWorkflow);
                    // Set the new container ID in the workflow...
                    // 
                    newWorkflow.setContainerId(containerId);
                    // Discard old log lines from the old workflow
                    // 
                    HopLogStore.discardLines(workflow.getLogChannelId(), true);
                    getWorkflowMap().replaceWorkflow(workflow, newWorkflow, workflowConfiguration);
                    workflow = newWorkflow;
                }
            }
            runWorkflow(workflow);
            String message = BaseMessages.getString(PKG, "StartWorkflowServlet.Log.WorkflowStarted", workflowName);
            if (useXML) {
                out.println(new WebResult(WebResult.STRING_OK, message, workflow.getContainerId()).getXml());
            } else {
                out.println("<H1>" + Encode.forHtml(message) + "</H1>");
                out.println("<a href=\"" + convertContextPath(GetWorkflowStatusServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(workflowName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">" + BaseMessages.getString(PKG, "WorkflowStatusServlet.BackToWorkflowStatusPage") + "</a><p>");
            }
        } else {
            String message = BaseMessages.getString(PKG, "StartWorkflowServlet.Log.SpecifiedWorkflowNotFound", workflowName);
            if (useXML) {
                out.println(new WebResult(WebResult.STRING_ERROR, message));
            } else {
                out.println("<H1>" + Encode.forHtml(message) + "</H1>");
                out.println("<a href=\"" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "\">" + BaseMessages.getString(PKG, "PipelineStatusServlet.BackToStatusPage") + "</a><p>");
                response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
            }
        }
    } catch (Exception ex) {
        if (useXML) {
            out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "StartWorkflowServlet.Error.UnexpectedError", Const.CR + Const.getStackTracker(ex))));
        } else {
            out.println("<p>");
            out.println("<pre>");
            out.println(Encode.forHtml(Const.getStackTracker(ex)));
            out.println("</pre>");
            response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
        }
    }
    if (!useXML) {
        out.println("<p>");
        out.println("</BODY>");
        out.println("</HTML>");
    }
}

Example 4 with WorkflowConfiguration

use of org.apache.hop.workflow.WorkflowConfiguration in project hop by apache.

the class RegisterWorkflowServlet method generateBody.

@Override
WebResult generateBody(HttpServletRequest request, HttpServletResponse response, boolean useXml, IVariables variables) throws IOException, HopException, HopException, ParseException {
    final String xml = IOUtils.toString(request.getInputStream());
    // Parse the XML, create a workflow configuration
    WorkflowConfiguration workflowConfiguration = WorkflowConfiguration.fromXml(xml, variables);
    IWorkflowEngine<WorkflowMeta> workflow = createWorkflow(workflowConfiguration);
    String message = "Workflow '" + workflow.getWorkflowName() + "' was added to the list with id " + workflow.getContainerId();
    return new WebResult(WebResult.STRING_OK, message, workflow.getContainerId());
}

Example 5 with WorkflowConfiguration

use of org.apache.hop.workflow.WorkflowConfiguration in project hop by apache.

the class AsyncRunServlet method doGet.

@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) {
    if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug(BaseMessages.getString(PKG, "AsyncRunServlet.Log.AsyncRunRequested"));
    }
    IVariables variables = pipelineMap.getHopServerConfig().getVariables();
    MultiMetadataProvider metadataProvider = new MultiMetadataProvider(Encr.getEncoder(), new ArrayList<>(), variables);
    metadataProvider.getProviders().add(HopMetadataUtil.getStandardHopMetadataProvider(variables));
    String metadataFolder = pipelineMap.getHopServerConfig().getMetadataFolder();
    if (StringUtils.isNotEmpty(metadataFolder)) {
        // Get the metadata from the specified metadata folder...
        // 
        metadataProvider.getProviders().add(new JsonMetadataProvider(Encr.getEncoder(), metadataFolder, variables));
    }
    String webServiceName = request.getParameter("service");
    if (StringUtils.isEmpty(webServiceName)) {
        log.logError("Please specify a service parameter pointing to the name of the asynchronous webservice object");
    }
    try {
        IHopMetadataSerializer<AsyncWebService> serializer = metadataProvider.getSerializer(AsyncWebService.class);
        AsyncWebService webService = serializer.load(webServiceName);
        if (webService == null) {
            throw new HopException("Unable to find asynchronous web service '" + webServiceName + "'.  You can set option metadata_folder in the Hop server XML configuration");
        }
        if (!webService.isEnabled()) {
            throw new HopException("Asynchronous Web service '" + webServiceName + "' is disabled.");
        }
        String filename = variables.resolve(webService.getFilename());
        // We give back the ID of the executing workflow...
        // 
        response.setContentType("application/json");
        response.setCharacterEncoding(Const.XML_ENCODING);
        String serverObjectId = UUID.randomUUID().toString();
        SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.HOP_SERVER, null);
        servletLoggingObject.setContainerObjectId(serverObjectId);
        // Load and start the workflow
        // Output the ID to the response output stream...
        // 
        WorkflowMeta workflowMeta = new WorkflowMeta(variables, filename, metadataProvider);
        LocalWorkflowEngine workflow = new LocalWorkflowEngine(workflowMeta, servletLoggingObject);
        workflow.setContainerId(serverObjectId);
        workflow.setMetadataProvider(metadataProvider);
        workflow.setLogLevel(LogLevel.BASIC);
        workflow.initializeFrom(variables);
        workflow.setVariable("SERVER_OBJECT_ID", serverObjectId);
        // See if we need to pass a variable with the content in it...
        // 
        // Read the content posted?
        // 
        String contentVariable = variables.resolve(webService.getBodyContentVariable());
        String content = "";
        if (StringUtils.isNotEmpty(contentVariable)) {
            try (InputStream in = request.getInputStream()) {
                try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
                    byte[] buffer = new byte[1024];
                    int length;
                    while ((length = in.read(buffer)) != -1) {
                        outputStream.write(buffer, 0, length);
                    }
                    outputStream.flush();
                    // Now we have the content...
                    // 
                    content = new String(outputStream.toByteArray(), StandardCharsets.UTF_8);
                }
            }
            workflow.setVariable(contentVariable, Const.NVL(content, ""));
        }
        // Set all the other parameters as variables/parameters...
        // 
        String[] pipelineParameters = workflowMeta.listParameters();
        workflow.copyParametersFromDefinitions(workflowMeta);
        for (String requestParameter : request.getParameterMap().keySet()) {
            if ("service".equals(requestParameter)) {
                continue;
            }
            String requestParameterValue = request.getParameter(requestParameter);
            if (Const.indexOfString(requestParameter, pipelineParameters) < 0) {
                workflow.setVariable(requestParameter, Const.NVL(requestParameterValue, ""));
            } else {
                try {
                    workflow.setParameterValue(requestParameter, Const.NVL(requestParameterValue, ""));
                } catch (UnknownParamException e) {
                    log.logError("Error running asynchronous web service", e);
                }
            }
        }
        workflow.activateParameters(workflow);
        // Add the workflow to the status map, so we can retrieve statuses later on
        // 
        WorkflowExecutionConfiguration workflowExecutionConfiguration = new WorkflowExecutionConfiguration();
        WorkflowConfiguration workflowConfiguration = new WorkflowConfiguration(workflowMeta, workflowExecutionConfiguration, new SerializableMetadataProvider(metadataProvider));
        // We use the service name to store the workflow under!
        // That way we don't have to look up the name of the workflow when retrieving the status.
        // 
        getWorkflowMap().addWorkflow(webServiceName, serverObjectId, workflow, workflowConfiguration);
        // Allocate the workflow in the background...
        // 
        new Thread(workflow::startExecution).start();
        try (OutputStream outputStream = response.getOutputStream()) {
            // Report the ID in a JSON block
            // 
            JSONObject json = new JSONObject();
            json.put("name", workflowMeta.getName());
            json.put("id", serverObjectId);
            String jsonString = json.toJSONString();
            outputStream.write(jsonString.getBytes(StandardCharsets.UTF_8));
            outputStream.write("\n".getBytes(StandardCharsets.UTF_8));
            outputStream.flush();
        } catch (IOException e) {
            log.logError("Error running asynchronous web service", e);
        }
        response.setStatus(HttpServletResponse.SC_OK);
    } catch (IOException | HopException e) {
        log.logError("Error running asynchronous web service", e);
    }
}