Examples with WorkflowMeta org.apache.hop.workflow.WorkflowMeta used on opensource projects

Example 21 with WorkflowMeta

use of org.apache.hop.workflow.WorkflowMeta in project hop by apache.

the class AsyncStatusServlet method doGet.

public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug(BaseMessages.getString(PKG, "AsyncStatusServlet.Log.AsyncStatusRequested"));
    }
    IVariables variables = pipelineMap.getHopServerConfig().getVariables();
    MultiMetadataProvider metadataProvider = new MultiMetadataProvider(Encr.getEncoder(), new ArrayList<>(), variables);
    metadataProvider.getProviders().add(HopMetadataUtil.getStandardHopMetadataProvider(variables));
    String metadataFolder = pipelineMap.getHopServerConfig().getMetadataFolder();
    if (StringUtils.isNotEmpty(metadataFolder)) {
        // Get the metadata from the specified metadata folder...
        // 
        metadataProvider.getProviders().add(new JsonMetadataProvider(Encr.getEncoder(), metadataFolder, variables));
    }
    String webServiceName = request.getParameter("service");
    if (StringUtils.isEmpty(webServiceName)) {
        throw new ServletException("Please specify a service parameter pointing to the name of the asynchronous webservice object");
    }
    String serverObjectId = request.getParameter("id");
    if (StringUtils.isEmpty(serverObjectId)) {
        throw new ServletException("Please specify an id parameter pointing to the unique ID of the asynchronous webservice object");
    }
    try {
        // Load the web service metadata
        // 
        IHopMetadataSerializer<AsyncWebService> serializer = metadataProvider.getSerializer(AsyncWebService.class);
        AsyncWebService webService = serializer.load(webServiceName);
        if (webService == null) {
            throw new HopException("Unable to find asynchronous web service '" + webServiceName + "'.  You can set option metadata_folder in the Hop server XML configuration");
        }
        // Get the workflow...
        // 
        IWorkflowEngine<WorkflowMeta> workflow = workflowMap.findWorkflow(webServiceName, serverObjectId);
        // Report back in JSON format
        // 
        AsyncStatus status = new AsyncStatus();
        status.setService(webServiceName);
        status.setId(serverObjectId);
        status.setStartDate(workflow.getExecutionStartDate());
        status.setEndDate(workflow.getExecutionEndDate());
        status.setStatusDescription(workflow.getStatusDescription());
        // 
        for (String statusVariable : webService.getStatusVariablesList(variables)) {
            String statusValue = workflow.getVariable(statusVariable);
            status.getStatusVariables().put(statusVariable, statusValue);
        }
        // 
        for (Object dataValue : workflow.getExtensionDataMap().values()) {
            if (dataValue instanceof HopServerPipelineStatus) {
                status.getPipelineStatuses().add((HopServerPipelineStatus) dataValue);
            }
        }
        // We give back all this information about the executing workflow in JSON format...
        // 
        response.setContentType("application/json");
        response.setCharacterEncoding(Const.XML_ENCODING);
        final OutputStream outputStream = response.getOutputStream();
        ObjectMapper mapper = new ObjectMapper();
        String jsonString = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(status);
        byte[] data = jsonString.getBytes(StandardCharsets.UTF_8);
        response.setContentLength(data.length);
        outputStream.write(data);
        outputStream.flush();
        response.setStatus(HttpServletResponse.SC_OK);
    } catch (Exception e) {
        throw new ServletException("Error getting asynchronous web service status", e);
    }
}

Example 22 with WorkflowMeta

use of org.apache.hop.workflow.WorkflowMeta in project hop by apache.

the class AsyncWebServiceEditor method createWorkflowFile.

/**
 * Create a new workflow file with a note to explain what's going on
 *
 * @param parent
 */
private void createWorkflowFile(Composite parent) {
    try {
        // Create an empty workflow...
        // 
        WorkflowMeta workflowMeta = new WorkflowMeta();
        // Add a note explaining what's going on.
        // 
        NotePadMeta note = new NotePadMeta("This workflow can set status variables which are picked up when you request the status of this workflow." + Const.CR + "You can use service asyncStatus to query the status.", 150, 350, -1, -1);
        workflowMeta.addNote(note);
        // Save it...
        // 
        HopWorkflowFileType<WorkflowMeta> type = new HopWorkflowFileType<>();
        String filename = BaseDialog.presentFileDialog(// save
        true, parent.getShell(), wFilename, manager.getVariables(), type.getFilterExtensions(), type.getFilterNames(), true);
        if (filename != null) {
            // User specified a pipeline filename
            // 
            String realFilename = manager.getVariables().resolve(filename);
            workflowMeta.setFilename(realFilename);
            workflowMeta.clearChanged();
            HopDataOrchestrationPerspective perspective = HopGui.getDataOrchestrationPerspective();
            // Switch to the perspective
            // 
            perspective.activate();
            // Open it in the Hop GUI
            // 
            HopGui.getDataOrchestrationPerspective().addWorkflow(hopGui, workflowMeta, type);
            // Save the file
            hopGui.fileDelegate.fileSave();
        }
    } catch (Exception e) {
        new ErrorDialog(parent.getShell(), "Error", "Error creating workflow", e);
    }
}

Example 23 with WorkflowMeta

use of org.apache.hop.workflow.WorkflowMeta in project hop by apache.

the class HopServerSingleton method installPurgeTimer.

public static void installPurgeTimer(final HopServerConfig config, final ILogChannel log, final PipelineMap pipelineMap, final WorkflowMap workflowMap) {
    final int objectTimeout;
    String systemTimeout = EnvUtil.getSystemProperty(Const.HOP_SERVER_OBJECT_TIMEOUT_MINUTES, null);
    // 
    if (config.getObjectTimeoutMinutes() > 0) {
        objectTimeout = config.getObjectTimeoutMinutes();
    } else if (!Utils.isEmpty(systemTimeout)) {
        objectTimeout = Const.toInt(systemTimeout, 1440);
    } else {
        // 1440 : default is a one day time-out
        objectTimeout = 24 * 60;
    }
    // 
    if (objectTimeout > 0) {
        log.logBasic("Installing timer to purge stale objects after " + objectTimeout + " minutes.");
        Timer timer = new Timer(true);
        final AtomicBoolean busy = new AtomicBoolean(false);
        TimerTask timerTask = new TimerTask() {

            @Override
            public void run() {
                if (!busy.get()) {
                    busy.set(true);
                    try {
                        // 
                        for (HopServerObjectEntry entry : pipelineMap.getPipelineObjects()) {
                            IPipelineEngine<PipelineMeta> pipeline = pipelineMap.getPipeline(entry);
                            // 
                            if (pipeline != null && (pipeline.isFinished() || pipeline.isStopped()) && pipeline.getExecutionStartDate() != null) {
                                // check the last log time
                                // 
                                int diffInMinutes = (int) Math.floor((System.currentTimeMillis() - pipeline.getExecutionStartDate().getTime()) / 60000);
                                if (diffInMinutes >= objectTimeout) {
                                    // Let's remove this from the pipeline map...
                                    // 
                                    pipelineMap.removePipeline(entry);
                                    // Remove the logging information from the log registry & central log store
                                    // 
                                    LoggingRegistry.getInstance().removeIncludingChildren(pipeline.getLogChannelId());
                                    HopLogStore.discardLines(pipeline.getLogChannelId(), false);
                                    log.logBasic("Cleaned up pipeline " + entry.getName() + " with id " + entry.getId() + " from " + pipeline.getExecutionStartDate() + ", diff=" + diffInMinutes);
                                }
                            }
                        }
                        // 
                        for (HopServerObjectEntry entry : workflowMap.getWorkflowObjects()) {
                            IWorkflowEngine<WorkflowMeta> workflow = workflowMap.getWorkflow(entry);
                            // 
                            if (workflow != null && (workflow.isFinished() || workflow.isStopped()) && workflow.getExecutionStartDate() != null) {
                                // check the last log time
                                // 
                                int diffInMinutes = (int) Math.floor((System.currentTimeMillis() - workflow.getExecutionStartDate().getTime()) / 60000);
                                if (diffInMinutes >= objectTimeout) {
                                    // Let's remove this from the workflow map...
                                    // 
                                    String id = workflowMap.getWorkflow(entry).getLogChannelId();
                                    LoggingRegistry.getInstance().removeLogChannelFileWriterBuffer(id);
                                    workflowMap.removeWorkflow(entry);
                                    log.logBasic("Cleaned up workflow " + entry.getName() + " with id " + entry.getId() + " from " + workflow.getExecutionStartDate());
                                }
                            }
                        }
                    } finally {
                        busy.set(false);
                    }
                }
            }
        };
        // Search for stale objects every 20 seconds:
        // 
        timer.schedule(timerTask, 20000, 20000);
    }
}

Example 24 with WorkflowMeta

use of org.apache.hop.workflow.WorkflowMeta in project hop by apache.

the class RemoveWorkflowServlet method doGet.

@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug(BaseMessages.getString(PKG, "RemoveWorkflowServlet.Log.RemoveWorkflowRequested"));
    }
    String workflowName = request.getParameter("name");
    String id = request.getParameter("id");
    boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml"));
    response.setStatus(HttpServletResponse.SC_OK);
    if (useXML) {
        response.setContentType("text/xml");
        response.setCharacterEncoding(Const.XML_ENCODING);
    } else {
        response.setContentType("text/html;charset=UTF-8");
    }
    PrintWriter out = response.getWriter();
    // ID is optional...
    // 
    IWorkflowEngine<WorkflowMeta> workflow;
    HopServerObjectEntry entry;
    if (Utils.isEmpty(id)) {
        // get the first pipeline that matches...
        // 
        entry = getWorkflowMap().getFirstHopServerObjectEntry(workflowName);
        if (entry == null) {
            workflow = null;
        } else {
            id = entry.getId();
            workflow = getWorkflowMap().getWorkflow(entry);
        }
    } else {
        // Take the ID into account!
        // 
        entry = new HopServerObjectEntry(workflowName, id);
        workflow = getWorkflowMap().getWorkflow(entry);
    }
    if (workflow != null) {
        HopLogStore.discardLines(workflow.getLogChannelId(), true);
        getWorkflowMap().removeWorkflow(entry);
        if (useXML) {
            response.setContentType("text/xml");
            response.setCharacterEncoding(Const.XML_ENCODING);
            out.print(XmlHandler.getXmlHeader(Const.XML_ENCODING));
            out.print(WebResult.OK.getXml());
        } else {
            response.setContentType("text/html;charset=UTF-8");
            out.println("<HTML>");
            out.println("<HEAD>");
            out.println("<TITLE>" + BaseMessages.getString(PKG, "RemoveWorkflowServlet.WorkflowRemoved") + "</TITLE>");
            out.println("<META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">");
            out.println("</HEAD>");
            out.println("<BODY>");
            out.println("<H3>" + Encode.forHtml(BaseMessages.getString(PKG, "RemoveWorkflowServlet.TheWorkflowWasRemoved", workflowName, id)) + "</H3>");
            out.print("<a href=\"" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "\">" + BaseMessages.getString(PKG, "PipelineStatusServlet.BackToStatusPage") + "</a><br>");
            out.println("<p>");
            out.println("</BODY>");
            out.println("</HTML>");
        }
    } else {
        if (useXML) {
            out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "RemoveWorkflowServlet.Log.CoundNotFindSpecWorkflow", workflowName)));
        } else {
            out.println("<H1>" + Encode.forHtml(BaseMessages.getString(PKG, "RemoveWorkflowServlet.WorkflowRemoved.Log.CoundNotFindWorkflow", workflowName, id)) + "</H1>");
            out.println("<a href=\"" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "\">" + BaseMessages.getString(PKG, "PipelineStatusServlet.BackToStatusPage") + "</a><p>");
            response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
        }
    }
}

Example 25 with WorkflowMeta

use of org.apache.hop.workflow.WorkflowMeta in project hop by apache.

the class StartWorkflowServlet method doGet.

@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug(BaseMessages.getString(PKG, "StartWorkflowServlet.Log.StartWorkflowRequested"));
    }
    String workflowName = request.getParameter("name");
    String id = request.getParameter("id");
    boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml"));
    response.setStatus(HttpServletResponse.SC_OK);
    PrintWriter out = response.getWriter();
    if (useXML) {
        response.setContentType("text/xml");
        response.setCharacterEncoding(Const.XML_ENCODING);
        out.print(XmlHandler.getXmlHeader(Const.XML_ENCODING));
    } else {
        response.setContentType("text/html;charset=UTF-8");
        out.println("<HTML>");
        out.println("<HEAD>");
        out.println("<TITLE>Start workflow</TITLE>");
        out.println("<META http-equiv=\"Refresh\" content=\"2;url=" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(workflowName, "UTF-8") + "\">");
        out.println("<META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">");
        out.println("</HEAD>");
        out.println("<BODY>");
    }
    try {
        // ID is optional...
        // 
        IWorkflowEngine<WorkflowMeta> workflow = getWorkflowMap().findWorkflow(workflowName, id);
        if (workflow != null) {
            // 
            if (workflow.isInitialized() && !workflow.isActive()) {
                // 
                synchronized (this) {
                    WorkflowConfiguration workflowConfiguration = getWorkflowMap().getConfiguration(workflowName);
                    IHopMetadataProvider metadataProvider = new MultiMetadataProvider(variables, getServerConfig().getMetadataProvider(), workflowConfiguration.getMetadataProvider());
                    // This new workflow execution engine instance needs a new container ID...
                    // 
                    String containerId = UUID.randomUUID().toString();
                    SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.HOP_SERVER, null);
                    servletLoggingObject.setContainerObjectId(containerId);
                    String runConfigurationName = workflowConfiguration.getWorkflowExecutionConfiguration().getRunConfiguration();
                    IWorkflowEngine<WorkflowMeta> newWorkflow = WorkflowEngineFactory.createWorkflowEngine(variables, runConfigurationName, metadataProvider, workflow.getWorkflowMeta(), servletLoggingObject);
                    newWorkflow.setLogLevel(workflow.getLogLevel());
                    // Variables are set in AddWorkflowServlet
                    // 
                    newWorkflow.activateParameters(newWorkflow);
                    // Set the new container ID in the workflow...
                    // 
                    newWorkflow.setContainerId(containerId);
                    // Discard old log lines from the old workflow
                    // 
                    HopLogStore.discardLines(workflow.getLogChannelId(), true);
                    getWorkflowMap().replaceWorkflow(workflow, newWorkflow, workflowConfiguration);
                    workflow = newWorkflow;
                }
            }
            runWorkflow(workflow);
            String message = BaseMessages.getString(PKG, "StartWorkflowServlet.Log.WorkflowStarted", workflowName);
            if (useXML) {
                out.println(new WebResult(WebResult.STRING_OK, message, workflow.getContainerId()).getXml());
            } else {
                out.println("<H1>" + Encode.forHtml(message) + "</H1>");
                out.println("<a href=\"" + convertContextPath(GetWorkflowStatusServlet.CONTEXT_PATH) + "?name=" + URLEncoder.encode(workflowName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">" + BaseMessages.getString(PKG, "WorkflowStatusServlet.BackToWorkflowStatusPage") + "</a><p>");
            }
        } else {
            String message = BaseMessages.getString(PKG, "StartWorkflowServlet.Log.SpecifiedWorkflowNotFound", workflowName);
            if (useXML) {
                out.println(new WebResult(WebResult.STRING_ERROR, message));
            } else {
                out.println("<H1>" + Encode.forHtml(message) + "</H1>");
                out.println("<a href=\"" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "\">" + BaseMessages.getString(PKG, "PipelineStatusServlet.BackToStatusPage") + "</a><p>");
                response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
            }
        }
    } catch (Exception ex) {
        if (useXML) {
            out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "StartWorkflowServlet.Error.UnexpectedError", Const.CR + Const.getStackTracker(ex))));
        } else {
            out.println("<p>");
            out.println("<pre>");
            out.println(Encode.forHtml(Const.getStackTracker(ex)));
            out.println("</pre>");
            response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
        }
    }
    if (!useXML) {
        out.println("<p>");
        out.println("</BODY>");
        out.println("</HTML>");
    }
}