Examples with PipelineExecutionConfiguration org.apache.hop.pipeline.PipelineExecutionConfiguration used on opensource projects

Example 1 with PipelineExecutionConfiguration

use of org.apache.hop.pipeline.PipelineExecutionConfiguration in project hop by apache.

the class AddExportServlet method doGet.

@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (isJettyMode() && !request.getRequestURI().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug("Addition of export requested");
    }
    PrintWriter out = response.getWriter();
    // read from the client
    InputStream in = request.getInputStream();
    if (log.isDetailed()) {
        logDetailed("Encoding: " + request.getCharacterEncoding());
    }
    boolean isWorkflow = TYPE_WORKFLOW.equalsIgnoreCase(request.getParameter(PARAMETER_TYPE));
    // the resource to load
    String load = request.getParameter(PARAMETER_LOAD);
    response.setContentType("text/xml");
    out.print(XmlHandler.getXmlHeader());
    response.setStatus(HttpServletResponse.SC_OK);
    OutputStream outputStream = null;
    try {
        FileObject tempFile = HopVfs.createTempFile("export", ".zip", System.getProperty("java.io.tmpdir"));
        outputStream = HopVfs.getOutputStream(tempFile, false);
        // Pass the input directly to a temporary file
        // 
        int c;
        while ((c = in.read()) != -1) {
            outputStream.write(c);
        }
        outputStream.flush();
        outputStream.close();
        // don't close it twice
        outputStream = null;
        String archiveUrl = tempFile.getName().toString();
        String fileUrl = null;
        String serverObjectId = null;
        SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.HOP_SERVER, null);
        // 
        if (!Utils.isEmpty(load)) {
            String metaStoreJson = RegisterPackageServlet.getMetaStoreJsonFromZIP("zip:" + archiveUrl + "!metadata.json");
            SerializableMetadataProvider metadataProvider = new SerializableMetadataProvider(metaStoreJson);
            fileUrl = "zip:" + archiveUrl + "!" + load;
            if (isWorkflow) {
                // Open the workflow from inside the ZIP archive
                // 
                HopVfs.getFileObject(fileUrl);
                WorkflowMeta workflowMeta = new WorkflowMeta(fileUrl);
                // Also read the execution configuration information
                // 
                String configUrl = "zip:" + archiveUrl + "!" + Workflow.CONFIGURATION_IN_EXPORT_FILENAME;
                Document configDoc = XmlHandler.loadXmlFile(configUrl);
                WorkflowExecutionConfiguration workflowExecutionConfiguration = new WorkflowExecutionConfiguration(XmlHandler.getSubNode(configDoc, WorkflowExecutionConfiguration.XML_TAG));
                serverObjectId = UUID.randomUUID().toString();
                servletLoggingObject.setContainerObjectId(serverObjectId);
                servletLoggingObject.setLogLevel(workflowExecutionConfiguration.getLogLevel());
                String runConfigurationName = workflowExecutionConfiguration.getRunConfiguration();
                // Inflate the metadata and simply store it into the workflow metadata
                // 
                workflowMeta.setMetadataProvider(metadataProvider);
                final IWorkflowEngine<WorkflowMeta> workflow = WorkflowEngineFactory.createWorkflowEngine(variables, runConfigurationName, metadataProvider, workflowMeta, servletLoggingObject);
                // store it all in the map...
                // 
                getWorkflowMap().addWorkflow(workflow.getWorkflowName(), serverObjectId, workflow, new WorkflowConfiguration(workflowMeta, workflowExecutionConfiguration, metadataProvider));
                // Apply the execution configuration...
                // 
                log.setLogLevel(workflowExecutionConfiguration.getLogLevel());
            } else {
                // Read the execution configuration information
                // 
                String configUrl = "zip:" + archiveUrl + "!" + Pipeline.CONFIGURATION_IN_EXPORT_FILENAME;
                Document configDoc = XmlHandler.loadXmlFile(configUrl);
                PipelineExecutionConfiguration executionConfiguration = new PipelineExecutionConfiguration(XmlHandler.getSubNode(configDoc, PipelineExecutionConfiguration.XML_TAG));
                // Open the pipeline from inside the ZIP archive
                // 
                PipelineMeta pipelineMeta = new PipelineMeta(fileUrl, metadataProvider, true, Variables.getADefaultVariableSpace());
                serverObjectId = UUID.randomUUID().toString();
                servletLoggingObject.setContainerObjectId(serverObjectId);
                servletLoggingObject.setLogLevel(executionConfiguration.getLogLevel());
                String runConfigurationName = executionConfiguration.getRunConfiguration();
                IPipelineEngine<PipelineMeta> pipeline = PipelineEngineFactory.createPipelineEngine(variables, runConfigurationName, metadataProvider, pipelineMeta);
                pipeline.setParent(servletLoggingObject);
                // store it all in the map...
                // 
                getPipelineMap().addPipeline(pipeline.getPipelineMeta().getName(), serverObjectId, pipeline, new PipelineConfiguration(pipelineMeta, executionConfiguration, metadataProvider));
            }
        } else {
            fileUrl = archiveUrl;
        }
        out.println(new WebResult(WebResult.STRING_OK, fileUrl, serverObjectId));
    } catch (Exception ex) {
        out.println(new WebResult(WebResult.STRING_ERROR, Const.getStackTracker(ex)));
    } finally {
        if (outputStream != null) {
            outputStream.close();
        }
    }
}

Example 2 with PipelineExecutionConfiguration

use of org.apache.hop.pipeline.PipelineExecutionConfiguration in project hop by apache.

the class AddPipelineServlet method doGet.

@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (isJettyMode() && !request.getRequestURI().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug("Addition of pipeline requested");
    }
    boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml"));
    PrintWriter out = response.getWriter();
    BufferedReader in = request.getReader();
    if (log.isDetailed()) {
        logDetailed("Encoding: " + request.getCharacterEncoding());
    }
    if (useXML) {
        response.setContentType("text/xml");
        out.print(XmlHandler.getXmlHeader());
    } else {
        response.setContentType("text/html");
        out.println("<HTML>");
        out.println("<HEAD><TITLE>Add pipeline</TITLE></HEAD>");
        out.println("<BODY>");
    }
    response.setStatus(HttpServletResponse.SC_OK);
    String realLogFilename = null;
    PipelineExecutionConfiguration pipelineExecutionConfiguration = null;
    try {
        // First read the complete pipeline in memory from the request
        // 
        StringBuilder xml = new StringBuilder(request.getContentLength());
        int c;
        while ((c = in.read()) != -1) {
            xml.append((char) c);
        }
        // Parse the XML, create a pipeline configuration
        // 
        PipelineConfiguration pipelineConfiguration = PipelineConfiguration.fromXml(xml.toString());
        PipelineMeta pipelineMeta = pipelineConfiguration.getPipelineMeta();
        pipelineExecutionConfiguration = pipelineConfiguration.getPipelineExecutionConfiguration();
        if (log.isDetailed()) {
            logDetailed("Logging level set to " + log.getLogLevel().getDescription());
        }
        String serverObjectId = UUID.randomUUID().toString();
        SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.HOP_SERVER, null);
        servletLoggingObject.setContainerObjectId(serverObjectId);
        servletLoggingObject.setLogLevel(pipelineExecutionConfiguration.getLogLevel());
        IHopMetadataProvider metadataProvider = new MultiMetadataProvider(variables, getServerConfig().getMetadataProvider(), pipelineConfiguration.getMetadataProvider());
        String runConfigurationName = pipelineExecutionConfiguration.getRunConfiguration();
        final IPipelineEngine<PipelineMeta> pipeline = PipelineEngineFactory.createPipelineEngine(variables, runConfigurationName, metadataProvider, pipelineMeta);
        pipeline.setParent(servletLoggingObject);
        if (pipelineExecutionConfiguration.isSetLogfile()) {
            realLogFilename = pipelineExecutionConfiguration.getLogFileName();
            final LogChannelFileWriter logChannelFileWriter;
            try {
                FileUtil.createParentFolder(AddPipelineServlet.class, realLogFilename, pipelineExecutionConfiguration.isCreateParentFolder(), pipeline.getLogChannel());
                logChannelFileWriter = new LogChannelFileWriter(servletLoggingObject.getLogChannelId(), HopVfs.getFileObject(realLogFilename), pipelineExecutionConfiguration.isSetAppendLogfile());
                logChannelFileWriter.startLogging();
                pipeline.addExecutionFinishedListener(pipelineEngine -> {
                    if (logChannelFileWriter != null) {
                        logChannelFileWriter.stopLogging();
                    }
                });
            } catch (HopException e) {
                logError(Const.getStackTracker(e));
            }
        }
        getPipelineMap().addPipeline(pipelineMeta.getName(), serverObjectId, pipeline, pipelineConfiguration);
        pipeline.setContainerId(serverObjectId);
        String message = "Pipeline '" + pipeline.getPipelineMeta().getName() + "' was added to HopServer with id " + serverObjectId;
        if (useXML) {
            // Return the log channel id as well
            // 
            out.println(new WebResult(WebResult.STRING_OK, message, serverObjectId));
        } else {
            out.println("<H1>" + message + "</H1>");
            out.println("<p><a href=\"" + convertContextPath(GetPipelineStatusServlet.CONTEXT_PATH) + "?name=" + pipeline.getPipelineMeta().getName() + "&id=" + serverObjectId + "\">Go to the pipeline status page</a><p>");
        }
    } catch (Exception ex) {
        if (useXML) {
            out.println(new WebResult(WebResult.STRING_ERROR, Const.getStackTracker(ex)));
        } else {
            out.println("<p>");
            out.println("<pre>");
            ex.printStackTrace(out);
            out.println("</pre>");
        }
    }
    if (!useXML) {
        out.println("<p>");
        out.println("</BODY>");
        out.println("</HTML>");
    }
}

Example 3 with PipelineExecutionConfiguration

use of org.apache.hop.pipeline.PipelineExecutionConfiguration in project hop by apache.

the class WebServiceServlet method doGet.

@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug(BaseMessages.getString(PKG, "WebServiceServlet.Log.WebServiceRequested"));
    }
    IVariables variables = pipelineMap.getHopServerConfig().getVariables();
    IHopMetadataProvider metadataProvider = pipelineMap.getHopServerConfig().getMetadataProvider();
    String webServiceName = request.getParameter("service");
    if (StringUtils.isEmpty(webServiceName)) {
        throw new ServletException("Please specify a service parameter pointing to the name of the web service object");
    }
    try {
        IHopMetadataSerializer<WebService> serializer = metadataProvider.getSerializer(WebService.class);
        WebService webService = serializer.load(webServiceName);
        if (webService == null) {
            throw new HopException("Unable to find web service '" + webServiceName + "'.  You can set the metadata_folder in the Hop server XML configuration");
        }
        if (!webService.isEnabled()) {
            throw new HopException("Web service '" + webServiceName + "' is disabled.");
        }
        String filename = variables.resolve(webService.getFilename());
        String transformName = variables.resolve(webService.getTransformName());
        String fieldName = variables.resolve(webService.getFieldName());
        String contentType = variables.resolve(webService.getContentType());
        if (StringUtils.isEmpty(contentType)) {
            response.setContentType("text/plain");
        } else {
            response.setContentType(contentType);
        }
        response.setCharacterEncoding(Const.XML_ENCODING);
        String serverObjectId = UUID.randomUUID().toString();
        SimpleLoggingObject servletLoggingObject = new SimpleLoggingObject(CONTEXT_PATH, LoggingObjectType.HOP_SERVER, null);
        servletLoggingObject.setContainerObjectId(serverObjectId);
        // Load and start the pipeline
        // Output the data to the response output stream...
        // 
        PipelineMeta pipelineMeta = new PipelineMeta(filename, metadataProvider, true, variables);
        LocalPipelineEngine pipeline = new LocalPipelineEngine(pipelineMeta, variables, servletLoggingObject);
        pipeline.setContainerId(serverObjectId);
        // Set all the other parameters as variables/parameters...
        // 
        String[] pipelineParameters = pipelineMeta.listParameters();
        pipeline.copyParametersFromDefinitions(pipelineMeta);
        for (String requestParameter : request.getParameterMap().keySet()) {
            if ("service".equals(requestParameter)) {
                continue;
            }
            String requestParameterValue = request.getParameter(requestParameter);
            if (Const.indexOfString(requestParameter, pipelineParameters) < 0) {
                pipeline.setVariable(requestParameter, Const.NVL(requestParameterValue, ""));
            } else {
                pipeline.setParameterValue(requestParameter, Const.NVL(requestParameterValue, ""));
            }
        }
        pipeline.activateParameters(pipeline);
        // 
        if (webService.isListingStatus()) {
            PipelineExecutionConfiguration pipelineExecutionConfiguration = new PipelineExecutionConfiguration();
            PipelineConfiguration pipelineConfiguration = new PipelineConfiguration(pipelineMeta, pipelineExecutionConfiguration, new SerializableMetadataProvider(metadataProvider));
            getPipelineMap().addPipeline(pipelineMeta.getName(), serverObjectId, pipeline, pipelineConfiguration);
        }
        // Allocate the threads...
        pipeline.prepareExecution();
        final OutputStream outputStream = response.getOutputStream();
        // Add the row listener to the transform/field...
        // TODO: add to all copies
        // 
        IEngineComponent component = pipeline.findComponent(transformName, 0);
        component.addRowListener(new RowAdapter() {

            @Override
            public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) throws HopTransformException {
                try {
                    String outputString = rowMeta.getString(row, fieldName, "");
                    outputStream.write(outputString.getBytes(StandardCharsets.UTF_8));
                    outputStream.flush();
                } catch (HopValueException e) {
                    throw new HopTransformException("Error getting output field '" + fieldName + " from row: " + rowMeta.toStringMeta(), e);
                } catch (IOException e) {
                    throw new HopTransformException("Error writing output of '" + fieldName + "'", e);
                }
            }
        });
        pipeline.startThreads();
        pipeline.waitUntilFinished();
        response.setStatus(HttpServletResponse.SC_OK);
    } catch (Exception e) {
        throw new ServletException("Error producing web service output", e);
    }
}

Example 4 with PipelineExecutionConfiguration

use of org.apache.hop.pipeline.PipelineExecutionConfiguration in project hop by apache.

the class PrepareExecutionPipelineServletTest method testPausePipelineServletEscapesHtmlWhenPipelineFound.

@Test
@PrepareForTest({ Encode.class })
public void testPausePipelineServletEscapesHtmlWhenPipelineFound() throws ServletException, IOException {
    HopLogStore.init();
    HttpServletRequest mockHttpServletRequest = mock(HttpServletRequest.class);
    HttpServletResponse mockHttpServletResponse = mock(HttpServletResponse.class);
    Pipeline mockPipeline = mock(Pipeline.class);
    PipelineConfiguration mockPipelineConf = mock(PipelineConfiguration.class);
    PipelineMeta mockPipelineMeta = mock(PipelineMeta.class);
    PipelineExecutionConfiguration mockPipelineExecutionConf = mock(PipelineExecutionConfiguration.class);
    ILogChannel mockChannelInterface = mock(ILogChannel.class);
    StringWriter out = new StringWriter();
    PrintWriter printWriter = new PrintWriter(out);
    PowerMockito.spy(Encode.class);
    when(mockHttpServletRequest.getContextPath()).thenReturn(PrepareExecutionPipelineServlet.CONTEXT_PATH);
    when(mockHttpServletRequest.getParameter(anyString())).thenReturn(ServletTestUtils.BAD_STRING_TO_TEST);
    when(mockHttpServletResponse.getWriter()).thenReturn(printWriter);
    when(mockPipelineMap.getPipeline(any(HopServerObjectEntry.class))).thenReturn(mockPipeline);
    when(mockPipelineMap.getConfiguration(any(HopServerObjectEntry.class))).thenReturn(mockPipelineConf);
    when(mockPipelineConf.getPipelineExecutionConfiguration()).thenReturn(mockPipelineExecutionConf);
    when(mockPipeline.getLogChannel()).thenReturn(mockChannelInterface);
    when(mockPipeline.getPipelineMeta()).thenReturn(mockPipelineMeta);
    when(mockPipelineMeta.getMaximum()).thenReturn(new Point(10, 10));
    prepareExecutionPipelineServlet.doGet(mockHttpServletRequest, mockHttpServletResponse);
    assertFalse(ServletTestUtils.hasBadText(ServletTestUtils.getInsideOfTag("H1", out.toString())));
    PowerMockito.verifyStatic(Encode.class);
    Encode.forHtml(anyString());
}

Example 5 with PipelineExecutionConfiguration

use of org.apache.hop.pipeline.PipelineExecutionConfiguration in project hop by apache.

the class HopGuiPipelineRunDelegate method executePipeline.

public PipelineExecutionConfiguration executePipeline(final ILogChannel log, final PipelineMeta pipelineMeta, final boolean preview, final boolean debug, LogLevel logLevel) throws HopException {
    if (pipelineMeta == null) {
        return null;
    }
    // See if we need to ask for debugging information...
    // 
    PipelineDebugMeta pipelineDebugMeta = null;
    PipelineExecutionConfiguration executionConfiguration = null;
    if (preview) {
        executionConfiguration = getPipelinePreviewExecutionConfiguration();
    } else if (debug) {
        executionConfiguration = getPipelineDebugExecutionConfiguration();
    } else {
        executionConfiguration = getPipelineExecutionConfiguration();
    }
    // Set MetaStore and safe mode information in both the exec config and the metadata
    // 
    pipelineMeta.setMetadataProvider(hopGui.getMetadataProvider());
    if (debug) {
        // See if we have debugging information stored somewhere?
        // 
        pipelineDebugMeta = pipelineDebugMetaMap.get(pipelineMeta);
        if (pipelineDebugMeta == null) {
            pipelineDebugMeta = new PipelineDebugMeta(pipelineMeta);
            pipelineDebugMetaMap.put(pipelineMeta, pipelineDebugMeta);
        }
        // Set the default number of rows to retrieve on all selected transforms...
        // 
        List<TransformMeta> selectedTransforms = pipelineMeta.getSelectedTransforms();
        if (selectedTransforms != null && selectedTransforms.size() > 0) {
            pipelineDebugMeta.getTransformDebugMetaMap().clear();
            for (TransformMeta transformMeta : pipelineMeta.getSelectedTransforms()) {
                TransformDebugMeta transformDebugMeta = new TransformDebugMeta(transformMeta);
                transformDebugMeta.setRowCount(PropsUi.getInstance().getDefaultPreviewSize());
                transformDebugMeta.setPausingOnBreakPoint(true);
                transformDebugMeta.setReadingFirstRows(false);
                pipelineDebugMeta.getTransformDebugMetaMap().put(transformMeta, transformDebugMeta);
            }
        }
    } else if (preview) {
        // See if we have preview information stored somewhere?
        // 
        pipelineDebugMeta = pipelinePreviewMetaMap.get(pipelineMeta);
        if (pipelineDebugMeta == null) {
            pipelineDebugMeta = new PipelineDebugMeta(pipelineMeta);
            pipelinePreviewMetaMap.put(pipelineMeta, pipelineDebugMeta);
        }
        // Set the default number of preview rows on all selected transforms...
        // 
        List<TransformMeta> selectedTransforms = pipelineMeta.getSelectedTransforms();
        if (selectedTransforms != null && selectedTransforms.size() > 0) {
            pipelineDebugMeta.getTransformDebugMetaMap().clear();
            for (TransformMeta transformMeta : pipelineMeta.getSelectedTransforms()) {
                TransformDebugMeta transformDebugMeta = new TransformDebugMeta(transformMeta);
                transformDebugMeta.setRowCount(PropsUi.getInstance().getDefaultPreviewSize());
                transformDebugMeta.setPausingOnBreakPoint(false);
                transformDebugMeta.setReadingFirstRows(true);
                pipelineDebugMeta.getTransformDebugMetaMap().put(transformMeta, transformDebugMeta);
            }
        }
    }
    int debugAnswer = PipelineDebugDialog.DEBUG_CONFIG;
    if (debug || preview) {
        PipelineDebugDialog pipelineDebugDialog = new PipelineDebugDialog(hopGui.getShell(), pipelineGraph.getVariables(), pipelineDebugMeta);
        debugAnswer = pipelineDebugDialog.open();
        if (debugAnswer == PipelineDebugDialog.DEBUG_CANCEL) {
            // 
            return null;
        }
    }
    Map<String, String> variableMap = new HashMap<>();
    // the default
    variableMap.putAll(executionConfiguration.getVariablesMap());
    executionConfiguration.setVariablesMap(variableMap);
    executionConfiguration.getUsedVariables(pipelineGraph.getVariables(), pipelineMeta);
    executionConfiguration.setLogLevel(logLevel);
    boolean execConfigAnswer = true;
    if (debugAnswer == PipelineDebugDialog.DEBUG_CONFIG && pipelineMeta.isShowDialog()) {
        PipelineExecutionConfigurationDialog dialog = new PipelineExecutionConfigurationDialog(hopGui.getShell(), executionConfiguration, pipelineMeta);
        execConfigAnswer = dialog.open();
    }
    if (execConfigAnswer) {
        pipelineGraph.pipelineGridDelegate.addPipelineGrid();
        pipelineGraph.pipelineLogDelegate.addPipelineLog();
        pipelineGraph.extraViewTabFolder.setSelection(0);
        // Set the run options
        pipelineMeta.setClearingLog(executionConfiguration.isClearingLog());
        ExtensionPointHandler.callExtensionPoint(log, pipelineGraph.getVariables(), HopExtensionPoint.HopGuiPipelineMetaExecutionStart.id, pipelineMeta);
        ExtensionPointHandler.callExtensionPoint(log, pipelineGraph.getVariables(), HopExtensionPoint.HopGuiPipelineExecutionConfiguration.id, executionConfiguration);
        // distinction
        if (debug || preview) {
            if (pipelineDebugMeta.getNrOfUsedTransforms() == 0) {
                MessageBox box = new MessageBox(hopGui.getShell(), SWT.ICON_WARNING | SWT.YES | SWT.NO);
                box.setText(BaseMessages.getString(PKG, "HopGui.Dialog.Warning.NoPreviewOrDebugTransforms.Title"));
                box.setMessage(BaseMessages.getString(PKG, "HopGui.Dialog.Warning.NoPreviewOrDebugTransforms.Message"));
                int answer = box.open();
                if (answer != SWT.YES) {
                    return null;
                }
            }
            pipelineGraph.debug(executionConfiguration, pipelineDebugMeta);
        } else {
            pipelineGraph.start(executionConfiguration);
        }
    }
    return executionConfiguration;
}