Examples with StepInterface org.pentaho.di.trans.step.StepInterface used on opensource projects

Example 71 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class SniffStepServlet method doGet.

/**
 *<div id="mindtouch">
 *    <h1>/kettle/sniffStep</h1>
 *    <a name="GET"></a>
 *    <h2>GET</h2>
 *    <p>Sniff metadata and data from the specified step of the specified transformation.</p>
 *
 *    <p><b>Example Request:</b><br />
 *    <pre function="syntax.xml">
 *    GET /kettle/sniffStep?trans=dummy-trans&step=tf&xml=Y&lines=10
 *    </pre>
 *
 *    </p>
 *    <h3>Parameters</h3>
 *    <table class="pentaho-table">
 *    <tbody>
 *    <tr>
 *      <th>name</th>
 *      <th>description</th>
 *      <th>type</th>
 *    </tr>
 *    <tr>
 *    <td>trans</td>
 *    <td>Name of the transformation containing required step.</td>
 *    <td>query</td>
 *    </tr>
 *    <tr>
 *    <td>stepName</td>
 *    <td>Name of the transformation step to collect data for.</td>
 *    <td>query</td>
 *    </tr>
 *    <tr>
 *    <td>copynr</td>
 *    <td>Copy number of the step to be used for collecting data. If not provided 0 is used.</td>
 *    <td>integer, optional</td>
 *    </tr>
 *    <tr>
 *    <td>type</td>
 *    <td>Type of the data to be collected (<code>input</code> or <code>output</code>).
 *    If not provided output data is collected.</td>
 *    <td>query, optional</td>
 *    </tr>
 *    <tr>
 *    <td>xml</td>
 *    <td>Boolean flag which defines output format <code>Y</code> forces XML output to be generated.
 *  HTML is returned otherwise.</td>
 *    <td>boolean, optional</td>
 *    </tr>
 *    <tr>
 *    <td>id</td>
 *    <td>Carte id of the transformation to be used for step lookup.</td>
 *    <td>query, optional</td>
 *    </tr>
 *    <tr>
 *    <td>lines</td>
 *    <td>Number of lines to collect and include into response. If not provided 0 lines will be collected.</td>
 *    <td>integer, optional</td>
 *    </tr>
 *    </tbody>
 *    </table>
 *
 *  <h3>Response Body</h3>
 *
 *  <table class="pentaho-table">
 *    <tbody>
 *      <tr>
 *        <td align="right">element:</td>
 *        <td>(custom)</td>
 *      </tr>
 *      <tr>
 *        <td align="right">media types:</td>
 *        <td>text/xml, text/html</td>
 *      </tr>
 *    </tbody>
 *  </table>
 *    <p>Response XML or HTML response containing data and metadata of the step.
 *  If an error occurs during method invocation <code>result</code> field of the response
 *  will contain <code>ERROR</code> status.</p>
 *
 *    <p><b>Example Response:</b></p>
 *    <pre function="syntax.xml">
 *    <?xml version="1.0" encoding="UTF-8"?>
 *    <step-sniff>
 *      <row-meta>
 *        <value-meta><type>String</type>
 *          <storagetype>normal</storagetype>
 *          <name>Field1</name>
 *          <length>0</length>
 *          <precision>-1</precision>
 *          <origin>tf</origin>
 *          <comments/>
 *          <conversion_Mask/>
 *          <decimal_symbol>.</decimal_symbol>
 *          <grouping_symbol>,</grouping_symbol>
 *          <currency_symbol>&#x24;</currency_symbol>
 *          <trim_type>none</trim_type>
 *          <case_insensitive>N</case_insensitive>
 *          <sort_descending>N</sort_descending>
 *          <output_padding>N</output_padding>
 *          <date_format_lenient>Y</date_format_lenient>
 *          <date_format_locale>en_US</date_format_locale>
 *          <date_format_timezone>America&#x2f;Bahia</date_format_timezone>
 *          <lenient_string_to_number>N</lenient_string_to_number>
 *        </value-meta>
 *      </row-meta>
 *      <nr_rows>10</nr_rows>
 *
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *      <row-data><value-data>my-data </value-data>
 *      </row-data>
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *    </step-sniff>
 *    </pre>
 *
 *    <h3>Status Codes</h3>
 *    <table class="pentaho-table">
 *  <tbody>
 *    <tr>
 *      <th>code</th>
 *      <th>description</th>
 *    </tr>
 *    <tr>
 *      <td>200</td>
 *      <td>Request was processed.</td>
 *    </tr>
 *    <tr>
 *      <td>500</td>
 *      <td>Internal server error occurs during request processing.</td>
 *    </tr>
 *  </tbody>
 *</table>
 *</div>
 */
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug(BaseMessages.getString(PKG, "TransStatusServlet.Log.SniffStepRequested"));
    }
    String transName = request.getParameter("trans");
    String id = request.getParameter("id");
    String stepName = request.getParameter("step");
    int copyNr = Const.toInt(request.getParameter("copynr"), 0);
    final int nrLines = Const.toInt(request.getParameter("lines"), 0);
    String type = Const.NVL(request.getParameter("type"), TYPE_OUTPUT);
    boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml"));
    response.setStatus(HttpServletResponse.SC_OK);
    if (useXML) {
        response.setContentType("text/xml");
        response.setCharacterEncoding(Const.XML_ENCODING);
    } else {
        response.setContentType("text/html;charset=UTF-8");
    }
    PrintWriter out = response.getWriter();
    // ID is optional...
    // 
    Trans trans;
    CarteObjectEntry entry;
    if (Utils.isEmpty(id)) {
        // get the first transformation that matches...
        // 
        entry = getTransformationMap().getFirstCarteObjectEntry(transName);
        if (entry == null) {
            trans = null;
        } else {
            id = entry.getId();
            trans = getTransformationMap().getTransformation(entry);
        }
    } else {
        // Take the ID into account!
        // 
        entry = new CarteObjectEntry(transName, id);
        trans = getTransformationMap().getTransformation(entry);
    }
    if (trans != null) {
        // Find the step to look at...
        // 
        StepInterface step = null;
        List<StepInterface> stepInterfaces = trans.findBaseSteps(stepName);
        for (int i = 0; i < stepInterfaces.size(); i++) {
            StepInterface look = stepInterfaces.get(i);
            if (look.getCopy() == copyNr) {
                step = look;
            }
        }
        if (step != null) {
            // Add a listener to the transformation step...
            // 
            final boolean read = type.equalsIgnoreCase(TYPE_INPUT);
            final boolean written = type.equalsIgnoreCase(TYPE_OUTPUT) || !read;
            final MetaAndData metaData = new MetaAndData();
            metaData.bufferRowMeta = null;
            metaData.bufferRowData = new ArrayList<Object[]>();
            RowListener rowListener = new RowListener() {

                public void rowReadEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
                    if (read && metaData.bufferRowData.size() < nrLines) {
                        metaData.bufferRowMeta = rowMeta;
                        metaData.bufferRowData.add(row);
                    }
                }

                public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
                    if (written && metaData.bufferRowData.size() < nrLines) {
                        metaData.bufferRowMeta = rowMeta;
                        metaData.bufferRowData.add(row);
                    }
                }

                public void errorRowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
                }
            };
            step.addRowListener(rowListener);
            // 
            while (metaData.bufferRowData.size() < nrLines && step.isRunning() && !trans.isFinished() && !trans.isStopped()) {
                try {
                    Thread.sleep(100);
                } catch (InterruptedException e) {
                    // 
                    break;
                }
            }
            // Remove the row listener
            // 
            step.removeRowListener(rowListener);
            // 
            if (useXML) {
                // Send the result back as XML
                // 
                response.setContentType("text/xml");
                response.setCharacterEncoding(Const.XML_ENCODING);
                out.print(XMLHandler.getXMLHeader(Const.XML_ENCODING));
                out.println(XMLHandler.openTag(XML_TAG));
                if (metaData.bufferRowMeta != null) {
                    // Row Meta data
                    // 
                    out.println(metaData.bufferRowMeta.getMetaXML());
                    // Nr of lines
                    // 
                    out.println(XMLHandler.addTagValue("nr_rows", metaData.bufferRowData.size()));
                    // 
                    for (int i = 0; i < metaData.bufferRowData.size(); i++) {
                        Object[] rowData = metaData.bufferRowData.get(i);
                        out.println(metaData.bufferRowMeta.getDataXML(rowData));
                    }
                }
                out.println(XMLHandler.closeTag(XML_TAG));
            } else {
                response.setContentType("text/html;charset=UTF-8");
                out.println("<HTML>");
                out.println("<HEAD>");
                out.println("<TITLE>" + BaseMessages.getString(PKG, "SniffStepServlet.SniffResults") + "</TITLE>");
                out.println("<META http-equiv=\"Refresh\" content=\"10;url=" + convertContextPath(CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">");
                out.println("<META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">");
                out.println("</HEAD>");
                out.println("<BODY>");
                out.println("<H1>" + Encode.forHtml(BaseMessages.getString(PKG, "SniffStepServlet.SniffResultsForStep", stepName)) + "</H1>");
                try {
                    out.println("<table border=\"1\">");
                    if (metaData.bufferRowMeta != null) {
                        // Print a header row containing all the field names...
                        // 
                        out.print("<tr><th>#</th>");
                        for (ValueMetaInterface valueMeta : metaData.bufferRowMeta.getValueMetaList()) {
                            out.print("<th>" + valueMeta.getName() + "</th>");
                        }
                        out.println("</tr>");
                        // 
                        for (int r = 0; r < metaData.bufferRowData.size(); r++) {
                            Object[] rowData = metaData.bufferRowData.get(r);
                            out.print("<tr>");
                            out.println("<td>" + (r + 1) + "</td>");
                            for (int v = 0; v < metaData.bufferRowMeta.size(); v++) {
                                ValueMetaInterface valueMeta = metaData.bufferRowMeta.getValueMeta(v);
                                Object valueData = rowData[v];
                                out.println("<td>" + valueMeta.getString(valueData) + "</td>");
                            }
                            out.println("</tr>");
                        }
                    }
                    out.println("</table>");
                    out.println("<p>");
                } catch (Exception ex) {
                    out.println("<p>");
                    out.println("<pre>");
                    out.println(Encode.forHtml(Const.getStackTracker(ex)));
                    out.println("</pre>");
                }
                out.println("<p>");
                out.println("</BODY>");
                out.println("</HTML>");
            }
        } else {
            if (useXML) {
                out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "SniffStepServlet.Log.CoundNotFindSpecStep", stepName)).getXML());
            } else {
                out.println("<H1>" + Encode.forHtml(BaseMessages.getString(PKG, "SniffStepServlet.Log.CoundNotFindSpecStep", stepName)) + "</H1>");
                out.println("<a href=\"" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.BackToStatusPage") + "</a><p>");
            }
        }
    } else {
        if (useXML) {
            out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "SniffStepServlet.Log.CoundNotFindSpecTrans", transName)).getXML());
        } else {
            out.println("<H1>" + Encode.forHtml(BaseMessages.getString(PKG, "SniffStepServlet.Log.CoundNotFindTrans", transName)) + "</H1>");
            out.println("<a href=\"" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.BackToStatusPage") + "</a><p>");
        }
    }
}

Example 72 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class SingleThreader method prepareMappingExecution.

public void prepareMappingExecution() throws KettleException {
    SingleThreaderData singleThreaderData = getData();
    // Set the type to single threaded in case the user forgot...
    // 
    singleThreaderData.mappingTransMeta.setTransformationType(TransformationType.SingleThreaded);
    // Create the transformation from meta-data...
    singleThreaderData.mappingTrans = new Trans(singleThreaderData.mappingTransMeta, getTrans());
    // Pass the parameters down to the sub-transformation.
    // 
    StepWithMappingMeta.activateParams(getData().mappingTrans, getData().mappingTrans, this, getData().mappingTrans.listParameters(), meta.getParameters(), meta.getParameterValues());
    getData().mappingTrans.activateParameters();
    // Disable thread priority managment as it will slow things down needlessly.
    // The single threaded engine doesn't use threads and doesn't need row locking.
    // 
    singleThreaderData.mappingTrans.getTransMeta().setUsingThreadPriorityManagment(false);
    // Leave a path up so that we can set variables in sub-transformations...
    // 
    singleThreaderData.mappingTrans.setParentTrans(getTrans());
    // Pass down the safe mode flag to the mapping...
    // 
    singleThreaderData.mappingTrans.setSafeModeEnabled(getTrans().isSafeModeEnabled());
    // Pass down the metrics gathering flag to the mapping...
    // 
    singleThreaderData.mappingTrans.setGatheringMetrics(getTrans().isGatheringMetrics());
    // Also set the name of this step in the mapping transformation for logging purposes
    // 
    singleThreaderData.mappingTrans.setMappingStepName(getStepname());
    initServletConfig();
    // prepare the execution
    // 
    singleThreaderData.mappingTrans.prepareExecution(null);
    // 
    if (singleThreaderData.injectStepMeta.isMappingInput()) {
        MappingInputData mappingInputData = (MappingInputData) singleThreaderData.mappingTrans.findDataInterface(singleThreaderData.injectStepMeta.getName());
        mappingInputData.sourceSteps = new StepInterface[0];
        mappingInputData.valueRenames = new ArrayList<MappingValueRename>();
    }
    // Add row producer & row listener
    singleThreaderData.rowProducer = singleThreaderData.mappingTrans.addRowProducer(meta.getInjectStep(), 0);
    StepInterface retrieveStep = singleThreaderData.mappingTrans.getStepInterface(meta.getRetrieveStep(), 0);
    retrieveStep.addRowListener(new RowAdapter() {

        @Override
        public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
            // Simply pass it along to the next steps after the SingleThreader
            // 
            SingleThreader.this.putRow(rowMeta, row);
        }
    });
    singleThreaderData.mappingTrans.startThreads();
    // Create the executor...
    singleThreaderData.executor = new SingleThreadedTransExecutor(singleThreaderData.mappingTrans);
    // 
    try {
        boolean ok = singleThreaderData.executor.init();
        if (!ok) {
            throw new KettleException(BaseMessages.getString(PKG, "SingleThreader.Exception.UnableToInitSingleThreadedTransformation"));
        }
    } catch (KettleException e) {
        throw new KettleException(BaseMessages.getString(PKG, "SingleThreader.Exception.UnableToPrepareExecutionOfMapping"), e);
    }
    // Add the mapping transformation to the active sub-transformations map in the parent transformation
    // 
    getTrans().addActiveSubTransformation(getStepname(), singleThreaderData.mappingTrans);
}

Example 73 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class StepsMetrics method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (StepsMetricsMeta) smi;
    data = (StepsMetricsData) sdi;
    if (first) {
        first = false;
        String[] stepnames = null;
        int stepnrs = 0;
        if (meta.getStepName() != null && meta.getStepName().length > 0) {
            stepnames = meta.getStepName();
            stepnrs = stepnames.length;
        } else {
            // error
            throw new KettleException(BaseMessages.getString(PKG, "StepsMetrics.Error.NotSteps"));
        }
        // check for output fields
        data.realstepnamefield = environmentSubstitute(meta.getStepNameFieldName());
        data.realstepidfield = environmentSubstitute(meta.getStepIdFieldName());
        data.realsteplinesinputfield = environmentSubstitute(meta.getStepLinesInputFieldName());
        data.realsteplinesoutputfield = environmentSubstitute(meta.getStepLinesOutputFieldName());
        data.realsteplinesreadfield = environmentSubstitute(meta.getStepLinesReadFieldName());
        data.realsteplineswrittentfield = environmentSubstitute(meta.getStepLinesWrittenFieldName());
        data.realsteplinesupdatedfield = environmentSubstitute(meta.getStepLinesUpdatedFieldName());
        data.realsteplineserrorsfield = environmentSubstitute(meta.getStepLinesErrorsFieldName());
        data.realstepsecondsfield = environmentSubstitute(meta.getStepSecondsFieldName());
        // Get target stepnames
        String[] targetSteps = getTransMeta().getNextStepNames(getStepMeta());
        data.stepInterfaces = new ConcurrentHashMap<Integer, StepInterface>();
        for (int i = 0; i < stepnrs; i++) {
            // We can not get metrics from current step
            if (stepnames[i].equals(getStepname())) {
                throw new KettleException("You can not get metrics for the current step [" + stepnames[i] + "]!");
            }
            if (targetSteps != null) {
                // We can not metrics from the target steps
                for (int j = 0; j < targetSteps.length; j++) {
                    if (stepnames[i].equals(targetSteps[j])) {
                        throw new KettleException("You can not get metrics for the target step [" + targetSteps[j] + "]!");
                    }
                }
            }
            int CopyNr = Const.toInt(meta.getStepCopyNr()[i], 0);
            StepInterface si = getTrans().getStepInterface(stepnames[i], CopyNr);
            if (si != null) {
                data.stepInterfaces.put(i, getDispatcher().findBaseSteps(stepnames[i]).get(CopyNr));
            } else {
                if (meta.getStepRequired()[i].equals(StepsMetricsMeta.YES)) {
                    throw new KettleException("We cannot get step [" + stepnames[i] + "] CopyNr=" + CopyNr + "!");
                }
            }
        }
        data.outputRowMeta = new RowMeta();
        meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
    }
    // end if first
    data.continueLoop = true;
    // Wait until all specified steps have finished!
    while (data.continueLoop && !isStopped()) {
        data.continueLoop = false;
        Iterator<Entry<Integer, StepInterface>> it = data.stepInterfaces.entrySet().iterator();
        while (it.hasNext()) {
            Entry<Integer, StepInterface> e = it.next();
            StepInterface step = e.getValue();
            if (step.getStatus() != StepExecutionStatus.STATUS_FINISHED) {
                // This step is still running...
                data.continueLoop = true;
            } else {
                // We have done with this step.
                // remove it from the map
                data.stepInterfaces.remove(e.getKey());
                if (log.isDetailed()) {
                    logDetailed("Finished running step [" + step.getStepname() + "(" + step.getCopy() + ")].");
                }
                // Build an empty row based on the meta-data
                Object[] rowData = buildEmptyRow();
                incrementLinesRead();
                int index = 0;
                // output step metrics
                if (!Utils.isEmpty(data.realstepnamefield)) {
                    rowData[index++] = step.getStepname();
                }
                if (!Utils.isEmpty(data.realstepidfield)) {
                    rowData[index++] = step.getStepID();
                }
                if (!Utils.isEmpty(data.realsteplinesinputfield)) {
                    rowData[index++] = step.getLinesInput();
                }
                if (!Utils.isEmpty(data.realsteplinesoutputfield)) {
                    rowData[index++] = step.getLinesOutput();
                }
                if (!Utils.isEmpty(data.realsteplinesreadfield)) {
                    rowData[index++] = step.getLinesRead();
                }
                if (!Utils.isEmpty(data.realsteplinesupdatedfield)) {
                    rowData[index++] = step.getLinesUpdated();
                }
                if (!Utils.isEmpty(data.realsteplineswrittentfield)) {
                    rowData[index++] = step.getLinesWritten();
                }
                if (!Utils.isEmpty(data.realsteplineserrorsfield)) {
                    rowData[index++] = step.getLinesRejected();
                }
                if (!Utils.isEmpty(data.realstepsecondsfield)) {
                    rowData[index++] = step.getRuntime();
                }
                // Send row to the buffer
                putRow(data.outputRowMeta, rowData);
            }
        }
        if (data.continueLoop) {
            try {
                Thread.sleep(200);
            } catch (Exception d) {
            // Ignore
            }
        }
    }
    setOutputDone();
    return false;
}

Example 74 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class ScriptValuesAddedFunctions method getEnvironmentVar.

// Returning EnvironmentVar
public static String getEnvironmentVar(Context actualContext, Scriptable actualObject, Object[] ArgList, Function FunctionContext) {
    String sRC = "";
    if (ArgList.length == 1) {
        try {
            String sArg1 = Context.toString(ArgList[0]);
            // check if the system property exists, and if it does not, try getting a Kettle var instead
            if (System.getProperties().contains(sArg1)) {
                sRC = System.getProperty(sArg1, "");
            } else {
                Object scmo = actualObject.get("_step_", actualObject);
                Object scmO = Context.jsToJava(scmo, StepInterface.class);
                if (scmO instanceof StepInterface) {
                    StepInterface scm = (StepInterface) Context.jsToJava(scmO, StepInterface.class);
                    sArg1 = Context.toString(ArgList[0]);
                    sRC = scm.getVariable(sArg1, "");
                } else {
                    // running in test mode, return ""
                    sRC = "";
                }
            }
        } catch (Exception e) {
            sRC = "";
        }
    } else {
        throw Context.reportRuntimeError("The function call getEnvironmentVar requires 1 argument.");
    }
    return sRC;
}

Example 75 with StepInterface

use of org.pentaho.di.trans.step.StepInterface in project pentaho-kettle by pentaho.

the class ScriptValuesAddedFunctions method getVariable.

// Returning EnvironmentVar
public static String getVariable(Context actualContext, Scriptable actualObject, Object[] ArgList, Function FunctionContext) {
    String sRC = "";
    String sArg1 = "";
    String sArg2 = "";
    if (ArgList.length == 2) {
        try {
            Object scmo = actualObject.get("_step_", actualObject);
            Object scmO = Context.jsToJava(scmo, StepInterface.class);
            if (scmO instanceof StepInterface) {
                StepInterface scm = (StepInterface) Context.jsToJava(scmO, StepInterface.class);
                sArg1 = Context.toString(ArgList[0]);
                sArg2 = Context.toString(ArgList[1]);
                return scm.getVariable(sArg1, sArg2);
            } else {
                // running via the Test button in a dialog
                sArg2 = Context.toString(ArgList[1]);
                return sArg2;
            }
        } catch (Exception e) {
            sRC = "";
        }
    } else {
        throw Context.reportRuntimeError("The function call getVariable requires 2 arguments.");
    }
    return sRC;
}