Examples with ValueMetaInterface org.pentaho.di.core.row.ValueMetaInterface used on opensource projects

Example 51 with ValueMetaInterface

use of org.pentaho.di.core.row.ValueMetaInterface in project pentaho-kettle by pentaho.

the class SniffStepServlet method doGet.

/**
 *<div id="mindtouch">
 *    <h1>/kettle/sniffStep</h1>
 *    <a name="GET"></a>
 *    <h2>GET</h2>
 *    <p>Sniff metadata and data from the specified step of the specified transformation.</p>
 *
 *    <p><b>Example Request:</b><br />
 *    <pre function="syntax.xml">
 *    GET /kettle/sniffStep?trans=dummy-trans&step=tf&xml=Y&lines=10
 *    </pre>
 *
 *    </p>
 *    <h3>Parameters</h3>
 *    <table class="pentaho-table">
 *    <tbody>
 *    <tr>
 *      <th>name</th>
 *      <th>description</th>
 *      <th>type</th>
 *    </tr>
 *    <tr>
 *    <td>trans</td>
 *    <td>Name of the transformation containing required step.</td>
 *    <td>query</td>
 *    </tr>
 *    <tr>
 *    <td>stepName</td>
 *    <td>Name of the transformation step to collect data for.</td>
 *    <td>query</td>
 *    </tr>
 *    <tr>
 *    <td>copynr</td>
 *    <td>Copy number of the step to be used for collecting data. If not provided 0 is used.</td>
 *    <td>integer, optional</td>
 *    </tr>
 *    <tr>
 *    <td>type</td>
 *    <td>Type of the data to be collected (<code>input</code> or <code>output</code>).
 *    If not provided output data is collected.</td>
 *    <td>query, optional</td>
 *    </tr>
 *    <tr>
 *    <td>xml</td>
 *    <td>Boolean flag which defines output format <code>Y</code> forces XML output to be generated.
 *  HTML is returned otherwise.</td>
 *    <td>boolean, optional</td>
 *    </tr>
 *    <tr>
 *    <td>id</td>
 *    <td>Carte id of the transformation to be used for step lookup.</td>
 *    <td>query, optional</td>
 *    </tr>
 *    <tr>
 *    <td>lines</td>
 *    <td>Number of lines to collect and include into response. If not provided 0 lines will be collected.</td>
 *    <td>integer, optional</td>
 *    </tr>
 *    </tbody>
 *    </table>
 *
 *  <h3>Response Body</h3>
 *
 *  <table class="pentaho-table">
 *    <tbody>
 *      <tr>
 *        <td align="right">element:</td>
 *        <td>(custom)</td>
 *      </tr>
 *      <tr>
 *        <td align="right">media types:</td>
 *        <td>text/xml, text/html</td>
 *      </tr>
 *    </tbody>
 *  </table>
 *    <p>Response XML or HTML response containing data and metadata of the step.
 *  If an error occurs during method invocation <code>result</code> field of the response
 *  will contain <code>ERROR</code> status.</p>
 *
 *    <p><b>Example Response:</b></p>
 *    <pre function="syntax.xml">
 *    <?xml version="1.0" encoding="UTF-8"?>
 *    <step-sniff>
 *      <row-meta>
 *        <value-meta><type>String</type>
 *          <storagetype>normal</storagetype>
 *          <name>Field1</name>
 *          <length>0</length>
 *          <precision>-1</precision>
 *          <origin>tf</origin>
 *          <comments/>
 *          <conversion_Mask/>
 *          <decimal_symbol>.</decimal_symbol>
 *          <grouping_symbol>,</grouping_symbol>
 *          <currency_symbol>&#x24;</currency_symbol>
 *          <trim_type>none</trim_type>
 *          <case_insensitive>N</case_insensitive>
 *          <sort_descending>N</sort_descending>
 *          <output_padding>N</output_padding>
 *          <date_format_lenient>Y</date_format_lenient>
 *          <date_format_locale>en_US</date_format_locale>
 *          <date_format_timezone>America&#x2f;Bahia</date_format_timezone>
 *          <lenient_string_to_number>N</lenient_string_to_number>
 *        </value-meta>
 *      </row-meta>
 *      <nr_rows>10</nr_rows>
 *
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *      <row-data><value-data>my-data </value-data>
 *      </row-data>
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *      <row-data><value-data>my-data</value-data>
 *      </row-data>
 *    </step-sniff>
 *    </pre>
 *
 *    <h3>Status Codes</h3>
 *    <table class="pentaho-table">
 *  <tbody>
 *    <tr>
 *      <th>code</th>
 *      <th>description</th>
 *    </tr>
 *    <tr>
 *      <td>200</td>
 *      <td>Request was processed.</td>
 *    </tr>
 *    <tr>
 *      <td>500</td>
 *      <td>Internal server error occurs during request processing.</td>
 *    </tr>
 *  </tbody>
 *</table>
 *</div>
 */
public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    if (isJettyMode() && !request.getContextPath().startsWith(CONTEXT_PATH)) {
        return;
    }
    if (log.isDebug()) {
        logDebug(BaseMessages.getString(PKG, "TransStatusServlet.Log.SniffStepRequested"));
    }
    String transName = request.getParameter("trans");
    String id = request.getParameter("id");
    String stepName = request.getParameter("step");
    int copyNr = Const.toInt(request.getParameter("copynr"), 0);
    final int nrLines = Const.toInt(request.getParameter("lines"), 0);
    String type = Const.NVL(request.getParameter("type"), TYPE_OUTPUT);
    boolean useXML = "Y".equalsIgnoreCase(request.getParameter("xml"));
    response.setStatus(HttpServletResponse.SC_OK);
    if (useXML) {
        response.setContentType("text/xml");
        response.setCharacterEncoding(Const.XML_ENCODING);
    } else {
        response.setContentType("text/html;charset=UTF-8");
    }
    PrintWriter out = response.getWriter();
    // ID is optional...
    // 
    Trans trans;
    CarteObjectEntry entry;
    if (Utils.isEmpty(id)) {
        // get the first transformation that matches...
        // 
        entry = getTransformationMap().getFirstCarteObjectEntry(transName);
        if (entry == null) {
            trans = null;
        } else {
            id = entry.getId();
            trans = getTransformationMap().getTransformation(entry);
        }
    } else {
        // Take the ID into account!
        // 
        entry = new CarteObjectEntry(transName, id);
        trans = getTransformationMap().getTransformation(entry);
    }
    if (trans != null) {
        // Find the step to look at...
        // 
        StepInterface step = null;
        List<StepInterface> stepInterfaces = trans.findBaseSteps(stepName);
        for (int i = 0; i < stepInterfaces.size(); i++) {
            StepInterface look = stepInterfaces.get(i);
            if (look.getCopy() == copyNr) {
                step = look;
            }
        }
        if (step != null) {
            // Add a listener to the transformation step...
            // 
            final boolean read = type.equalsIgnoreCase(TYPE_INPUT);
            final boolean written = type.equalsIgnoreCase(TYPE_OUTPUT) || !read;
            final MetaAndData metaData = new MetaAndData();
            metaData.bufferRowMeta = null;
            metaData.bufferRowData = new ArrayList<Object[]>();
            RowListener rowListener = new RowListener() {

                public void rowReadEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
                    if (read && metaData.bufferRowData.size() < nrLines) {
                        metaData.bufferRowMeta = rowMeta;
                        metaData.bufferRowData.add(row);
                    }
                }

                public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
                    if (written && metaData.bufferRowData.size() < nrLines) {
                        metaData.bufferRowMeta = rowMeta;
                        metaData.bufferRowData.add(row);
                    }
                }

                public void errorRowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
                }
            };
            step.addRowListener(rowListener);
            // 
            while (metaData.bufferRowData.size() < nrLines && step.isRunning() && !trans.isFinished() && !trans.isStopped()) {
                try {
                    Thread.sleep(100);
                } catch (InterruptedException e) {
                    // 
                    break;
                }
            }
            // Remove the row listener
            // 
            step.removeRowListener(rowListener);
            // 
            if (useXML) {
                // Send the result back as XML
                // 
                response.setContentType("text/xml");
                response.setCharacterEncoding(Const.XML_ENCODING);
                out.print(XMLHandler.getXMLHeader(Const.XML_ENCODING));
                out.println(XMLHandler.openTag(XML_TAG));
                if (metaData.bufferRowMeta != null) {
                    // Row Meta data
                    // 
                    out.println(metaData.bufferRowMeta.getMetaXML());
                    // Nr of lines
                    // 
                    out.println(XMLHandler.addTagValue("nr_rows", metaData.bufferRowData.size()));
                    // 
                    for (int i = 0; i < metaData.bufferRowData.size(); i++) {
                        Object[] rowData = metaData.bufferRowData.get(i);
                        out.println(metaData.bufferRowMeta.getDataXML(rowData));
                    }
                }
                out.println(XMLHandler.closeTag(XML_TAG));
            } else {
                response.setContentType("text/html;charset=UTF-8");
                out.println("<HTML>");
                out.println("<HEAD>");
                out.println("<TITLE>" + BaseMessages.getString(PKG, "SniffStepServlet.SniffResults") + "</TITLE>");
                out.println("<META http-equiv=\"Refresh\" content=\"10;url=" + convertContextPath(CONTEXT_PATH) + "?name=" + URLEncoder.encode(transName, "UTF-8") + "&id=" + URLEncoder.encode(id, "UTF-8") + "\">");
                out.println("<META http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">");
                out.println("</HEAD>");
                out.println("<BODY>");
                out.println("<H1>" + Encode.forHtml(BaseMessages.getString(PKG, "SniffStepServlet.SniffResultsForStep", stepName)) + "</H1>");
                try {
                    out.println("<table border=\"1\">");
                    if (metaData.bufferRowMeta != null) {
                        // Print a header row containing all the field names...
                        // 
                        out.print("<tr><th>#</th>");
                        for (ValueMetaInterface valueMeta : metaData.bufferRowMeta.getValueMetaList()) {
                            out.print("<th>" + valueMeta.getName() + "</th>");
                        }
                        out.println("</tr>");
                        // 
                        for (int r = 0; r < metaData.bufferRowData.size(); r++) {
                            Object[] rowData = metaData.bufferRowData.get(r);
                            out.print("<tr>");
                            out.println("<td>" + (r + 1) + "</td>");
                            for (int v = 0; v < metaData.bufferRowMeta.size(); v++) {
                                ValueMetaInterface valueMeta = metaData.bufferRowMeta.getValueMeta(v);
                                Object valueData = rowData[v];
                                out.println("<td>" + valueMeta.getString(valueData) + "</td>");
                            }
                            out.println("</tr>");
                        }
                    }
                    out.println("</table>");
                    out.println("<p>");
                } catch (Exception ex) {
                    out.println("<p>");
                    out.println("<pre>");
                    out.println(Encode.forHtml(Const.getStackTracker(ex)));
                    out.println("</pre>");
                }
                out.println("<p>");
                out.println("</BODY>");
                out.println("</HTML>");
            }
        } else {
            if (useXML) {
                out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "SniffStepServlet.Log.CoundNotFindSpecStep", stepName)).getXML());
            } else {
                out.println("<H1>" + Encode.forHtml(BaseMessages.getString(PKG, "SniffStepServlet.Log.CoundNotFindSpecStep", stepName)) + "</H1>");
                out.println("<a href=\"" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.BackToStatusPage") + "</a><p>");
            }
        }
    } else {
        if (useXML) {
            out.println(new WebResult(WebResult.STRING_ERROR, BaseMessages.getString(PKG, "SniffStepServlet.Log.CoundNotFindSpecTrans", transName)).getXML());
        } else {
            out.println("<H1>" + Encode.forHtml(BaseMessages.getString(PKG, "SniffStepServlet.Log.CoundNotFindTrans", transName)) + "</H1>");
            out.println("<a href=\"" + convertContextPath(GetStatusServlet.CONTEXT_PATH) + "\">" + BaseMessages.getString(PKG, "TransStatusServlet.BackToStatusPage") + "</a><p>");
        }
    }
}

Example 52 with ValueMetaInterface

use of org.pentaho.di.core.row.ValueMetaInterface in project pentaho-kettle by pentaho.

the class YamlInputMeta method getFields.

@Override
public void getFields(RowMetaInterface r, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException {
    int i;
    for (i = 0; i < inputFields.length; i++) {
        YamlInputField field = inputFields[i];
        int type = field.getType();
        if (type == ValueMetaInterface.TYPE_NONE) {
            type = ValueMetaInterface.TYPE_STRING;
        }
        String valueName = space.environmentSubstitute(field.getName());
        ValueMetaInterface v;
        try {
            v = ValueMetaFactory.createValueMeta(valueName, type);
        } catch (KettlePluginException e) {
            v = new ValueMetaString(valueName);
        }
        v.setLength(field.getLength());
        v.setPrecision(field.getPrecision());
        v.setOrigin(name);
        v.setConversionMask(field.getFormat());
        v.setDecimalSymbol(field.getDecimalSymbol());
        v.setGroupingSymbol(field.getGroupSymbol());
        v.setCurrencySymbol(field.getCurrencySymbol());
        r.addValueMeta(v);
    }
    if (includeFilename) {
        ValueMetaInterface v = new ValueMetaString(space.environmentSubstitute(filenameField));
        v.setLength(250);
        v.setPrecision(-1);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
    if (includeRowNumber) {
        ValueMetaInterface v = new ValueMetaInteger(space.environmentSubstitute(rowNumberField));
        v.setLength(ValueMetaInterface.DEFAULT_INTEGER_LENGTH, 0);
        v.setOrigin(name);
        r.addValueMeta(v);
    }
}

Example 53 with ValueMetaInterface

use of org.pentaho.di.core.row.ValueMetaInterface in project pentaho-kettle by pentaho.

the class UpdateSQLTest method testRowsTransform.

@Test
public void testRowsTransform() {
    String[] keyLookup = new String[] { "Name" };
    String[] keyStream = new String[] { "FirstName" };
    String[] updateLookup = new String[] { "SecondName", "PostAddress", "ZIP" };
    String[] updateStream = new String[] { "SurName", "Address", "ZIP" };
    RowMetaInterface prev = new RowMeta();
    prev.addValueMeta(new ValueMetaString(keyStream[0]));
    prev.addValueMeta(new ValueMetaString(updateStream[0]));
    prev.addValueMeta(new ValueMetaString(updateStream[1]));
    prev.addValueMeta(new ValueMetaString(updateStream[2]));
    try {
        RowMetaInterface result = RowMetaUtils.getRowMetaForUpdate(prev, keyLookup, keyStream, updateLookup, updateStream);
        ValueMetaInterface vmi = result.getValueMeta(0);
        assertEquals(vmi.getName(), keyLookup[0]);
        assertEquals(prev.getValueMeta(0).getName(), keyStream[0]);
        assertEquals(result.getValueMeta(1).getName(), updateLookup[0]);
        assertEquals(prev.getValueMeta(1).getName(), updateStream[0]);
        assertEquals(result.getValueMeta(2).getName(), updateLookup[1]);
        assertEquals(prev.getValueMeta(2).getName(), updateStream[1]);
        assertEquals(result.getValueMeta(3).getName(), updateStream[2]);
        assertEquals(prev.getValueMeta(3).getName(), updateStream[2]);
    } catch (Exception ex) {
        ex.printStackTrace();
        fail();
    }
}

Example 54 with ValueMetaInterface

use of org.pentaho.di.core.row.ValueMetaInterface in project pentaho-kettle by pentaho.

the class GPLoadDataOutput method writeLine.

public void writeLine(RowMetaInterface mi, Object[] row) throws KettleException {
    if (first) {
        first = false;
        enclosure = meta.getEnclosure();
        if (enclosure == null) {
            enclosure = "";
        } else {
            enclosure = gpLoad.environmentSubstitute(enclosure);
        }
        delimiter = meta.getDelimiter();
        if (delimiter == null) {
            throw new KettleException(BaseMessages.getString(PKG, "GPload.Exception.DelimiterMissing"));
        } else {
            delimiter = gpLoad.environmentSubstitute(delimiter);
            if (Utils.isEmpty(delimiter)) {
                throw new KettleException(BaseMessages.getString(PKG, "GPload.Exception.DelimiterMissing"));
            }
        }
        // Setup up the fields we need to take for each of the rows
        // as this speeds up processing.
        fieldNumbers = new int[meta.getFieldStream().length];
        for (int i = 0; i < fieldNumbers.length; i++) {
            fieldNumbers[i] = mi.indexOfValue(meta.getFieldStream()[i]);
            if (fieldNumbers[i] < 0) {
                throw new KettleException(BaseMessages.getString(PKG, "GPLoadDataOutput.Exception.FieldNotFound", meta.getFieldStream()[i]));
            }
        }
        sdfDate = new SimpleDateFormat("yyyy-MM-dd");
        sdfDateTime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
    }
    // Write the data to the output
    ValueMetaInterface v = null;
    int number = 0;
    for (int i = 0; i < fieldNumbers.length; i++) {
        // TODO: variable substitution
        if (i != 0) {
            output.print(delimiter);
        }
        number = fieldNumbers[i];
        v = mi.getValueMeta(number);
        if (row[number] == null) {
            // TODO (SB): special check for null in case of Strings.
            output.print(enclosure);
            output.print(enclosure);
        } else {
            switch(v.getType()) {
                case ValueMetaInterface.TYPE_STRING:
                    String s = mi.getString(row, number);
                    if (s.indexOf(enclosure) >= 0) {
                        s = createEscapedString(s, enclosure);
                    }
                    output.print(enclosure);
                    output.print(s);
                    output.print(enclosure);
                    break;
                case ValueMetaInterface.TYPE_INTEGER:
                    Long l = mi.getInteger(row, number);
                    if (meta.getEncloseNumbers()) {
                        output.print(enclosure);
                        output.print(l);
                        output.print(enclosure);
                    } else {
                        output.print(l);
                    }
                    break;
                case ValueMetaInterface.TYPE_NUMBER:
                    Double d = mi.getNumber(row, number);
                    if (meta.getEncloseNumbers()) {
                        output.print(enclosure);
                        output.print(d);
                        output.print(enclosure);
                    } else {
                        output.print(d);
                    }
                    break;
                case ValueMetaInterface.TYPE_BIGNUMBER:
                    BigDecimal bd = mi.getBigNumber(row, number);
                    if (meta.getEncloseNumbers()) {
                        output.print(enclosure);
                        output.print(bd);
                        output.print(enclosure);
                    } else {
                        output.print(bd);
                    }
                    break;
                case ValueMetaInterface.TYPE_DATE:
                    Date dt = mi.getDate(row, number);
                    output.print(enclosure);
                    output.print(sdfDate.format(dt));
                    output.print(enclosure);
                    break;
                case ValueMetaInterface.TYPE_BOOLEAN:
                    Boolean b = mi.getBoolean(row, number);
                    output.print(enclosure);
                    if (b.booleanValue()) {
                        output.print("Y");
                    } else {
                        output.print("N");
                    }
                    output.print(enclosure);
                    break;
                case ValueMetaInterface.TYPE_BINARY:
                    byte[] byt = mi.getBinary(row, number);
                    output.print("<startlob>");
                    output.print(byt);
                    output.print("<endlob>");
                    break;
                case ValueMetaInterface.TYPE_TIMESTAMP:
                    Date time = mi.getDate(row, number);
                    output.print(enclosure);
                    output.print(sdfDateTime.format(time));
                    output.print(enclosure);
                    break;
                default:
                    throw new KettleException(BaseMessages.getString(PKG, "GPLoadDataOutput.Exception.TypeNotSupported", v.getType()));
            }
        }
    }
    output.print(Const.CR);
}

Example 55 with ValueMetaInterface

use of org.pentaho.di.core.row.ValueMetaInterface in project pentaho-kettle by pentaho.

the class GPLoadDialog method getUpdate.

private void getUpdate() {
    try {
        RowMetaInterface r = transMeta.getPrevStepFields(stepname);
        if (r != null) {
            TableItemInsertListener listener = new TableItemInsertListener() {

                public boolean tableItemInserted(TableItem tableItem, ValueMetaInterface v) {
                    if (v.getType() == ValueMetaInterface.TYPE_DATE) {
                        // The default is date mask.
                        tableItem.setText(3, BaseMessages.getString(PKG, "GPLoadDialog.DateMask.Label"));
                    } else {
                        tableItem.setText(3, "");
                    }
                    return true;
                }
            };
            BaseStepDialog.getFieldsFromPrevious(r, wReturn, 1, new int[] { 1, 2 }, new int[] {}, -1, -1, listener);
        }
    } catch (KettleException ke) {
        new ErrorDialog(shell, BaseMessages.getString(PKG, "GPLoadDialog.FailedToGetFields.DialogTitle"), BaseMessages.getString(PKG, "GPLoadDialog.FailedToGetFields.DialogMessage"), ke);
    }
}