Examples with RowStepCollector org.pentaho.di.trans.RowStepCollector used on opensource projects

Example 1 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class JoinRowsTest method testJoinRowsStep.

@Test
public void testJoinRowsStep() throws Exception {
    JoinRowsMeta joinRowsMeta = new JoinRowsMeta();
    joinRowsMeta.setMainStepname("main step name");
    joinRowsMeta.setPrefix("out");
    joinRowsMeta.setCacheSize(3);
    JoinRowsData joinRowsData = new JoinRowsData();
    JoinRows joinRows = getJoinRows();
    joinRows.getTrans().setRunning(true);
    joinRows.init(joinRowsMeta, joinRowsData);
    List<RowSet> rowSets = new ArrayList<>();
    rowSets.add(getRowSetWithData(3, "main --", true));
    rowSets.add(getRowSetWithData(3, "secondary --", false));
    joinRows.setInputRowSets(rowSets);
    RowStepCollector rowStepCollector = new RowStepCollector();
    joinRows.addRowListener(rowStepCollector);
    joinRows.getLogChannel().setLogLevel(LogLevel.ROWLEVEL);
    KettleLogStore.init();
    while (true) {
        if (!joinRows.processRow(joinRowsMeta, joinRowsData)) {
            break;
        }
    }
    rowStepCollector.getRowsWritten();
    // since we have data join of two row sets with size 3 then we must have 9 written rows
    assertEquals(9, rowStepCollector.getRowsWritten().size());
    assertEquals(6, rowStepCollector.getRowsRead().size());
    Object[][] expectedResult = createExpectedResult();
    List<Object[]> rowWritten = rowStepCollector.getRowsWritten().stream().map(RowMetaAndData::getData).collect(Collectors.toList());
    for (int i = 0; i < 9; i++) {
        assertTrue(Arrays.equals(expectedResult[i], rowWritten.get(i)));
    }
}

Example 2 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class WebServiceIT method testProcessRow.

public void testProcessRow() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("WebServiceTest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a String Cut step
    // 
    String webServiceStepname = "web service step";
    WebServiceMeta scm = new WebServiceMeta();
    // scm.setUrl(HTTP_LOCALHOST_9998+ "wsdl");
    // scm.setOperationName("CelciusToFahrenheit");
    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
    DocumentBuilder db = dbf.newDocumentBuilder();
    Document doc = db.parse(new InputSource(new java.io.StringReader(STEP_META)));
    scm.loadXML(doc.getFirstChild(), null, (IMetaStore) null);
    String webServicePid = registry.getPluginId(StepPluginType.class, scm);
    StepMeta webServiceStep = new StepMeta(webServicePid, webServiceStepname, scm);
    transMeta.addStep(webServiceStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, webServiceStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(webServiceStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(webServiceStepname, 0);
    RowStepCollector webServiceRc = new RowStepCollector();
    si.addRowListener(webServiceRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData(createRowMetaInterface(), new Object[][] { new Object[] { 10 } });
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> goldRows = createData(createOutputRowMetaInterface(), new Object[][] { new Object[] { 10, new BigDecimal(20) } });
    List<RowMetaAndData> resultRows2 = webServiceRc.getRowsWritten();
    assertEquals(goldRows, resultRows2);
}

Example 3 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class ConstantIT method constantFieldsAddedOnceWithCorrectFormatting.

@Test
public void constantFieldsAddedOnceWithCorrectFormatting() throws KettleException {
    String path = getClass().getResource("/constants-test.ktr").getPath();
    TransMeta constantTrans = new TransMeta(path, new Variables());
    Trans trans = new Trans(constantTrans);
    trans.prepareExecution(new String[] {});
    RowStepCollector collector = new RowStepCollector();
    trans.getSteps().get(1).step.addRowListener(collector);
    List<RowMetaAndData> rowsWritten = collector.getRowsWritten();
    trans.setPreview(true);
    trans.startThreads();
    trans.waitUntilFinished();
    assertEquals(1, rowsWritten.size());
    Object[] data = rowsWritten.get(0).getData();
    RowMetaInterface rowMeta = rowsWritten.get(0).getRowMeta();
    assertEquals(7, rowMeta.size());
    assertEquals("a", data[0]);
    assertEquals("88.00", rowMeta.getValueMeta(1).getString(data[1]));
    assertEquals("b", data[2]);
    assertEquals("c", data[3]);
    assertEquals("d", data[4]);
    assertEquals("e", data[5]);
    assertEquals("1,234.568", rowMeta.getValueMeta(6).getString(data[6]));
}

Example 4 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class JavaScriptStringIT method testStringsPadCase.

/**
 * Test case for javascript functionality: lpad(), rpad(), upper(), lower().
 */
public void testStringsPadCase() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("test javascript pad casing");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a javascript step
    // 
    String javaScriptStepname = "javascript step";
    ScriptValuesMetaMod svm = new ScriptValuesMetaMod();
    ScriptValuesScript[] js = new ScriptValuesScript[] { new ScriptValuesScript(ScriptValuesScript.TRANSFORM_SCRIPT, "script", "var lpadded1 = lpad(string, \"x\", 10);\n" + "var lpadded2 = lpad(string, \" \", 9);\n" + "var rpadded1 = rpad(string, \"x\", 10);\n" + "var rpadded2 = rpad(string, \" \", 9);\n" + "var upperStr = upper(string);\n" + "var lowerStr = lower(string);\n") };
    svm.setJSScripts(js);
    svm.setFieldname(new String[] { "lpadded1", "lpadded2", "rpadded1", "rpadded2", "upperStr", "lowerStr" });
    svm.setRename(new String[] { "", "", "", "", "", "", "" });
    svm.setType(new int[] { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING });
    svm.setLength(new int[] { -1, -1, -1, -1, -1, -1, -1 });
    svm.setPrecision(new int[] { -1, -1, -1, -1, -1, -1, -1 });
    svm.setReplace(new boolean[] { false, false, false, false, false, false, false });
    svm.setCompatible(true);
    String javaScriptStepPid = registry.getPluginId(StepPluginType.class, svm);
    StepMeta javaScriptStep = new StepMeta(javaScriptStepPid, javaScriptStepname, svm);
    transMeta.addStep(javaScriptStep);
    TransHopMeta hi1 = new TransHopMeta(injectorStep, javaScriptStep);
    transMeta.addTransHop(hi1);
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    TransHopMeta hi2 = new TransHopMeta(javaScriptStep, dummyStep);
    transMeta.addTransHop(hi2);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si;
    si = trans.getStepInterface(javaScriptStepname, 0);
    RowStepCollector javaScriptRc = new RowStepCollector();
    si.addRowListener(javaScriptRc);
    si = trans.getStepInterface(dummyStepname, 0);
    RowStepCollector dummyRc = new RowStepCollector();
    si.addRowListener(dummyRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData2();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> goldenImageRows = createResultData2();
    List<RowMetaAndData> resultRows1 = javaScriptRc.getRowsWritten();
    checkRows(resultRows1, goldenImageRows);
    List<RowMetaAndData> resultRows2 = dummyRc.getRowsRead();
    checkRows(resultRows2, goldenImageRows);
}

Example 5 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class SplitFieldToRowsIT method splitFieldToRows.

/**
 * Splits the "stringToSplit" with the passed "delimiter". The "delimiter" is assumed by this method to be a Kettle
 * variable. The parameter "delimiterVariableValue" should contain the variables value.
 *
 * The "isDelimiterRegex" parameter will process the use regex for pattern matching if true.
 *
 * @param testName
 * @param stringToSplit
 * @param isDelimiterRegex
 * @param delimiter
 * @param delimiterVariableValue
 * @return
 * @throws Exception
 */
private List<RowMetaAndData> splitFieldToRows(String testName, String stringToSplit, boolean isDelimiterRegex, String delimiter, String delimiterVariableValue) {
    RowStepCollector rc = new RowStepCollector();
    try {
        KettleEnvironment.init();
        // Create a new transformation...
        TransMeta transMeta = new TransMeta();
        transMeta.setName("Split field to rows test");
        PluginRegistry registry = PluginRegistry.getInstance();
        // create an injector step...
        String injectorStepname = "injector step";
        InjectorMeta im = new InjectorMeta();
        // Set the information of the injector.
        String injectorPid = registry.getPluginId(StepPluginType.class, im);
        StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
        transMeta.addStep(injectorStep);
        // Create a Split Field to Rows step
        String splitfieldToRowsName = "Split field to rows";
        SplitFieldToRowsMeta splitFieldtoRowsMeta = new SplitFieldToRowsMeta();
        splitFieldtoRowsMeta.setDelimiter(delimiter);
        splitFieldtoRowsMeta.setDelimiterRegex(isDelimiterRegex);
        splitFieldtoRowsMeta.setSplitField(FIELD_TO_SPLIT_NAME);
        splitFieldtoRowsMeta.setNewFieldname(NEW_FIELD_NAME);
        String splitFieldTotRowsPid = registry.getPluginId(StepPluginType.class, splitFieldtoRowsMeta);
        StepMeta splitFieldToRows = new StepMeta(splitFieldTotRowsPid, splitfieldToRowsName, splitFieldtoRowsMeta);
        transMeta.addStep(splitFieldToRows);
        // hop the injector to the split field to rows step
        TransHopMeta hop_injector_splitfieldToRows = new TransHopMeta(injectorStep, splitFieldToRows);
        transMeta.addTransHop(hop_injector_splitfieldToRows);
        // Create a dummy step
        String dummyStepname = "dummy step";
        DummyTransMeta dm = new DummyTransMeta();
        String dummyPid = registry.getPluginId(StepPluginType.class, dm);
        StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
        transMeta.addStep(dummyStep);
        TransHopMeta hop_SplitFieldToRows_Dummy = new TransHopMeta(splitFieldToRows, dummyStep);
        transMeta.addTransHop(hop_SplitFieldToRows_Dummy);
        if (!Utils.isEmpty(delimiterVariableValue)) {
            String delimiterVariableName = delimiter.replace("${", "");
            delimiterVariableName = delimiterVariableName.replace("}", "");
            transMeta.setVariable(delimiterVariableName, delimiterVariableValue);
        }
        // Now execute the transformation...
        Trans trans = new Trans(transMeta);
        trans.prepareExecution(null);
        StepInterface si = trans.getStepInterface(dummyStepname, 0);
        si.addRowListener(rc);
        RowProducer rowProducer = trans.addRowProducer(injectorStepname, 0);
        trans.startThreads();
        // add rows
        List<RowMetaAndData> inputList = createData(stringToSplit);
        for (RowMetaAndData rm : inputList) {
            rowProducer.putRow(rm.getRowMeta(), rm.getData());
        }
        rowProducer.finished();
        trans.waitUntilFinished();
    } catch (KettleException e) {
        fail("KettleEnvironment exception" + e.getMessage());
    }
    List<RowMetaAndData> resultRows = rc.getRowsWritten();
    return resultRows;
}