Examples with RowStepCollector org.pentaho.di.trans.RowStepCollector used on opensource projects

Example 66 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class TableCompareIT method testValueComparsion.

/**
 * Test that step reports value comparison errors
 *
 * @throws IOException
 * @throws KettleException
 */
@Test
public void testValueComparsion() throws IOException, KettleException {
    executeSqlPrecondition("complex_key_test.sql");
    TableCompareMeta meta = getTableCompareMeta();
    List<RowMetaAndData> inputData = new ArrayList<RowMetaAndData>();
    inputData.add(new RowMetaAndData(getRowMeta(), getData3()));
    TransMeta trMeta = TransTestFactory.generateTestTransformationError(null, meta, "junit");
    Map<String, RowStepCollector> result = TransTestFactory.executeTestTransformationError(trMeta, "junit", inputData);
    List<RowMetaAndData> read = result.get(TransTestFactory.DUMMY_STEPNAME).getRowsRead();
    List<RowMetaAndData> errors = result.get(TransTestFactory.ERROR_STEPNAME).getRowsRead();
    Assert.assertEquals("One row passed to positive output", 1, read.size());
    Assert.assertEquals("One row passed to negative output", 1, errors.size());
    // check error is properly reported:
    RowMetaAndData row = read.get(0);
    Assert.assertEquals("One errors reported", 1, row.getInteger(8).intValue());
    Assert.assertEquals("Reference table row count", 4, row.getInteger(9).intValue());
    Assert.assertEquals("Compare table row count", 4, row.getInteger(10).intValue());
    Assert.assertEquals("Number of left joins errors", 0, row.getInteger(11).intValue());
    Assert.assertEquals("Number of inner joins errors", 1, row.getInteger(12).intValue());
    Assert.assertEquals("Number of right joins errors", 0, row.getInteger(13).intValue());
    row = errors.get(0);
    Assert.assertEquals("Reported key for not match value", "KEY1 = '2'", row.getString(ehlkd, null));
    Assert.assertEquals("Reported reference table value", "2", row.getString(ehlrvi, null));
    Assert.assertEquals("Reported compare table value", "1", row.getString(ehlcvif, null));
}

Example 67 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class JavaScriptSpecialIT method disabledTestJavaInterface.

/**
 * Test case for JavaScript/Java/JavaScript interfacing.
 */
public void disabledTestJavaInterface() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("test javascript interface");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a javascript step
    // 
    String javaScriptStepname = "javascript step";
    ScriptValuesMetaMod svm = new ScriptValuesMetaMod();
    // process 3 rows and skip the rest.
    ScriptValuesScript[] js = new ScriptValuesScript[] { new ScriptValuesScript(ScriptValuesScript.TRANSFORM_SCRIPT, "script1", "java;\n\n" + "var obj     = new Packages.org.pentaho.di.trans.steps.scriptvalues_mod.JavaScriptTest();\n" + "var long1   = obj.add1ToLong(getInputRowMeta().getInteger(row, 0));\n" + "var number1 = obj.add1ToNumber(getInputRowMeta().getNumber(row, 1));\n" + "var string1 = obj.add1ToString(getInputRowMeta().getString(row, 2));\n" + "var long2   = Packages.org.pentaho.di.trans.steps.scriptvalues_mod." + "JavaScriptTest.add1ToLongStatic(getInputRowMeta().getInteger(row, 0));\n" + "var number2 = Packages.org.pentaho.di.trans.steps.scriptvalues_mod." + "JavaScriptTest.add1ToNumberStatic(getInputRowMeta().getNumber(row, 1));\n" + "var string2 = Packages.org.pentaho.di.trans.steps.scriptvalues_mod." + "JavaScriptTest.add1ToStringStatic(getInputRowMeta().getString(row, 2));\n") };
    svm.setJSScripts(js);
    svm.setFieldname(new String[] { "long1", "number1", "string1", "long2", "number2", "string2" });
    svm.setRename(new String[] { "long1", "number1", "string1", "long2", "number2", "string2" });
    svm.setType(new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_NUMBER, ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_NUMBER, ValueMetaInterface.TYPE_STRING });
    svm.setLength(new int[] { -1, -1, -1, -1, -1, -1, -1 });
    svm.setPrecision(new int[] { -1, -1, -1, -1, -1, -1, -1 });
    svm.setReplace(new boolean[] { false, false, false, false, false, false });
    svm.setCompatible(false);
    String javaScriptStepPid = registry.getPluginId(StepPluginType.class, svm);
    StepMeta javaScriptStep = new StepMeta(javaScriptStepPid, javaScriptStepname, svm);
    transMeta.addStep(javaScriptStep);
    TransHopMeta hi1 = new TransHopMeta(injectorStep, javaScriptStep);
    transMeta.addTransHop(hi1);
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    TransHopMeta hi2 = new TransHopMeta(javaScriptStep, dummyStep);
    transMeta.addTransHop(hi2);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si;
    si = trans.getStepInterface(javaScriptStepname, 0);
    RowStepCollector javaScriptRc = new RowStepCollector();
    si.addRowListener(javaScriptRc);
    si = trans.getStepInterface(dummyStepname, 0);
    RowStepCollector dummyRc = new RowStepCollector();
    si.addRowListener(dummyRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData3();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> goldenImageRows = createResultData3();
    List<RowMetaAndData> resultRows1 = javaScriptRc.getRowsWritten();
    checkRows(resultRows1, goldenImageRows);
    List<RowMetaAndData> resultRows2 = dummyRc.getRowsRead();
    checkRows(resultRows2, goldenImageRows);
}

Example 68 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class JavaScriptStringIT method testStringsTrim.

/**
 * Test case for javascript functionality: ltrim(), rtrim(), trim().
 */
public void testStringsTrim() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("test javascript trim");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a javascript step
    // 
    String javaScriptStepname = "javascript step";
    ScriptValuesMetaMod svm = new ScriptValuesMetaMod();
    ScriptValuesScript[] js = new ScriptValuesScript[] { new ScriptValuesScript(ScriptValuesScript.TRANSFORM_SCRIPT, "script", "var original = 'b' + string.getString() + 'e';\n" + "var ltrimStr = 'b' + ltrim(string.getString()) + 'e';\n" + "var rtrimStr = 'b' + rtrim(string.getString()) + 'e';\n" + "var trimStr  = 'b' + trim(string.getString()) + 'e';\n") };
    svm.setJSScripts(js);
    svm.setFieldname(new String[] { "original", "ltrimStr", "rtrimStr", "trimStr" });
    svm.setRename(new String[] { "", "", "", "" });
    svm.setType(new int[] { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING });
    svm.setLength(new int[] { -1, -1, -1, -1 });
    svm.setPrecision(new int[] { -1, -1, -1, -1 });
    svm.setReplace(new boolean[] { false, false, false, false });
    svm.setCompatible(true);
    String javaScriptStepPid = registry.getPluginId(StepPluginType.class, svm);
    StepMeta javaScriptStep = new StepMeta(javaScriptStepPid, javaScriptStepname, svm);
    transMeta.addStep(javaScriptStep);
    TransHopMeta hi1 = new TransHopMeta(injectorStep, javaScriptStep);
    transMeta.addTransHop(hi1);
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    TransHopMeta hi2 = new TransHopMeta(javaScriptStep, dummyStep);
    transMeta.addTransHop(hi2);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si;
    si = trans.getStepInterface(javaScriptStepname, 0);
    RowStepCollector javaScriptRc = new RowStepCollector();
    si.addRowListener(javaScriptRc);
    si = trans.getStepInterface(dummyStepname, 0);
    RowStepCollector dummyRc = new RowStepCollector();
    si.addRowListener(dummyRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData1();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> goldenImageRows = createResultData1();
    List<RowMetaAndData> resultRows1 = javaScriptRc.getRowsWritten();
    checkRows(resultRows1, goldenImageRows);
    List<RowMetaAndData> resultRows2 = dummyRc.getRowsRead();
    checkRows(resultRows2, goldenImageRows);
}

Example 69 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class TextFileOutputIT method testTextFileOutput6.

/**
 * Tests the normal output capability of the TextFileOutput step
 *
 * @throws Exception
 */
@Test
public void testTextFileOutput6() throws Exception {
    KettleEnvironment.init();
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("testTextFileOutput6");
    PluginRegistry registry = PluginRegistry.getInstance();
    // create an injector step
    String injectorStepName = "injector step";
    StepMeta injectorStep = TestUtilities.createInjectorStep(injectorStepName, registry);
    transMeta.addStep(injectorStep);
    // create a row generator step
    StepMeta rowGeneratorStep = createRowGeneratorStep("Create rows for testTextFileOutput5", registry);
    transMeta.addStep(rowGeneratorStep);
    // create a TransHopMeta for injector and add it to the transMeta
    TransHopMeta hop_injectory_rowGenerator = new TransHopMeta(injectorStep, rowGeneratorStep);
    transMeta.addTransHop(hop_injectory_rowGenerator);
    // create the text file output step with no compression
    // but first lets get a filename
    String textFileName = "testTextFileOutput6";
    String textFileOutputStepName = "text file output step";
    StepMeta textFileOutputStep = createTextFileOutputStep(textFileOutputStepName, textFileName, "None", registry);
    transMeta.addStep(textFileOutputStep);
    // create a TransHopMeta for textFileOutputStep and add it to the transMeta
    TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta(rowGeneratorStep, textFileOutputStep);
    transMeta.addTransHop(hop_RowGenerator_outputTextFile);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    // Create a row collector and add it to the dummy step interface
    StepInterface dummyStepInterface = trans.getStepInterface(textFileOutputStepName, 0);
    RowStepCollector dummyRowCollector = new RowStepCollector();
    dummyStepInterface.addRowListener(dummyRowCollector);
    trans.startThreads();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRowCollector.getRowsWritten();
    Object[][] rows = new Object[10][3];
    File f = new File(textFileName + "." + EXTENSION);
    f.deleteOnExit();
    try {
        FileInputStream fin = new FileInputStream(f);
        InputStreamReader xover = new InputStreamReader(fin);
        BufferedReader input = new BufferedReader(xover);
        readData1Rows(rows, input);
        fin.close();
    } catch (IOException e) {
        fail(e.getLocalizedMessage());
    }
    List<RowMetaAndData> outFileRows = createResultDataFromObjects(rows);
    try {
        TestUtilities.checkRows(resultRows, outFileRows);
    } catch (TestFailedException tfe) {
        fail(tfe.getMessage());
    }
}

Example 70 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class TransExecutorIT method subTransOutputIsAccessibleOutside.

@Test
public void subTransOutputIsAccessibleOutside() throws Exception {
    TransExecutorMeta executorMeta = getExecutorMeta(transExecutor);
    executorMeta.setOutputRowsSourceStepMeta(dummy);
    Trans trans = createTrans(transMeta);
    RowStepCollector endRc = listenExecutor(trans);
    RowProducer rp = trans.addRowProducer(injector.getName(), 0);
    trans.startThreads();
    RowMetaAndData testInput = new RowMetaAndData(createRowMetaForOneField(), SAMPLE_INPUT);
    rp.putRow(testInput.getRowMeta(), testInput.getData());
    rp.finished();
    trans.waitUntilFinished();
    assertEquals(EXPECTED_SUBTRANS_OUTPUT_AMOUNT, endRc.getRowsWritten().size());
    assertThat(asList(endRc.getRowsWritten().get(0).getData()), hasItem((Object) EXPECTED_SUBTRANS_OUTPUT_PATTERN));
}