Examples with RowStepCollector org.pentaho.di.trans.RowStepCollector used on opensource projects

Example 61 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class StringCutIT method testStringCut2.

/**
 * This is a generic test using two input fields and different cuts for each
 */
@Test
public void testStringCut2() throws KettleException {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("stringcuttest1");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a String Cut step
    // 
    String stringcutStepname = "string cut step";
    StringCutMeta scm = new StringCutMeta();
    scm.setFieldInStream(new String[] { "field1", "field2" });
    scm.setFieldOutStream(new String[] { "outf3", "outf4" });
    scm.setCutFrom(new String[] { "0", "1" });
    scm.setCutTo(new String[] { "1", "3" });
    String stringCutStepPid = registry.getPluginId(StepPluginType.class, scm);
    StepMeta stringCutStep = new StepMeta(stringCutStepPid, stringcutStepname, scm);
    transMeta.addStep(stringCutStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, stringCutStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(stringCutStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(stringcutStepname, 0);
    RowStepCollector stringCutRc = new RowStepCollector();
    si.addRowListener(stringCutRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData1();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows1 = dummyRc1.getRowsRead();
    checkRows(resultRows1, inputList);
    List<RowMetaAndData> goldRows = createResultData2();
    List<RowMetaAndData> resultRows2 = stringCutRc.getRowsWritten();
    checkRows(resultRows2, goldRows);
}

Example 62 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class StringCutIT method testPDI11236.

/**
 * This is a generic test using two input fields and different cuts for each
 */
public void testPDI11236() throws KettleException {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("stringcuttest3");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a String Cut step
    // 
    String stringcutStepname = "string cut step";
    StringCutMeta scm = new StringCutMeta();
    scm.setFieldInStream(new String[] { "field1", "field1" });
    scm.setFieldOutStream(new String[] { "", "field1" });
    scm.setCutFrom(new String[] { "1", "0" });
    scm.setCutTo(new String[] { "3", "1" });
    String stringCutStepPid = registry.getPluginId(StepPluginType.class, scm);
    StepMeta stringCutStep = new StepMeta(stringCutStepPid, stringcutStepname, scm);
    transMeta.addStep(stringCutStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, stringCutStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(stringCutStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(stringcutStepname, 0);
    RowStepCollector stringCutRc = new RowStepCollector();
    si.addRowListener(stringCutRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createDataBinary();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows1 = dummyRc1.getRowsRead();
    checkRows(resultRows1, inputList);
    List<RowMetaAndData> goldRows = createBinaryResultData();
    List<RowMetaAndData> resultRows2 = stringCutRc.getRowsWritten();
    checkRows(resultRows2, goldRows);
}

Example 63 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class StringCutIT method testStringCut1.

/**
 * This is a test for PDI-8042, where the first row of meta has no output field but the second does.
 */
@Test
public void testStringCut1() throws KettleException {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("stringcuttest1");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a String Cut step
    // 
    String stringcutStepname = "string cut step";
    StringCutMeta scm = new StringCutMeta();
    scm.setFieldInStream(new String[] { "field1", "field2" });
    scm.setFieldOutStream(new String[] { null, "outf3" });
    scm.setCutFrom(new String[] { null, "1" });
    scm.setCutTo(new String[] { null, "3" });
    String stringCutStepPid = registry.getPluginId(StepPluginType.class, scm);
    StepMeta stringCutStep = new StepMeta(stringCutStepPid, stringcutStepname, scm);
    transMeta.addStep(stringCutStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, stringCutStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(stringCutStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(stringcutStepname, 0);
    RowStepCollector stringCutRc = new RowStepCollector();
    si.addRowListener(stringCutRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData1();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows1 = dummyRc1.getRowsRead();
    checkRows(resultRows1, inputList);
    List<RowMetaAndData> goldRows = createResultData1();
    List<RowMetaAndData> resultRows2 = stringCutRc.getRowsWritten();
    checkRows(resultRows2, goldRows);
}

Example 64 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class TableCompareIT method testMissedReferenceLinesAreRepored.

/**
 * PDI-7255 - The table compare step reports incorrect record as missing
 *
 * @throws KettleException
 * @throws IOException
 */
@Test
public void testMissedReferenceLinesAreRepored() throws KettleException, IOException {
    // prepare database
    executeSqlPrecondition("PDI-7255.sql");
    TableCompareMeta meta = getTableCompareMeta();
    // prepare input date
    List<RowMetaAndData> inputData = new ArrayList<RowMetaAndData>();
    inputData.add(new RowMetaAndData(getRowMeta(), getData1()));
    // execute transformations
    TransMeta trMeta = TransTestFactory.generateTestTransformationError(null, meta, "junit");
    Map<String, RowStepCollector> result = TransTestFactory.executeTestTransformationError(trMeta, "junit", inputData);
    // check the results
    List<RowMetaAndData> read = result.get(TransTestFactory.DUMMY_STEPNAME).getRowsRead();
    Assert.assertTrue("Step achieve comparsion data", read.size() == 1);
    RowMetaAndData row = read.get(0);
    Assert.assertEquals("Number of errors", 1, row.getInteger(8).intValue());
    Assert.assertEquals("Reference table row count", 3, row.getInteger(9).intValue());
    Assert.assertEquals("Compare table row count", 2, row.getInteger(10).intValue());
    Assert.assertEquals("Number of left joins errors", 0, row.getInteger(11).intValue());
    Assert.assertEquals("Number of inner joins errors", 0, row.getInteger(12).intValue());
    Assert.assertEquals("Number of right joins errors", 1, row.getInteger(13).intValue());
    List<RowMetaAndData> errors = result.get(TransTestFactory.ERROR_STEPNAME).getRowsRead();
    Assert.assertTrue("One error output to negative step", errors.size() == 1);
    row = errors.get(0);
    Assert.assertEquals("Reported one missing key", "KEY1 = '2'", row.getString(ehlkd, null));
}

Example 65 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class TableCompareIT method testComplexKeysComparsion.

/**
 * Test table compare test can handle complex keys comparison
 *
 * @throws IOException
 * @throws KettleException
 */
@Test
public void testComplexKeysComparsion() throws IOException, KettleException {
    executeSqlPrecondition("complex_key_test.sql");
    TableCompareMeta meta = getTableCompareMeta();
    List<RowMetaAndData> inputData = new ArrayList<RowMetaAndData>();
    inputData.add(new RowMetaAndData(getRowMeta(), getData2()));
    TransMeta trMeta = TransTestFactory.generateTestTransformationError(null, meta, "junit");
    Map<String, RowStepCollector> result = TransTestFactory.executeTestTransformationError(trMeta, "junit", inputData);
    List<RowMetaAndData> read = result.get(TransTestFactory.DUMMY_STEPNAME).getRowsRead();
    List<RowMetaAndData> errors = result.get(TransTestFactory.ERROR_STEPNAME).getRowsRead();
    Assert.assertEquals("One row passed to positive step", 1, read.size());
    Assert.assertEquals("Two rows passed to negative step", 2, errors.size());
    // check positive step output
    RowMetaAndData row = read.get(0);
    Assert.assertEquals("Number errors", 2, row.getInteger(8).intValue());
    Assert.assertEquals("Reference table row count", 4, row.getInteger(9).intValue());
    Assert.assertEquals("Compare table row count", 4, row.getInteger(10).intValue());
    Assert.assertEquals("Number of left joins errors", 1, row.getInteger(11).intValue());
    Assert.assertEquals("Number of inner joins errors", 0, row.getInteger(12).intValue());
    Assert.assertEquals("Number of right joins errors", 1, row.getInteger(13).intValue());
    // check error step output
    row = errors.get(0);
    Assert.assertEquals("error composite key is mentioned", "KEY1 = '2' and KEY2 = '1'", row.getString(0, null));
    row = errors.get(1);
    Assert.assertEquals("error composite key is mentioned", "KEY1 = '2' and KEY2 = '2'", row.getString(0, null));
}