Examples with RowProducer org.pentaho.di.trans.RowProducer used on opensource projects

Example 46 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class StringCutIT method testPDI11236.

/**
 * This is a generic test using two input fields and different cuts for each
 */
public void testPDI11236() throws KettleException {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("stringcuttest3");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a String Cut step
    // 
    String stringcutStepname = "string cut step";
    StringCutMeta scm = new StringCutMeta();
    scm.setFieldInStream(new String[] { "field1", "field1" });
    scm.setFieldOutStream(new String[] { "", "field1" });
    scm.setCutFrom(new String[] { "1", "0" });
    scm.setCutTo(new String[] { "3", "1" });
    String stringCutStepPid = registry.getPluginId(StepPluginType.class, scm);
    StepMeta stringCutStep = new StepMeta(stringCutStepPid, stringcutStepname, scm);
    transMeta.addStep(stringCutStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, stringCutStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(stringCutStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(stringcutStepname, 0);
    RowStepCollector stringCutRc = new RowStepCollector();
    si.addRowListener(stringCutRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createDataBinary();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows1 = dummyRc1.getRowsRead();
    checkRows(resultRows1, inputList);
    List<RowMetaAndData> goldRows = createBinaryResultData();
    List<RowMetaAndData> resultRows2 = stringCutRc.getRowsWritten();
    checkRows(resultRows2, goldRows);
}

Example 47 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class StringCutIT method testStringCut1.

/**
 * This is a test for PDI-8042, where the first row of meta has no output field but the second does.
 */
@Test
public void testStringCut1() throws KettleException {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("stringcuttest1");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a String Cut step
    // 
    String stringcutStepname = "string cut step";
    StringCutMeta scm = new StringCutMeta();
    scm.setFieldInStream(new String[] { "field1", "field2" });
    scm.setFieldOutStream(new String[] { null, "outf3" });
    scm.setCutFrom(new String[] { null, "1" });
    scm.setCutTo(new String[] { null, "3" });
    String stringCutStepPid = registry.getPluginId(StepPluginType.class, scm);
    StepMeta stringCutStep = new StepMeta(stringCutStepPid, stringcutStepname, scm);
    transMeta.addStep(stringCutStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, stringCutStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(stringCutStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(stringcutStepname, 0);
    RowStepCollector stringCutRc = new RowStepCollector();
    si.addRowListener(stringCutRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData1();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows1 = dummyRc1.getRowsRead();
    checkRows(resultRows1, inputList);
    List<RowMetaAndData> goldRows = createResultData1();
    List<RowMetaAndData> resultRows2 = stringCutRc.getRowsWritten();
    checkRows(resultRows2, goldRows);
}

Example 48 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class StringCutIT method testStringCut2.

/**
 * This is a generic test using two input fields and different cuts for each
 */
@Test
public void testStringCut2() throws KettleException {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("stringcuttest1");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a String Cut step
    // 
    String stringcutStepname = "string cut step";
    StringCutMeta scm = new StringCutMeta();
    scm.setFieldInStream(new String[] { "field1", "field2" });
    scm.setFieldOutStream(new String[] { "outf3", "outf4" });
    scm.setCutFrom(new String[] { "0", "1" });
    scm.setCutTo(new String[] { "1", "3" });
    String stringCutStepPid = registry.getPluginId(StepPluginType.class, scm);
    StepMeta stringCutStep = new StepMeta(stringCutStepPid, stringcutStepname, scm);
    transMeta.addStep(stringCutStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, stringCutStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(stringCutStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(stringcutStepname, 0);
    RowStepCollector stringCutRc = new RowStepCollector();
    si.addRowListener(stringCutRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData1();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows1 = dummyRc1.getRowsRead();
    checkRows(resultRows1, inputList);
    List<RowMetaAndData> goldRows = createResultData2();
    List<RowMetaAndData> resultRows2 = stringCutRc.getRowsWritten();
    checkRows(resultRows2, goldRows);
}

Example 49 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class BlockingStepIT method testBlockingStepPassAll.

/**
 * Test case for blocking step step passing all rows. Injector step to a blocking step to a dummy step. rows go in,
 * all rows should be output.
 */
public void testBlockingStepPassAll() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("blockingsteptest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a blocking step
    // 
    String blockingStepname = "blocking step";
    BlockingStepMeta bm = new BlockingStepMeta();
    bm.setPassAllRows(true);
    String blockingStepPid = registry.getPluginId(StepPluginType.class, bm);
    StepMeta blockingStep = new StepMeta(blockingStepPid, blockingStepname, bm);
    transMeta.addStep(blockingStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, blockingStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(blockingStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(blockingStepname, 0);
    RowStepCollector blockingRc = new RowStepCollector();
    si.addRowListener(blockingRc);
    si = trans.getStepInterface(dummyStepname2, 0);
    RowStepCollector dummyRc2 = new RowStepCollector();
    si.addRowListener(dummyRc2);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // The results should be that dummy1 gets all rows.
    // blocking step should receive all rows (but only send the
    // last one through). dummy2 should only get the last row.
    List<RowMetaAndData> resultRows1 = dummyRc1.getRowsWritten();
    checkRows(resultRows1, inputList);
    List<RowMetaAndData> resultRows2 = blockingRc.getRowsWritten();
    checkRows(resultRows2, inputList);
    List<RowMetaAndData> resultRows3 = dummyRc2.getRowsWritten();
    checkRows(resultRows3, inputList);
}

Example 50 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class CsvInputBase method createAndTestTrans.

protected Trans createAndTestTrans(PluginRegistry registry, TransMeta transMeta, StepMeta injectorStep, StepMeta csvInputStep, String fileName, int numRows) throws KettleException {
    TransHopMeta hi = new TransHopMeta(injectorStep, csvInputStep);
    transMeta.addTransHop(hi);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi1 = new TransHopMeta(csvInputStep, dummyStep1);
    transMeta.addTransHop(hi1);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    RowProducer rp = trans.addRowProducer(injectorStep.getName(), 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData(fileName);
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData1();
    checkRows(goldenImageRows, resultRows, numRows);
    return trans;
}