Examples with RowProducer org.pentaho.di.trans.RowProducer used on opensource projects

Example 16 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class ValueMapperIT method testValueMapper2.

/**
 * Test case for valuemapper step. Injector step to a valuemapper step to a dummy step. Rows go in and should be
 * mapped accordingly.
 *
 * This test will write the mappings in the same field.
 */
public void testValueMapper2() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("valuemappertest2");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a step
    // 
    String valueMapperName = "valuemapper step";
    ValueMapperMeta vm = new ValueMapperMeta();
    vm.setFieldToUse("field1");
    vm.setTargetField("");
    vm.setSourceValue(new String[] { "abc", "ABC", "def", "ghij" });
    vm.setTargetValue(new String[] { "begin", "BEG_LONGER", "test", "" });
    String valueMapperPid = registry.getPluginId(StepPluginType.class, vm);
    StepMeta valueMapperStep = new StepMeta(valueMapperPid, valueMapperName, vm);
    transMeta.addStep(valueMapperStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, valueMapperStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(valueMapperStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(valueMapperName, 0);
    RowStepCollector valueMapperRc = new RowStepCollector();
    si.addRowListener(valueMapperRc);
    si = trans.getStepInterface(dummyStepname2, 0);
    RowStepCollector dummyRc = new RowStepCollector();
    si.addRowListener(dummyRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData2();
    checkRows(goldenImageRows, resultRows);
}

Example 17 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class WebServiceIT method testProcessRow.

public void testProcessRow() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("WebServiceTest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a String Cut step
    // 
    String webServiceStepname = "web service step";
    WebServiceMeta scm = new WebServiceMeta();
    // scm.setUrl(HTTP_LOCALHOST_9998+ "wsdl");
    // scm.setOperationName("CelciusToFahrenheit");
    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
    DocumentBuilder db = dbf.newDocumentBuilder();
    Document doc = db.parse(new InputSource(new java.io.StringReader(STEP_META)));
    scm.loadXML(doc.getFirstChild(), null, (IMetaStore) null);
    String webServicePid = registry.getPluginId(StepPluginType.class, scm);
    StepMeta webServiceStep = new StepMeta(webServicePid, webServiceStepname, scm);
    transMeta.addStep(webServiceStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, webServiceStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(webServiceStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(webServiceStepname, 0);
    RowStepCollector webServiceRc = new RowStepCollector();
    si.addRowListener(webServiceRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData(createRowMetaInterface(), new Object[][] { new Object[] { 10 } });
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> goldRows = createData(createOutputRowMetaInterface(), new Object[][] { new Object[] { 10, new BigDecimal(20) } });
    List<RowMetaAndData> resultRows2 = webServiceRc.getRowsWritten();
    assertEquals(goldRows, resultRows2);
}

Example 18 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class AppendIT method testAppendStep.

/**
 * Test case for Append step. 2 Injector steps to an append step to a dummy step. Rows go in, the order should be as
 * defined in the append step.
 */
public void testAppendStep() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("Appendtest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step 1...
    // 
    String injectorStepname1 = "injector step 1";
    InjectorMeta im1 = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid1 = registry.getPluginId(StepPluginType.class, im1);
    StepMeta injectorStep1 = new StepMeta(injectorPid1, injectorStepname1, im1);
    transMeta.addStep(injectorStep1);
    // 
    // create an injector step 2...
    // 
    String injectorStepname2 = "injector step 2";
    InjectorMeta im2 = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid2 = registry.getPluginId(StepPluginType.class, im2);
    StepMeta injectorStep2 = new StepMeta(injectorPid2, injectorStepname2, im2);
    transMeta.addStep(injectorStep2);
    // 
    // Create an append step
    // 
    String appendName = "append step";
    AppendMeta am = new AppendMeta();
    List<StreamInterface> infoStreams = am.getStepIOMeta().getInfoStreams();
    infoStreams.get(0).setStepMeta(injectorStep1);
    infoStreams.get(1).setStepMeta(injectorStep2);
    String appendPid = registry.getPluginId(StepPluginType.class, am);
    StepMeta append = new StepMeta(appendPid, appendName, am);
    transMeta.addStep(append);
    TransHopMeta hi2 = new TransHopMeta(injectorStep1, append);
    transMeta.addTransHop(hi2);
    TransHopMeta hi3 = new TransHopMeta(injectorStep2, append);
    transMeta.addTransHop(hi3);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi4 = new TransHopMeta(append, dummyStep1);
    transMeta.addTransHop(hi4);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(appendName, 0);
    RowStepCollector blockingRc = new RowStepCollector();
    si.addRowListener(blockingRc);
    si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    RowProducer rp1 = trans.addRowProducer(injectorStepname1, 0);
    RowProducer rp2 = trans.addRowProducer(injectorStepname2, 0);
    trans.startThreads();
    // add rows to tail step
    List<RowMetaAndData> inputList2 = createData2();
    Iterator<RowMetaAndData> it2 = inputList2.iterator();
    while (it2.hasNext()) {
        RowMetaAndData rm = it2.next();
        rp2.putRow(rm.getRowMeta(), rm.getData());
    }
    rp2.finished();
    // add rows to head step
    List<RowMetaAndData> inputList1 = createData1();
    Iterator<RowMetaAndData> it1 = inputList1.iterator();
    while (it1.hasNext()) {
        RowMetaAndData rm = it1.next();
        rp1.putRow(rm.getRowMeta(), rm.getData());
    }
    rp1.finished();
    trans.waitUntilFinished();
    // The result should be that first all rows from injector 1 and
    // then all rows from injector step 2
    List<RowMetaAndData> expectedList = new ArrayList<RowMetaAndData>();
    expectedList.addAll(inputList1);
    expectedList.addAll(inputList2);
    List<RowMetaAndData> resultRows1 = dummyRc1.getRowsWritten();
    checkRows(resultRows1, expectedList);
}

Example 19 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class BlockingStepIT method testBlockingStep.

/**
 * Test case for blocking step step. Injector step to a blocking step to a dummy step. rows go in, only 1 row should
 * be output (the last one).
 */
public void testBlockingStep() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("blockingsteptest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a blocking step
    // 
    String blockingStepname = "blocking step";
    BlockingStepMeta bm = new BlockingStepMeta();
    String blockingStepPid = registry.getPluginId(StepPluginType.class, bm);
    StepMeta blockingStep = new StepMeta(blockingStepPid, blockingStepname, bm);
    transMeta.addStep(blockingStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, blockingStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(blockingStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(blockingStepname, 0);
    RowStepCollector blockingRc = new RowStepCollector();
    si.addRowListener(blockingRc);
    si = trans.getStepInterface(dummyStepname2, 0);
    RowStepCollector dummyRc2 = new RowStepCollector();
    si.addRowListener(dummyRc2);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // The results should be that dummy1 gets all rows.
    // blocking step should receive all rows (but only send the
    // last one through). dummy2 should only get the last row.
    List<RowMetaAndData> resultRows1 = dummyRc1.getRowsRead();
    checkRows(resultRows1, inputList);
    List<RowMetaAndData> resultRows2 = blockingRc.getRowsRead();
    checkRows(resultRows2, inputList);
    List<RowMetaAndData> resultRows3 = dummyRc2.getRowsRead();
    List<RowMetaAndData> lastList = new ArrayList<RowMetaAndData>();
    lastList.add(inputList.get(inputList.size() - 1));
    checkRows(resultRows3, lastList);
}

Example 20 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class InjectorIT method testInjector.

/**
 * Test case for injector step... also a show case on how to use injector.
 */
public void testInjector() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("injectortest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep);
    transMeta.addTransHop(hi);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = rc.getRowsWritten();
    checkRows(resultRows, inputList);
}