Examples with RowStepCollector org.pentaho.di.trans.RowStepCollector used on opensource projects

Example 26 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class AddSequenceIT method testAddSequence.

/**
 * Test case for add sequence step. Row generator attached to several add sequence steps and checking whether the end
 * result is as expected.
 */
public void testAddSequence() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("addsequencetest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create a row generator step...
    // 
    String rowGeneratorStepname = "row generator step";
    RowGeneratorMeta rm = new RowGeneratorMeta();
    // Set the information of the row generator.
    String rowGeneratorPid = registry.getPluginId(rm);
    StepMeta rowGeneratorStep = new StepMeta(rowGeneratorPid, rowGeneratorStepname, rm);
    transMeta.addStep(rowGeneratorStep);
    // 
    // Generate 10 empty rows
    // 
    String[] fieldName = {};
    String[] type = {};
    String[] value = {};
    String[] fieldFormat = {};
    String[] group = {};
    String[] decimal = {};
    int[] intDummies = {};
    rm.setDefault();
    rm.setFieldName(fieldName);
    rm.setFieldType(type);
    rm.setValue(value);
    rm.setFieldLength(intDummies);
    rm.setFieldPrecision(intDummies);
    rm.setRowLimit("10");
    rm.setFieldFormat(fieldFormat);
    rm.setGroup(group);
    rm.setDecimal(decimal);
    // 
    // Create first add sequence
    // 
    String seqStepname1 = "add sequence 1";
    AddSequenceMeta asm1 = new AddSequenceMeta();
    asm1.setUseCounter(true);
    asm1.setValuename("counter1");
    asm1.setStartAt(10);
    asm1.setIncrementBy(1);
    asm1.setMaxValue(100);
    String addSeqPid1 = registry.getPluginId(asm1);
    StepMeta addSeqStep1 = new StepMeta(addSeqPid1, seqStepname1, asm1);
    transMeta.addStep(addSeqStep1);
    TransHopMeta hi1 = new TransHopMeta(rowGeneratorStep, addSeqStep1);
    transMeta.addTransHop(hi1);
    // 
    // Create second add sequence
    // 
    String seqStepname2 = "add sequence 2";
    AddSequenceMeta asm2 = new AddSequenceMeta();
    asm2.setUseCounter(true);
    asm2.setValuename("valuename2");
    asm2.setStartAt(1);
    asm2.setIncrementBy(1);
    asm2.setMaxValue(5);
    String addSeqPid2 = registry.getPluginId(asm2);
    StepMeta addSeqStep2 = new StepMeta(addSeqPid2, seqStepname2, asm2);
    transMeta.addStep(addSeqStep2);
    TransHopMeta hi2 = new TransHopMeta(addSeqStep1, addSeqStep2);
    transMeta.addTransHop(hi2);
    // 
    // Create third add sequence
    // 
    String seqStepname3 = "add sequence 3";
    AddSequenceMeta asm3 = new AddSequenceMeta();
    asm3.setUseCounter(true);
    asm3.setValuename("valuename3");
    asm3.setStartAt(1);
    asm3.setIncrementBy(3);
    asm3.setMaxValue(17);
    String addSeqPid3 = registry.getPluginId(asm3);
    StepMeta addSeqStep3 = new StepMeta(addSeqPid3, seqStepname3, asm3);
    transMeta.addStep(addSeqStep3);
    TransHopMeta hi3 = new TransHopMeta(addSeqStep2, addSeqStep3);
    transMeta.addTransHop(hi3);
    // 
    // Create fourth add sequence
    // 
    String seqStepname4 = "add sequence 4";
    AddSequenceMeta asm4 = new AddSequenceMeta();
    asm4.setUseCounter(true);
    asm4.setValuename("valuename4");
    asm4.setStartAt(10);
    asm4.setIncrementBy(-2);
    asm4.setMaxValue(3);
    String addSeqPid4 = registry.getPluginId(asm4);
    StepMeta addSeqStep4 = new StepMeta(addSeqPid4, seqStepname4, asm4);
    transMeta.addStep(addSeqStep4);
    TransHopMeta hi4 = new TransHopMeta(addSeqStep3, addSeqStep4);
    transMeta.addTransHop(hi4);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(seqStepname4, 0);
    RowStepCollector endRc = new RowStepCollector();
    si.addRowListener(endRc);
    trans.startThreads();
    trans.waitUntilFinished();
    // Now check whether the output is still as we expect.
    List<RowMetaAndData> goldenImageRows = createResultData1();
    List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
    checkRows(resultRows1, goldenImageRows);
}

Example 27 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class AppendIT method testAppendStep.

/**
 * Test case for Append step. 2 Injector steps to an append step to a dummy step. Rows go in, the order should be as
 * defined in the append step.
 */
public void testAppendStep() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("Appendtest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step 1...
    // 
    String injectorStepname1 = "injector step 1";
    InjectorMeta im1 = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid1 = registry.getPluginId(StepPluginType.class, im1);
    StepMeta injectorStep1 = new StepMeta(injectorPid1, injectorStepname1, im1);
    transMeta.addStep(injectorStep1);
    // 
    // create an injector step 2...
    // 
    String injectorStepname2 = "injector step 2";
    InjectorMeta im2 = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid2 = registry.getPluginId(StepPluginType.class, im2);
    StepMeta injectorStep2 = new StepMeta(injectorPid2, injectorStepname2, im2);
    transMeta.addStep(injectorStep2);
    // 
    // Create an append step
    // 
    String appendName = "append step";
    AppendMeta am = new AppendMeta();
    List<StreamInterface> infoStreams = am.getStepIOMeta().getInfoStreams();
    infoStreams.get(0).setStepMeta(injectorStep1);
    infoStreams.get(1).setStepMeta(injectorStep2);
    String appendPid = registry.getPluginId(StepPluginType.class, am);
    StepMeta append = new StepMeta(appendPid, appendName, am);
    transMeta.addStep(append);
    TransHopMeta hi2 = new TransHopMeta(injectorStep1, append);
    transMeta.addTransHop(hi2);
    TransHopMeta hi3 = new TransHopMeta(injectorStep2, append);
    transMeta.addTransHop(hi3);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi4 = new TransHopMeta(append, dummyStep1);
    transMeta.addTransHop(hi4);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(appendName, 0);
    RowStepCollector blockingRc = new RowStepCollector();
    si.addRowListener(blockingRc);
    si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    RowProducer rp1 = trans.addRowProducer(injectorStepname1, 0);
    RowProducer rp2 = trans.addRowProducer(injectorStepname2, 0);
    trans.startThreads();
    // add rows to tail step
    List<RowMetaAndData> inputList2 = createData2();
    Iterator<RowMetaAndData> it2 = inputList2.iterator();
    while (it2.hasNext()) {
        RowMetaAndData rm = it2.next();
        rp2.putRow(rm.getRowMeta(), rm.getData());
    }
    rp2.finished();
    // add rows to head step
    List<RowMetaAndData> inputList1 = createData1();
    Iterator<RowMetaAndData> it1 = inputList1.iterator();
    while (it1.hasNext()) {
        RowMetaAndData rm = it1.next();
        rp1.putRow(rm.getRowMeta(), rm.getData());
    }
    rp1.finished();
    trans.waitUntilFinished();
    // The result should be that first all rows from injector 1 and
    // then all rows from injector step 2
    List<RowMetaAndData> expectedList = new ArrayList<RowMetaAndData>();
    expectedList.addAll(inputList1);
    expectedList.addAll(inputList2);
    List<RowMetaAndData> resultRows1 = dummyRc1.getRowsWritten();
    checkRows(resultRows1, expectedList);
}

Example 28 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class BlockingStepIT method testBlockingStep.

/**
 * Test case for blocking step step. Injector step to a blocking step to a dummy step. rows go in, only 1 row should
 * be output (the last one).
 */
public void testBlockingStep() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("blockingsteptest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a blocking step
    // 
    String blockingStepname = "blocking step";
    BlockingStepMeta bm = new BlockingStepMeta();
    String blockingStepPid = registry.getPluginId(StepPluginType.class, bm);
    StepMeta blockingStep = new StepMeta(blockingStepPid, blockingStepname, bm);
    transMeta.addStep(blockingStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, blockingStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(blockingStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(blockingStepname, 0);
    RowStepCollector blockingRc = new RowStepCollector();
    si.addRowListener(blockingRc);
    si = trans.getStepInterface(dummyStepname2, 0);
    RowStepCollector dummyRc2 = new RowStepCollector();
    si.addRowListener(dummyRc2);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // The results should be that dummy1 gets all rows.
    // blocking step should receive all rows (but only send the
    // last one through). dummy2 should only get the last row.
    List<RowMetaAndData> resultRows1 = dummyRc1.getRowsRead();
    checkRows(resultRows1, inputList);
    List<RowMetaAndData> resultRows2 = blockingRc.getRowsRead();
    checkRows(resultRows2, inputList);
    List<RowMetaAndData> resultRows3 = dummyRc2.getRowsRead();
    List<RowMetaAndData> lastList = new ArrayList<RowMetaAndData>();
    lastList.add(inputList.get(inputList.size() - 1));
    checkRows(resultRows3, lastList);
}

Example 29 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class ParameterSimpleTransIT method testParameterSimpleTrans2.

/**
 * Test case for parameters using a simple transformation. Here 1 parameter is not provided as value, so the default
 * will be used.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans2() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("parameter_simple_trans2");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create a get variables step...
    // 
    String getVariablesStepname = "get variables step";
    GetVariableMeta gvm = new GetVariableMeta();
    // Set the information of the get variables step.
    String getVariablesPid = registry.getPluginId(StepPluginType.class, gvm);
    StepMeta getVariablesStep = new StepMeta(getVariablesPid, getVariablesStepname, gvm);
    transMeta.addStep(getVariablesStep);
    // 
    // Generate 1 row
    // 
    String[] fieldName = { "Param1", "PARAM2" };
    String[] varName = { "${Param1}", "%%PARAM2%%" };
    int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
    int[] length = { -1, -1 };
    int[] precision = { -1, -1 };
    String[] format = { "", "" };
    String[] currency = { "", "" };
    String[] decimal = { "", "" };
    String[] grouping = { "", "" };
    int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };
    FieldDefinition[] fields = new FieldDefinition[fieldName.length];
    for (int i = 0; i < fields.length; i++) {
        FieldDefinition field = new FieldDefinition();
        field.setFieldName(fieldName[i]);
        field.setVariableString(varName[i]);
        field.setFieldType(fieldType[i]);
        field.setFieldLength(length[i]);
        field.setFieldPrecision(precision[i]);
        field.setFieldFormat(format[i]);
        field.setCurrency(currency[i]);
        field.setDecimal(decimal[i]);
        field.setGroup(grouping[i]);
        field.setTrimType(trimType[i]);
        fields[i] = field;
    }
    gvm.setFieldDefinitions(fields);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi1 = new TransHopMeta(getVariablesStep, dummyStep1);
    transMeta.addTransHop(hi1);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.addParameterDefinition("Param1", "default1", "Parameter 1");
    trans.addParameterDefinition("PARAM2", "default2", "Parameter 2");
    trans.setParameterValue("Param1", "ParamValue1");
    // PARAM2 is not set
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector endRc = new RowStepCollector();
    si.addRowListener(endRc);
    trans.startThreads();
    trans.waitUntilFinished();
    // Now check whether the output is still as we expect.
    List<RowMetaAndData> goldenImageRows = createResultData2();
    List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
    checkRows(resultRows1, goldenImageRows);
}

Example 30 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class ParameterSimpleTransIT method testParameterSimpleTrans3.

/**
 * Test case for parameters using a simple transformation. Here blocking some unwise usage of parameters.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans3() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("parameter_simple_trans3");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create a get variables step...
    // 
    String getVariablesStepname = "get variables step";
    GetVariableMeta gvm = new GetVariableMeta();
    // Set the information of the get variables step.
    String getVariablesPid = registry.getPluginId(StepPluginType.class, gvm);
    StepMeta getVariablesStep = new StepMeta(getVariablesPid, getVariablesStepname, gvm);
    transMeta.addStep(getVariablesStep);
    // 
    // Generate 1 row
    // 
    String[] fieldName = { "PARAM1", "PARAM2" };
    String[] varName = { "${JAVA_HOME}", "%%PARAM2%%" };
    int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
    int[] length = { -1, -1 };
    int[] precision = { -1, -1 };
    String[] format = { "", "" };
    String[] currency = { "", "" };
    String[] decimal = { "", "" };
    String[] grouping = { "", "" };
    int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };
    FieldDefinition[] fields = new FieldDefinition[fieldName.length];
    for (int i = 0; i < fields.length; i++) {
        FieldDefinition field = new FieldDefinition();
        field.setFieldName(fieldName[i]);
        field.setVariableString(varName[i]);
        field.setFieldType(fieldType[i]);
        field.setFieldLength(length[i]);
        field.setFieldPrecision(precision[i]);
        field.setFieldFormat(format[i]);
        field.setCurrency(currency[i]);
        field.setDecimal(decimal[i]);
        field.setGroup(grouping[i]);
        field.setTrimType(trimType[i]);
        fields[i] = field;
    }
    gvm.setFieldDefinitions(fields);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi1 = new TransHopMeta(getVariablesStep, dummyStep1);
    transMeta.addTransHop(hi1);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.addParameterDefinition("${JAVA_HOME}", "default1", "Parameter 1");
    trans.addParameterDefinition("PARAM2", "default2", "Parameter 2");
    trans.setParameterValue("${JAVA_HOME}", "param1");
    // PARAM2 is not set
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector endRc = new RowStepCollector();
    si.addRowListener(endRc);
    trans.startThreads();
    trans.waitUntilFinished();
    // Now check whether the output is still as we expect.
    List<RowMetaAndData> goldenImageRows = createResultData3();
    List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
    checkRows(resultRows1, goldenImageRows);
}