Examples with RowStepCollector org.pentaho.di.trans.RowStepCollector used on opensource projects

Example 46 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class DatabaseLookupIT method CacheAndLoadAllRowsDatabaseLookup.

/**
 * Test "Load All Rows" version of BasicDatabaseLookup test.
 */
@Test
public void CacheAndLoadAllRowsDatabaseLookup() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("transname");
    // Add the database connections
    for (int i = 0; i < databasesXML.length; i++) {
        DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
        transMeta.addDatabase(databaseMeta);
    }
    DatabaseMeta dbInfo = transMeta.findDatabase("db");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // create the lookup step...
    // 
    String lookupName = "look up from [" + lookup_table + "]";
    DatabaseLookupMeta dbl = new DatabaseLookupMeta();
    dbl.setDatabaseMeta(transMeta.findDatabase("db"));
    dbl.setTablename(lookup_table);
    dbl.setCached(true);
    dbl.setLoadingAllDataInCache(true);
    dbl.setEatingRowOnLookupFailure(false);
    dbl.setFailingOnMultipleResults(false);
    dbl.setOrderByClause("");
    dbl.setTableKeyField(new String[] { "ID" });
    dbl.setKeyCondition(new String[] { "=" });
    dbl.setStreamKeyField1(new String[] { "int_field" });
    dbl.setStreamKeyField2(new String[] { "" });
    dbl.setReturnValueField(new String[] { "CODE", "STRING" });
    dbl.setReturnValueDefaultType(new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING });
    dbl.setReturnValueDefault(new String[] { "-1", "UNDEF" });
    dbl.setReturnValueNewName(new String[] { "RET_CODE", "RET_STRING" });
    String lookupId = registry.getPluginId(StepPluginType.class, dbl);
    StepMeta lookupStep = new StepMeta(lookupId, lookupName, dbl);
    lookupStep.setDescription("Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]");
    transMeta.addStep(lookupStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, lookupStep);
    transMeta.addTransHop(hi);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(lookupName, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createDataRows();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = rc.getRowsWritten();
    List<RowMetaAndData> goldRows = createResultDataRows();
    checkRows(goldRows, resultRows);
}

Example 47 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class AppendIT method testAppendStep.

/**
 * Test case for Append step. 2 Injector steps to an append step to a dummy step. Rows go in, the order should be as
 * defined in the append step.
 */
public void testAppendStep() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("Appendtest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step 1...
    // 
    String injectorStepname1 = "injector step 1";
    InjectorMeta im1 = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid1 = registry.getPluginId(StepPluginType.class, im1);
    StepMeta injectorStep1 = new StepMeta(injectorPid1, injectorStepname1, im1);
    transMeta.addStep(injectorStep1);
    // 
    // create an injector step 2...
    // 
    String injectorStepname2 = "injector step 2";
    InjectorMeta im2 = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid2 = registry.getPluginId(StepPluginType.class, im2);
    StepMeta injectorStep2 = new StepMeta(injectorPid2, injectorStepname2, im2);
    transMeta.addStep(injectorStep2);
    // 
    // Create an append step
    // 
    String appendName = "append step";
    AppendMeta am = new AppendMeta();
    List<StreamInterface> infoStreams = am.getStepIOMeta().getInfoStreams();
    infoStreams.get(0).setStepMeta(injectorStep1);
    infoStreams.get(1).setStepMeta(injectorStep2);
    String appendPid = registry.getPluginId(StepPluginType.class, am);
    StepMeta append = new StepMeta(appendPid, appendName, am);
    transMeta.addStep(append);
    TransHopMeta hi2 = new TransHopMeta(injectorStep1, append);
    transMeta.addTransHop(hi2);
    TransHopMeta hi3 = new TransHopMeta(injectorStep2, append);
    transMeta.addTransHop(hi3);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi4 = new TransHopMeta(append, dummyStep1);
    transMeta.addTransHop(hi4);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(appendName, 0);
    RowStepCollector blockingRc = new RowStepCollector();
    si.addRowListener(blockingRc);
    si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    RowProducer rp1 = trans.addRowProducer(injectorStepname1, 0);
    RowProducer rp2 = trans.addRowProducer(injectorStepname2, 0);
    trans.startThreads();
    // add rows to tail step
    List<RowMetaAndData> inputList2 = createData2();
    Iterator<RowMetaAndData> it2 = inputList2.iterator();
    while (it2.hasNext()) {
        RowMetaAndData rm = it2.next();
        rp2.putRow(rm.getRowMeta(), rm.getData());
    }
    rp2.finished();
    // add rows to head step
    List<RowMetaAndData> inputList1 = createData1();
    Iterator<RowMetaAndData> it1 = inputList1.iterator();
    while (it1.hasNext()) {
        RowMetaAndData rm = it1.next();
        rp1.putRow(rm.getRowMeta(), rm.getData());
    }
    rp1.finished();
    trans.waitUntilFinished();
    // The result should be that first all rows from injector 1 and
    // then all rows from injector step 2
    List<RowMetaAndData> expectedList = new ArrayList<RowMetaAndData>();
    expectedList.addAll(inputList1);
    expectedList.addAll(inputList2);
    List<RowMetaAndData> resultRows1 = dummyRc1.getRowsWritten();
    checkRows(resultRows1, expectedList);
}

Example 48 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class CalculatorIT method testCalculator1.

public void testCalculator1() throws Exception {
    KettleEnvironment.init();
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("calculatortest1");
    // 
    // create a row generator step...
    // 
    String rowGeneratorStepname = "row generator step";
    RowGeneratorMeta rm = new RowGeneratorMeta();
    // Set the information of the row generator.
    String rowGeneratorPid = registry.getPluginId(StepPluginType.class, rm);
    StepMeta rowGeneratorStep = new StepMeta(rowGeneratorPid, rowGeneratorStepname, rm);
    transMeta.addStep(rowGeneratorStep);
    // 
    // Generate 1 empty row
    // 
    String[] strDummies = {};
    int[] intDummies = {};
    rm.setDefault();
    rm.setFieldName(strDummies);
    rm.setFieldType(strDummies);
    rm.setValue(strDummies);
    rm.setFieldLength(intDummies);
    rm.setFieldPrecision(intDummies);
    rm.setRowLimit("1");
    rm.setFieldFormat(strDummies);
    rm.setGroup(strDummies);
    rm.setDecimal(strDummies);
    // 
    // Add calculator step.
    // 
    String calculatorStepname1 = "calculator 1";
    CalculatorMeta calc1 = new CalculatorMeta();
    CalculatorMetaFunction[] calculations = new CalculatorMetaFunction[] { new CalculatorMetaFunction(// fieldName
    "timestamp1", // calctype
    CalculatorMetaFunction.CALC_CONSTANT, // fieldA
    "1970-01-01 00:00:00.100100", // String fieldB
    "", // String fieldC
    "", // valueType,
    ValueMetaInterface.TYPE_TIMESTAMP, // int valueLength,
    0, // int valuePrecision,
    0, // boolean removedFromResult,
    false, // String conversionMask,
    "", // String decimalSymbol,
    "", // String groupingSymbol,
    "", // String currencySymbol
    ""), new CalculatorMetaFunction(// fieldName
    "int1", // calctype
    CalculatorMetaFunction.CALC_CONSTANT, // fieldA
    "1", // String fieldB
    "", // String fieldC
    "", // valueType,
    ValueMetaInterface.TYPE_INTEGER, // int valueLength,
    0, // int valuePrecision,
    0, // boolean removedFromResult,
    false, // String conversionMask,
    "", // String decimalSymbol,
    "", // String groupingSymbol,
    "", // String currencySymbol
    ""), new CalculatorMetaFunction(// fieldName
    "timestamp plus 1 day", // calctype
    CalculatorMetaFunction.CALC_ADD_DAYS, // fieldA
    "timestamp1", // String fieldB
    "int1", // String fieldC
    "", // valueType,
    ValueMetaInterface.TYPE_DATE, // int valueLength,
    0, // int valuePrecision,
    0, // boolean removedFromResult,
    false, // String conversionMask,
    "", // String decimalSymbol,
    "", // String groupingSymbol,
    "", // String currencySymbol
    "") };
    calc1.setCalculation(calculations);
    // 
    String calculatorPid1 = registry.getPluginId(StepPluginType.class, calc1);
    StepMeta calcualtorStep1 = new StepMeta(calculatorPid1, calculatorStepname1, calc1);
    transMeta.addStep(calcualtorStep1);
    // 
    TransHopMeta hi1 = new TransHopMeta(rowGeneratorStep, calcualtorStep1);
    transMeta.addTransHop(hi1);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(calculatorStepname1, 0);
    RowStepCollector endRc = new RowStepCollector();
    si.addRowListener(endRc);
    trans.startThreads();
    trans.waitUntilFinished();
    // Now check whether the output is still as we expect.
    List<RowMetaAndData> goldenImageRows = createResultData1();
    List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
    checkRows(resultRows1, goldenImageRows);
}

Example 49 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class ConstantIT method testConstant1.

/**
 * Test case for Constant step. Row generator attached to a constant step.
 */
public void testConstant1() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("constanttest1");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create a row generator step...
    // 
    String rowGeneratorStepname = "row generator step";
    RowGeneratorMeta rm = new RowGeneratorMeta();
    // Set the information of the row generator.
    String rowGeneratorPid = registry.getPluginId(StepPluginType.class, rm);
    StepMeta rowGeneratorStep = new StepMeta(rowGeneratorPid, rowGeneratorStepname, rm);
    transMeta.addStep(rowGeneratorStep);
    // 
    // Generate 1 empty row
    // 
    String[] fieldName = {};
    String[] type = {};
    String[] value = {};
    String[] fieldFormat = {};
    String[] group = {};
    String[] decimal = {};
    int[] intDummies = {};
    rm.setDefault();
    rm.setFieldName(fieldName);
    rm.setFieldType(type);
    rm.setValue(value);
    rm.setFieldLength(intDummies);
    rm.setFieldPrecision(intDummies);
    rm.setRowLimit("1");
    rm.setFieldFormat(fieldFormat);
    rm.setGroup(group);
    rm.setDecimal(decimal);
    // 
    // Add constant step.
    // 
    String constStepname1 = "constant 1";
    ConstantMeta cnst1 = new ConstantMeta();
    String[] fieldName1 = { "boolean1", "boolean2", "boolean3", "boolean4", "boolean5", "boolean6", "boolean7", "string1", "string2", "string3", "integer1", "integer2", "integer3", "integer4", "number1", "number2", "number3", "number4", "timestamp1" };
    String[] type1 = { "boolean", "Boolean", "bOOLEAN", "BOOLEAN", "boolean", "boolean", "boolean", "string", "string", "String", "integer", "integer", "integer", "integer", "number", "number", "number", "number", "timestamp" };
    String[] value1 = { "Y", "T", "a", "TRUE", "0", "9", "", "AAAAAAAAAAAAAA", "   ", "", "-100", "0", "212", "", "-100.2", "0.0", "212.23", "", "1970-01-01 00:00:00.000" };
    String[] fieldFormat1 = { "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" };
    String[] group1 = { "", "", "", "", "", "", "", "", "", "", "", "", "", "", ",", ",", ",", ",", "" };
    String[] decimal1 = { "", "", "", "", "", "", "", "", "", "", "", "", "", "", ".", ".", ".", ".", "" };
    String[] currency = { "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" };
    int[] intDummies1 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
    boolean[] setEmptystring = { false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false };
    cnst1.setFieldName(fieldName1);
    cnst1.setFieldType(type1);
    cnst1.setValue(value1);
    cnst1.setFieldLength(intDummies1);
    cnst1.setFieldPrecision(intDummies1);
    cnst1.setFieldFormat(fieldFormat1);
    cnst1.setGroup(group1);
    cnst1.setDecimal(decimal1);
    cnst1.setCurrency(currency);
    cnst1.setEmptyString(setEmptystring);
    String addSeqPid1 = registry.getPluginId(StepPluginType.class, cnst1);
    StepMeta addSeqStep1 = new StepMeta(addSeqPid1, constStepname1, cnst1);
    transMeta.addStep(addSeqStep1);
    TransHopMeta hi1 = new TransHopMeta(rowGeneratorStep, addSeqStep1);
    transMeta.addTransHop(hi1);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(constStepname1, 0);
    RowStepCollector endRc = new RowStepCollector();
    si.addRowListener(endRc);
    trans.startThreads();
    trans.waitUntilFinished();
    // Now check whether the output is still as we expect.
    List<RowMetaAndData> goldenImageRows = createResultData1();
    List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
    checkRows(resultRows1, goldenImageRows);
}

Example 50 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class XsltTest method runTestWithParams.

public void runTestWithParams(String xmlFieldname, String resultFieldname, boolean xslInField, boolean xslFileInField, String xslFileField, String xslFilename, String xslFactory) throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("xslt");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a XSLT step
    // 
    String xsltName = "xslt step";
    XsltMeta xm = new XsltMeta();
    String xsltPid = registry.getPluginId(StepPluginType.class, xm);
    StepMeta xsltStep = new StepMeta(xsltPid, xsltName, xm);
    transMeta.addStep(xsltStep);
    TextFileInputField[] fields = new TextFileInputField[3];
    for (int idx = 0; idx < fields.length; idx++) {
        fields[idx] = new TextFileInputField();
    }
    fields[0].setName("XML");
    fields[0].setType(ValueMetaInterface.TYPE_STRING);
    fields[0].setFormat("");
    fields[0].setLength(-1);
    fields[0].setPrecision(-1);
    fields[0].setCurrencySymbol("");
    fields[0].setDecimalSymbol("");
    fields[0].setGroupSymbol("");
    fields[0].setTrimType(ValueMetaInterface.TRIM_TYPE_NONE);
    fields[1].setName("XSL");
    fields[1].setType(ValueMetaInterface.TYPE_STRING);
    fields[1].setFormat("");
    fields[1].setLength(-1);
    fields[1].setPrecision(-1);
    fields[1].setCurrencySymbol("");
    fields[1].setDecimalSymbol("");
    fields[1].setGroupSymbol("");
    fields[1].setTrimType(ValueMetaInterface.TRIM_TYPE_NONE);
    fields[2].setName("filename");
    fields[2].setType(ValueMetaInterface.TYPE_STRING);
    fields[2].setFormat("");
    fields[2].setLength(-1);
    fields[2].setPrecision(-1);
    fields[2].setCurrencySymbol("");
    fields[2].setDecimalSymbol("");
    fields[2].setGroupSymbol("");
    fields[2].setTrimType(ValueMetaInterface.TRIM_TYPE_NONE);
    xm.setFieldname(xmlFieldname);
    xm.setResultfieldname(resultFieldname);
    xm.setXSLField(xslInField);
    xm.setXSLFileField(xslFileField);
    xm.setXSLFieldIsAFile(xslFileInField);
    xm.setXslFilename(xslFilename);
    xm.setXSLFactory(xslFactory);
    TransHopMeta hi = new TransHopMeta(injectorStep, xsltStep);
    transMeta.addTransHop(hi);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi1 = new TransHopMeta(xsltStep, dummyStep1);
    transMeta.addTransHop(hi1);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData(xslFilename);
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData1();
    checkRows(goldenImageRows, resultRows, 2);
}