Examples with RowStepCollector org.pentaho.di.trans.RowStepCollector used on opensource projects

Example 56 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class DatabaseLookupIT method NOTCachedAndLoadAllRowsDatabaseLookup.

/**
 * Test with cache turned off but "Load All Rows" enabled (Load all rows should have no effect) See JIRA PDI-1910
 */
@Test
public void NOTCachedAndLoadAllRowsDatabaseLookup() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("transname");
    // Add the database connections
    for (int i = 0; i < databasesXML.length; i++) {
        DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
        transMeta.addDatabase(databaseMeta);
    }
    DatabaseMeta dbInfo = transMeta.findDatabase("db");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // create the lookup step...
    // 
    String lookupName = "look up from [" + lookup_table + "]";
    DatabaseLookupMeta dbl = new DatabaseLookupMeta();
    dbl.setDatabaseMeta(transMeta.findDatabase("db"));
    dbl.setTablename(lookup_table);
    dbl.setCached(false);
    dbl.setLoadingAllDataInCache(true);
    dbl.setEatingRowOnLookupFailure(false);
    dbl.setFailingOnMultipleResults(false);
    dbl.setOrderByClause("");
    dbl.setTableKeyField(new String[] { "ID" });
    dbl.setKeyCondition(new String[] { "=" });
    dbl.setStreamKeyField1(new String[] { "int_field" });
    dbl.setStreamKeyField2(new String[] { "" });
    dbl.setReturnValueField(new String[] { "CODE", "STRING" });
    dbl.setReturnValueDefaultType(new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING });
    dbl.setReturnValueDefault(new String[] { "-1", "UNDEF" });
    dbl.setReturnValueNewName(new String[] { "RET_CODE", "RET_STRING" });
    String lookupId = registry.getPluginId(StepPluginType.class, dbl);
    StepMeta lookupStep = new StepMeta(lookupId, lookupName, dbl);
    lookupStep.setDescription("Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]");
    transMeta.addStep(lookupStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, lookupStep);
    transMeta.addTransHop(hi);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(lookupName, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createDataRows();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = rc.getRowsWritten();
    List<RowMetaAndData> goldRows = createResultDataRows();
    checkRows(goldRows, resultRows);
}

Example 57 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class BlockingStepIT method testBlockingStepPassAll.

/**
 * Test case for blocking step step passing all rows. Injector step to a blocking step to a dummy step. rows go in,
 * all rows should be output.
 */
public void testBlockingStepPassAll() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("blockingsteptest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a blocking step
    // 
    String blockingStepname = "blocking step";
    BlockingStepMeta bm = new BlockingStepMeta();
    bm.setPassAllRows(true);
    String blockingStepPid = registry.getPluginId(StepPluginType.class, bm);
    StepMeta blockingStep = new StepMeta(blockingStepPid, blockingStepname, bm);
    transMeta.addStep(blockingStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, blockingStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(blockingStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(blockingStepname, 0);
    RowStepCollector blockingRc = new RowStepCollector();
    si.addRowListener(blockingRc);
    si = trans.getStepInterface(dummyStepname2, 0);
    RowStepCollector dummyRc2 = new RowStepCollector();
    si.addRowListener(dummyRc2);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // The results should be that dummy1 gets all rows.
    // blocking step should receive all rows (but only send the
    // last one through). dummy2 should only get the last row.
    List<RowMetaAndData> resultRows1 = dummyRc1.getRowsWritten();
    checkRows(resultRows1, inputList);
    List<RowMetaAndData> resultRows2 = blockingRc.getRowsWritten();
    checkRows(resultRows2, inputList);
    List<RowMetaAndData> resultRows3 = dummyRc2.getRowsWritten();
    checkRows(resultRows3, inputList);
}

Example 58 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class ParameterSimpleTransIT method testParameterSimpleTrans5.

/**
 * Test case for parameters using a simple transformation. Check whether parameters override variables.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans5() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("parameter_simple_trans4");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create a get variables step...
    // 
    String getVariablesStepname = "get variables step";
    GetVariableMeta gvm = new GetVariableMeta();
    // Set the information of the get variables step.
    String getVariablesPid = registry.getPluginId(StepPluginType.class, gvm);
    StepMeta getVariablesStep = new StepMeta(getVariablesPid, getVariablesStepname, gvm);
    transMeta.addStep(getVariablesStep);
    // 
    // Generate 1 row
    // 
    String[] fieldName = { "PARAM1", "PARAM2" };
    String[] varName = { "${Param1}", "%%PARAM2%%" };
    int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
    int[] length = { -1, -1 };
    int[] precision = { -1, -1 };
    String[] format = { "", "" };
    String[] currency = { "", "" };
    String[] decimal = { "", "" };
    String[] grouping = { "", "" };
    int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };
    FieldDefinition[] fields = new FieldDefinition[fieldName.length];
    for (int i = 0; i < fields.length; i++) {
        FieldDefinition field = new FieldDefinition();
        field.setFieldName(fieldName[i]);
        field.setVariableString(varName[i]);
        field.setFieldType(fieldType[i]);
        field.setFieldLength(length[i]);
        field.setFieldPrecision(precision[i]);
        field.setFieldFormat(format[i]);
        field.setCurrency(currency[i]);
        field.setDecimal(decimal[i]);
        field.setGroup(grouping[i]);
        field.setTrimType(trimType[i]);
        fields[i] = field;
    }
    gvm.setFieldDefinitions(fields);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi1 = new TransHopMeta(getVariablesStep, dummyStep1);
    transMeta.addTransHop(hi1);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.addParameterDefinition("Param1", "default1", "Parameter 1");
    trans.addParameterDefinition("PARAM2", "", "Parameter 2");
    trans.setParameterValue("PARAM2", "PARAMVALUE2");
    // See whether this variable overrides the parameter... it should NOT. Param1
    // is defined but not set, so defaults should kick in.
    trans.setVariable("Param1", "Variable1");
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector endRc = new RowStepCollector();
    si.addRowListener(endRc);
    trans.startThreads();
    trans.waitUntilFinished();
    // Now check whether the output is still as we expect.
    List<RowMetaAndData> goldenImageRows = createResultData5();
    List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
    checkRows(resultRows1, goldenImageRows);
}

Example 59 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class GetXMLDataTest method testGetXMLData.

/**
 * Test case for Get XML Data step, very simple example.
 *
 * @throws Exception
 *           Upon any exception
 */
private void testGetXMLData(String missingElementValue) throws Exception {
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("getxmldata1");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a Get XML Data step
    // 
    String getXMLDataName = "get xml data step";
    GetXMLDataMeta gxdm = new GetXMLDataMeta();
    String getXMLDataPid = registry.getPluginId(StepPluginType.class, gxdm);
    StepMeta getXMLDataStep = new StepMeta(getXMLDataPid, getXMLDataName, gxdm);
    transMeta.addStep(getXMLDataStep);
    GetXMLDataField[] fields = new GetXMLDataField[5];
    for (int idx = 0; idx < fields.length; idx++) {
        fields[idx] = new GetXMLDataField();
    }
    fields[0].setName("objectid");
    fields[0].setXPath("ObjectID");
    fields[0].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
    fields[0].setType(ValueMetaInterface.TYPE_STRING);
    fields[0].setFormat("");
    fields[0].setLength(-1);
    fields[0].setPrecision(-1);
    fields[0].setCurrencySymbol("");
    fields[0].setDecimalSymbol("");
    fields[0].setGroupSymbol("");
    fields[0].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
    fields[1].setName("sapident");
    fields[1].setXPath("SAPIDENT");
    fields[1].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
    fields[1].setType(ValueMetaInterface.TYPE_STRING);
    fields[1].setFormat("");
    fields[1].setLength(-1);
    fields[1].setPrecision(-1);
    fields[1].setCurrencySymbol("");
    fields[1].setDecimalSymbol("");
    fields[1].setGroupSymbol("");
    fields[1].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
    fields[2].setName("quantity");
    fields[2].setXPath("Quantity");
    fields[2].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
    fields[2].setType(ValueMetaInterface.TYPE_STRING);
    fields[2].setFormat("");
    fields[2].setLength(-1);
    fields[2].setPrecision(-1);
    fields[2].setCurrencySymbol("");
    fields[2].setDecimalSymbol("");
    fields[2].setGroupSymbol("");
    fields[2].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
    fields[3].setName("merkmalname");
    fields[3].setXPath("Merkmalname");
    fields[3].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
    fields[3].setType(ValueMetaInterface.TYPE_STRING);
    fields[3].setFormat("");
    fields[3].setLength(-1);
    fields[3].setPrecision(-1);
    fields[3].setCurrencySymbol("");
    fields[3].setDecimalSymbol("");
    fields[3].setGroupSymbol("");
    fields[3].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
    fields[4].setName("merkmalswert");
    fields[4].setXPath("Merkmalswert");
    fields[4].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
    fields[4].setType(ValueMetaInterface.TYPE_STRING);
    fields[4].setFormat("");
    fields[4].setLength(-1);
    fields[4].setPrecision(-1);
    fields[4].setCurrencySymbol("");
    fields[4].setDecimalSymbol("");
    fields[4].setGroupSymbol("");
    fields[4].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
    gxdm.setEncoding("UTF-8");
    gxdm.setIsAFile(false);
    gxdm.setInFields(true);
    gxdm.setLoopXPath("Level1/Level2/Props");
    gxdm.setXMLField("field1");
    gxdm.setInputFields(fields);
    TransHopMeta hi = new TransHopMeta(injectorStep, getXMLDataStep);
    transMeta.addTransHop(hi);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi1 = new TransHopMeta(getXMLDataStep, dummyStep1);
    transMeta.addTransHop(hi1);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData(missingElementValue);
    checkRows(goldenImageRows, resultRows);
}

Example 60 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class TextFileInputIT method testInputFilesAndFoldersFromPreviousStep.

/**
 * This test case uses a datagrid to test multiple issues when using the "Accept filenames from previous step"
 *    * Tests both files and folders as value of the same field.
 *    * Ensures additional fields match the expected value for each row.
 *   KTR:
 *      datagrid -> text file input -> switch (on nulls) -> dummys steps
 *   Cases:
 *     PDI-17117
 *     PDI-18752
 *     PDI-18818
 *     BACKLOG-34414
 * @throws KettleException
 */
@Test
public void testInputFilesAndFoldersFromPreviousStep() throws KettleException {
    final int TEXT_FILE_INPUT = 0;
    final int DUMMY_CATCH_NULLS = 3;
    final int DATA_GRID = 4;
    final String TEST_FOLDER_PATH = "${testfolder}/multiple";
    KettleEnvironment.init();
    String path = getClass().getResource("text-file-input-from-datagrid.ktr").getPath();
    Variables variables = new Variables();
    variables.setVariable("testfolder", getClass().getResource("").getPath());
    RowStepCollector collector = new RowStepCollector();
    TransMeta transMeta = new TransMeta(path, variables);
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    trans.getSteps().get(TEXT_FILE_INPUT).step.addRowListener(collector);
    List<RowMetaAndData> rowsWritten = collector.getRowsWritten();
    trans.startThreads();
    trans.waitUntilFinished();
    // Did we read expected values from the ?
    assertEquals(4, trans.getSteps().get(DATA_GRID).step.getLinesWritten());
    // Did we read expected amount of rows? ( Expect to read contents 6 times. 6 files * 3 rows = 18 total )
    assertEquals(18, trans.getSteps().get(TEXT_FILE_INPUT).step.getLinesWritten());
    // Did we find any nulls?
    assertEquals(0, trans.getSteps().get(DUMMY_CATCH_NULLS).step.getLinesRead());
    // Check Additional fields to work as expected.
    for (int i = 0; i < rowsWritten.size(); i++) {
        Object[] data = rowsWritten.get(i).getData();
        switch(i) {
            case 0:
                // Single File "multiple/debug.csv"
                assertEquals("${testfolder}/multiple/debug.csv", data[0]);
                assertEquals("1111", data[1]);
                break;
            case 3:
            case 6:
                // Folder "multiple" file 1
                assertEquals(TEST_FOLDER_PATH, data[0]);
                assertEquals("2222", data[1]);
                break;
            case 9:
                // Single File "multiple/debug2.csv"
                assertEquals("${testfolder}/multiple/debug2.csv", data[0]);
                assertEquals("3333", data[1]);
                break;
            case 12:
            case 15:
                // Folder "multiple" file 1
                assertEquals(TEST_FOLDER_PATH, data[0]);
                assertEquals("4444", data[1]);
                break;
            default:
                // / Do nothing
                continue;
        }
    }
}