Examples with RowStepCollector org.pentaho.di.trans.RowStepCollector used on opensource projects

Example 36 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class JaninoStepIT method testJaninoStep.

/**
 * Test case for janino step.
 */
@Test
public void testJaninoStep() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("janino test");
    PluginRegistry registry = PluginRegistry.getInstance();
    // create an injector step...
    String injectorStepName = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepName, im);
    transMeta.addStep(injectorStep);
    // 
    // create a janino step...
    // 
    String stepname = "janino";
    JaninoMeta jm = new JaninoMeta();
    // Set the information of the step
    String janinoPid = registry.getPluginId(StepPluginType.class, jm);
    StepMeta janinoStep = new StepMeta(janinoPid, stepname, jm);
    transMeta.addStep(janinoStep);
    jm.setDefault();
    JaninoMetaFunction[] formulas = { new JaninoMetaFunction("string", "(string==null)?null:\"string-value\"", ValueMetaInterface.TYPE_STRING, -1, -1, "string"), new JaninoMetaFunction("integer", "(integer==null)?null:new Long(42L)", ValueMetaInterface.TYPE_INTEGER, -1, -1, "integer"), new JaninoMetaFunction("number", "(number==null)?null:new Double(23.0)", ValueMetaInterface.TYPE_NUMBER, -1, -1, "number"), new JaninoMetaFunction("bigdecimal", "(bigdecimal==null)?null:new java.math.BigDecimal(11.0)", ValueMetaInterface.TYPE_BIGNUMBER, -1, -1, "bigdecimal"), new JaninoMetaFunction("date", "(date==null)?null:new java.util.Date(10000000)", ValueMetaInterface.TYPE_DATE, -1, -1, "date"), new JaninoMetaFunction("binary", "(binary==null)?null:new byte[]{1,2,3,4,5}", ValueMetaInterface.TYPE_BINARY, -1, -1, "binary"), new JaninoMetaFunction("bool", "(bool==null)?null:Boolean.TRUE", ValueMetaInterface.TYPE_BOOLEAN, -1, -1, "bool"), new JaninoMetaFunction("timestamp", "(timestamp==null)?null:new java.sql.Timestamp(0L)", ValueMetaInterface.TYPE_TIMESTAMP, -1, -1, "timestamp"), new JaninoMetaFunction("inetaddress", "(inetaddress==null)?null:java.net.InetAddress.getByAddress( new byte[]{ 127, 0, 0, 1} )", ValueMetaInterface.TYPE_INET, -1, -1, "inetaddress") };
    jm.setFormula(formulas);
    transMeta.addTransHop(new TransHopMeta(injectorStep, janinoStep));
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    TransHopMeta hi = new TransHopMeta(janinoStep, dummyStep);
    transMeta.addTransHop(hi);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer(injectorStepName, 0);
    trans.startThreads();
    for (RowMetaAndData rm : createInputList()) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> checkList = createExpectedList();
    List<RowMetaAndData> resultRows = rc.getRowsWritten();
    checkRows(resultRows, checkList);
}

Example 37 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class MappingIT method testInfoStreams_single.

/**
 * Tests that info steps are correctly identified via StepMetaInterface.getStepIOMeta()
 */
public void testInfoStreams_single() throws Exception {
    KettleEnvironment.init();
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // Create a new transformation with a row generator that feeds a Mapping (Sub-Transformation) Step
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("Mapping Info Test");
    StepMeta rowGenerator = buildRowGeneratorStep(registry, "Generate Rows");
    transMeta.addStep(rowGenerator);
    String mappingName = "mapping";
    MappingMeta mappingMeta = new MappingMeta();
    mappingMeta.setSpecificationMethod(ObjectLocationSpecificationMethod.FILENAME);
    mappingMeta.setFileName("test/org/pentaho/di/trans/steps/mapping/subtrans.ktr");
    String mappingInputStepName = "input";
    mappingMeta.setInputMappings(Collections.singletonList(createMappingDef(rowGenerator.getName(), mappingInputStepName, "string", "a")));
    String mappingPid = registry.getPluginId(StepPluginType.class, mappingMeta);
    StepMeta mapping = new StepMeta(mappingPid, mappingName, mappingMeta);
    transMeta.addStep(mapping);
    TransHopMeta hopGeneratorToMapping = new TransHopMeta(rowGenerator, mapping);
    transMeta.addTransHop(hopGeneratorToMapping);
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    // (Copied from TransMeta.loadXML())
    for (int i = 0; i < transMeta.nrSteps(); i++) {
        StepMeta stepMeta = transMeta.getStep(i);
        StepMetaInterface sii = stepMeta.getStepMetaInterface();
        if (sii != null) {
            sii.searchInfoAndTargetSteps(transMeta.getSteps());
        }
    }
    // Verify the transformation was configured properly
    assertEquals("Transformation not initialized properly", 2, transMeta.nrSteps());
    StepMeta meta = transMeta.getStep(1);
    assertTrue("Transformation not initialized properly", meta.getStepMetaInterface() instanceof MappingMeta);
    MappingMeta loadedMappingMeta = (MappingMeta) meta.getStepMetaInterface();
    assertEquals("Expected a single input mapping definition", 1, loadedMappingMeta.getInputMappings().size());
    StepIOMetaInterface ioMeta = loadedMappingMeta.getStepIOMeta();
    assertEquals("Expected a single Info Stream", 1, ioMeta.getInfoStreams().size());
    assertEquals("Expected a single Info Step", 1, loadedMappingMeta.getInfoSteps().length);
    // Verify the transformation can be executed
    StepInterface si = trans.getStepInterface(mappingName, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    trans.startThreads();
    trans.waitUntilFinished();
    assertEquals(1, rc.getRowsRead().size());
    assertEquals(1, rc.getRowsWritten().size());
}

Example 38 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class MergeRowsIT method testOneRow.

void testOneRow(String transName, String[] referenceValues, String[] comparisonValues, Object[] goldenImageRowValues) throws Exception {
    KettleEnvironment.init();
    // Create a new transformation...
    TransMeta transMeta = new TransMeta();
    transMeta.setName(transName);
    PluginRegistry registry = PluginRegistry.getInstance();
    // Create a merge rows step
    String mergeRowsStepName = "merge rows step";
    MergeRowsMeta mergeRowsMeta = new MergeRowsMeta();
    String mergeRowsStepPid = registry.getPluginId(StepPluginType.class, mergeRowsMeta);
    StepMeta mergeRowsStep = new StepMeta(mergeRowsStepPid, mergeRowsStepName, mergeRowsMeta);
    transMeta.addStep(mergeRowsStep);
    mergeRowsMeta.setKeyFields(new String[] { keyField });
    mergeRowsMeta.setValueFields(new String[] { compareField });
    mergeRowsMeta.setFlagField(flagField);
    List<StreamInterface> infoStreams = mergeRowsMeta.getStepIOMeta().getInfoStreams();
    // 
    // create a reference stream (row generator step)
    // 
    createRowGenerator(transMeta, registry, "reference row generator", referenceValues, mergeRowsStep, mergeRowsMeta, 0);
    // 
    // create a comparison stream (row generator step)
    // 
    createRowGenerator(transMeta, registry, "comparison row generator", comparisonValues, mergeRowsStep, mergeRowsMeta, 1);
    // Now execute the transformation
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(mergeRowsStepName, 0);
    RowStepCollector endRc = new RowStepCollector();
    si.addRowListener(endRc);
    trans.startThreads();
    trans.waitUntilFinished();
    // Now check whether the output is still as we expect.
    List<RowMetaAndData> goldenImageRows = createResultData(goldenImageRowValues);
    List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
    checkRows(resultRows1, goldenImageRows);
}

Example 39 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class RestInputIT method testRESTInput.

@Test
public void testRESTInput() throws Exception {
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("restinput");
    PluginRegistry registry = PluginRegistry.getInstance();
    StepMeta inputStep = createRestInputStep(transMeta, registry);
    RowStepCollector rowStepCollector = new RowStepCollector();
    final Trans trans = createAndTestTrans(registry, transMeta, inputStep, rowStepCollector, "limit", 5);
    trans.startThreads();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = rowStepCollector.getRowsWritten();
    assertTrue(rowStepCollector.getRowsError().isEmpty());
    assertEquals(1, rowStepCollector.getRowsWritten().size());
    final RowMetaAndData rowMetaAndData = resultRows.get(0);
    final RowMetaInterface rowMeta = rowMetaAndData.getRowMeta();
    final String[] fieldNames = rowMeta.getFieldNames();
    final Object[] data = rowMetaAndData.getData();
    assertEquals("pageSize", fieldNames[0]);
    assertEquals("name", fieldNames[1]);
    assertEquals("result", fieldNames[2]);
    assertEquals(Integer.valueOf(5), data[0]);
    assertEquals("limit", data[1]);
    assertEquals("limit:5", data[2]);
}

Example 40 with RowStepCollector

use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.

the class RowGeneratorIT method testRowGenerator.

/**
 * Test case for Row Generator step.
 */
public void testRowGenerator() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("row generatortest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create a row generator step...
    // 
    String rowGeneratorStepname = "row generator step";
    RowGeneratorMeta rm = new RowGeneratorMeta();
    // Set the information of the row generator.
    String rowGeneratorPid = registry.getPluginId(StepPluginType.class, rm);
    StepMeta rowGeneratorStep = new StepMeta(rowGeneratorPid, rowGeneratorStepname, rm);
    transMeta.addStep(rowGeneratorStep);
    // 
    // Do the following specs 3 times.
    // 
    String[] fieldName = { "string", "boolean", "integer", "timestamp" };
    String[] type = { "String", "Boolean", "Integer", "Timestamp" };
    String[] value = { "string_value", "true", "20", "1970-01-01 00:00:00.000" };
    String[] fieldFormat = { "", "", "", "" };
    String[] group = { "", "", "", "" };
    String[] decimal = { "", "", "", "" };
    String[] currency = { "", "", "", "" };
    int[] intDummies = { -1, -1, -1, -1 };
    boolean[] setEmptystring = { false, false, false, false };
    rm.setDefault();
    rm.setFieldName(fieldName);
    rm.setFieldType(type);
    rm.setValue(value);
    rm.setFieldLength(intDummies);
    rm.setFieldPrecision(intDummies);
    rm.setRowLimit("3");
    rm.setFieldFormat(fieldFormat);
    rm.setGroup(group);
    rm.setDecimal(decimal);
    rm.setCurrency(currency);
    rm.setEmptyString(setEmptystring);
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    TransHopMeta hi = new TransHopMeta(rowGeneratorStep, dummyStep);
    transMeta.addTransHop(hi);
    // Now execute the transformation
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    trans.startThreads();
    trans.waitUntilFinished();
    List<RowMetaAndData> checkList = createData();
    List<RowMetaAndData> resultRows = rc.getRowsWritten();
    checkRows(resultRows, checkList);
}