Examples with RowProducer org.pentaho.di.trans.RowProducer used on opensource projects

Example 41 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class SimpleMappingTest method testStepShouldStopProcessingInput_IfUnderlyingTransitionIsStopped.

@Test
public void testStepShouldStopProcessingInput_IfUnderlyingTransitionIsStopped() throws Exception {
    MappingInput mappingInput = mock(MappingInput.class);
    when(mappingInput.getStepname()).thenReturn(MAPPING_INPUT_STEP_NAME);
    stepMockHelper.processRowsStepDataInterface.mappingInput = mappingInput;
    RowProducer rowProducer = mock(RowProducer.class);
    when(rowProducer.putRow(any(RowMetaInterface.class), any(Object[].class), anyBoolean())).thenReturn(true);
    StepInterface stepInterface = mock(StepInterface.class);
    Trans mappingTrans = mock(Trans.class);
    when(mappingTrans.addRowProducer(anyString(), anyInt())).thenReturn(rowProducer);
    when(mappingTrans.findStepInterface(anyString(), anyInt())).thenReturn(stepInterface);
    when(mappingTrans.isFinishedOrStopped()).thenReturn(Boolean.FALSE).thenReturn(Boolean.TRUE);
    stepMockHelper.processRowsStepDataInterface.mappingTrans = mappingTrans;
    MappingOutput mappingOutput = mock(MappingOutput.class);
    when(mappingOutput.getStepname()).thenReturn(MAPPING_OUTPUT_STEP_NAME);
    stepMockHelper.processRowsStepDataInterface.mappingOutput = mappingOutput;
    smp = new SimpleMapping(stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta, stepMockHelper.trans);
    smp.init(stepMockHelper.initStepMetaInterface, simpleMpData);
    smp.addRowSetToInputRowSets(stepMockHelper.getMockInputRowSet(new Object[] {}));
    smp.addRowSetToInputRowSets(stepMockHelper.getMockInputRowSet(new Object[] {}));
    assertTrue(smp.processRow(stepMockHelper.processRowsStepMetaInterface, stepMockHelper.processRowsStepDataInterface));
    assertFalse(smp.processRow(stepMockHelper.processRowsStepMetaInterface, stepMockHelper.processRowsStepDataInterface));
}

Example 42 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class NullIfIT method testNullIf1.

/**
 * Test case for nullif step. Injector step to a nullif step to a dummy step.
 */
public void testNullIf1() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("nulliftest1");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a NullIf step
    // 
    String nullIfName = "nullif step";
    NullIfMeta ni = new NullIfMeta();
    Field[] fields = new Field[2];
    fields[0] = new Field();
    fields[0].setFieldName("field2");
    fields[0].setFieldValue("1");
    fields[1] = new Field();
    fields[1].setFieldName("field1");
    fields[1].setFieldValue("tst");
    ni.setFields(fields);
    String nullIfPid = registry.getPluginId(StepPluginType.class, ni);
    StepMeta nullIfStep = new StepMeta(nullIfPid, nullIfName, ni);
    transMeta.addStep(nullIfStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, nullIfStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(nullIfStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(nullIfName, 0);
    RowStepCollector nullIfRc = new RowStepCollector();
    si.addRowListener(nullIfRc);
    si = trans.getStepInterface(dummyStepname2, 0);
    RowStepCollector dummyRc = new RowStepCollector();
    si.addRowListener(dummyRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData1();
    checkRows(goldenImageRows, resultRows);
}

Example 43 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class JavaScriptSpecialIT method testTransStatus.

/**
 * Test case for javascript functionality: trans_Status and SKIP_TRANSFORMATION. Regression test case for JIRA defect
 * PDI-364.
 */
public void testTransStatus() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("test javascript trans_Status");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a javascript step
    // 
    String javaScriptStepname = "javascript step";
    ScriptValuesMetaMod svm = new ScriptValuesMetaMod();
    // process 3 rows and skip the rest.
    ScriptValuesScript[] js = new ScriptValuesScript[] { new ScriptValuesScript(ScriptValuesScript.TRANSFORM_SCRIPT, "script", "trans_Status = CONTINUE_TRANSFORMATION;\n" + "if (getProcessCount(\"r\") > 3) {\n" + " \ttrans_Status = SKIP_TRANSFORMATION;\n" + "}") };
    svm.setJSScripts(js);
    svm.setFieldname(new String[] {});
    svm.setRename(new String[] {});
    svm.setType(new int[] {});
    svm.setLength(new int[] {});
    svm.setPrecision(new int[] {});
    svm.setCompatible(false);
    String javaScriptStepPid = registry.getPluginId(StepPluginType.class, svm);
    StepMeta javaScriptStep = new StepMeta(javaScriptStepPid, javaScriptStepname, svm);
    transMeta.addStep(javaScriptStep);
    TransHopMeta hi1 = new TransHopMeta(injectorStep, javaScriptStep);
    transMeta.addTransHop(hi1);
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    TransHopMeta hi2 = new TransHopMeta(javaScriptStep, dummyStep);
    transMeta.addTransHop(hi2);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si;
    si = trans.getStepInterface(javaScriptStepname, 0);
    RowStepCollector javaScriptRc = new RowStepCollector();
    si.addRowListener(javaScriptRc);
    si = trans.getStepInterface(dummyStepname, 0);
    RowStepCollector dummyRc = new RowStepCollector();
    si.addRowListener(dummyRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData1();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> goldenImageRows = createResultData2();
    List<RowMetaAndData> resultRows1 = javaScriptRc.getRowsWritten();
    checkRows(resultRows1, goldenImageRows);
    List<RowMetaAndData> resultRows2 = dummyRc.getRowsRead();
    checkRows(resultRows2, goldenImageRows);
}

Example 44 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class JavaScriptSpecialIT method disabledTestJavaInterface.

/**
 * Test case for JavaScript/Java/JavaScript interfacing.
 */
public void disabledTestJavaInterface() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("test javascript interface");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a javascript step
    // 
    String javaScriptStepname = "javascript step";
    ScriptValuesMetaMod svm = new ScriptValuesMetaMod();
    // process 3 rows and skip the rest.
    ScriptValuesScript[] js = new ScriptValuesScript[] { new ScriptValuesScript(ScriptValuesScript.TRANSFORM_SCRIPT, "script1", "java;\n\n" + "var obj     = new Packages.org.pentaho.di.trans.steps.scriptvalues_mod.JavaScriptTest();\n" + "var long1   = obj.add1ToLong(getInputRowMeta().getInteger(row, 0));\n" + "var number1 = obj.add1ToNumber(getInputRowMeta().getNumber(row, 1));\n" + "var string1 = obj.add1ToString(getInputRowMeta().getString(row, 2));\n" + "var long2   = Packages.org.pentaho.di.trans.steps.scriptvalues_mod." + "JavaScriptTest.add1ToLongStatic(getInputRowMeta().getInteger(row, 0));\n" + "var number2 = Packages.org.pentaho.di.trans.steps.scriptvalues_mod." + "JavaScriptTest.add1ToNumberStatic(getInputRowMeta().getNumber(row, 1));\n" + "var string2 = Packages.org.pentaho.di.trans.steps.scriptvalues_mod." + "JavaScriptTest.add1ToStringStatic(getInputRowMeta().getString(row, 2));\n") };
    svm.setJSScripts(js);
    svm.setFieldname(new String[] { "long1", "number1", "string1", "long2", "number2", "string2" });
    svm.setRename(new String[] { "long1", "number1", "string1", "long2", "number2", "string2" });
    svm.setType(new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_NUMBER, ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_NUMBER, ValueMetaInterface.TYPE_STRING });
    svm.setLength(new int[] { -1, -1, -1, -1, -1, -1, -1 });
    svm.setPrecision(new int[] { -1, -1, -1, -1, -1, -1, -1 });
    svm.setReplace(new boolean[] { false, false, false, false, false, false });
    svm.setCompatible(false);
    String javaScriptStepPid = registry.getPluginId(StepPluginType.class, svm);
    StepMeta javaScriptStep = new StepMeta(javaScriptStepPid, javaScriptStepname, svm);
    transMeta.addStep(javaScriptStep);
    TransHopMeta hi1 = new TransHopMeta(injectorStep, javaScriptStep);
    transMeta.addTransHop(hi1);
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    TransHopMeta hi2 = new TransHopMeta(javaScriptStep, dummyStep);
    transMeta.addTransHop(hi2);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si;
    si = trans.getStepInterface(javaScriptStepname, 0);
    RowStepCollector javaScriptRc = new RowStepCollector();
    si.addRowListener(javaScriptRc);
    si = trans.getStepInterface(dummyStepname, 0);
    RowStepCollector dummyRc = new RowStepCollector();
    si.addRowListener(dummyRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData3();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> goldenImageRows = createResultData3();
    List<RowMetaAndData> resultRows1 = javaScriptRc.getRowsWritten();
    checkRows(resultRows1, goldenImageRows);
    List<RowMetaAndData> resultRows2 = dummyRc.getRowsRead();
    checkRows(resultRows2, goldenImageRows);
}

Example 45 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class JavaScriptStringIT method testStringsTrim.

/**
 * Test case for javascript functionality: ltrim(), rtrim(), trim().
 */
public void testStringsTrim() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("test javascript trim");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a javascript step
    // 
    String javaScriptStepname = "javascript step";
    ScriptValuesMetaMod svm = new ScriptValuesMetaMod();
    ScriptValuesScript[] js = new ScriptValuesScript[] { new ScriptValuesScript(ScriptValuesScript.TRANSFORM_SCRIPT, "script", "var original = 'b' + string.getString() + 'e';\n" + "var ltrimStr = 'b' + ltrim(string.getString()) + 'e';\n" + "var rtrimStr = 'b' + rtrim(string.getString()) + 'e';\n" + "var trimStr  = 'b' + trim(string.getString()) + 'e';\n") };
    svm.setJSScripts(js);
    svm.setFieldname(new String[] { "original", "ltrimStr", "rtrimStr", "trimStr" });
    svm.setRename(new String[] { "", "", "", "" });
    svm.setType(new int[] { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING });
    svm.setLength(new int[] { -1, -1, -1, -1 });
    svm.setPrecision(new int[] { -1, -1, -1, -1 });
    svm.setReplace(new boolean[] { false, false, false, false });
    svm.setCompatible(true);
    String javaScriptStepPid = registry.getPluginId(StepPluginType.class, svm);
    StepMeta javaScriptStep = new StepMeta(javaScriptStepPid, javaScriptStepname, svm);
    transMeta.addStep(javaScriptStep);
    TransHopMeta hi1 = new TransHopMeta(injectorStep, javaScriptStep);
    transMeta.addTransHop(hi1);
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    TransHopMeta hi2 = new TransHopMeta(javaScriptStep, dummyStep);
    transMeta.addTransHop(hi2);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si;
    si = trans.getStepInterface(javaScriptStepname, 0);
    RowStepCollector javaScriptRc = new RowStepCollector();
    si.addRowListener(javaScriptRc);
    si = trans.getStepInterface(dummyStepname, 0);
    RowStepCollector dummyRc = new RowStepCollector();
    si.addRowListener(dummyRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData1();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> goldenImageRows = createResultData1();
    List<RowMetaAndData> resultRows1 = javaScriptRc.getRowsWritten();
    checkRows(resultRows1, goldenImageRows);
    List<RowMetaAndData> resultRows2 = dummyRc.getRowsRead();
    checkRows(resultRows2, goldenImageRows);
}