Examples with RowProducer org.pentaho.di.trans.RowProducer used on opensource projects

Example 36 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class SimpleMapping method processRow.

/**
 * Process a single row. In our case, we send one row of data to a piece of transformation. In the transformation, we
 * look up the MappingInput step to send our rows to it. As a consequence, for the time being, there can only be one
 * MappingInput and one MappingOutput step in the Mapping.
 */
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (SimpleMappingMeta) smi;
    setData((SimpleMappingData) sdi);
    SimpleMappingData simpleMappingData = getData();
    try {
        if (first) {
            first = false;
            simpleMappingData.wasStarted = true;
            // Rows read are injected into the one available Mapping Input step
            // 
            String mappingInputStepname = simpleMappingData.mappingInput.getStepname();
            RowProducer rowProducer = simpleMappingData.mappingTrans.addRowProducer(mappingInputStepname, 0);
            simpleMappingData.rowDataInputMapper = new RowDataInputMapper(meta.getInputMapping(), rowProducer);
            // Rows produced by the mapping are read and passed on.
            // 
            String mappingOutputStepname = simpleMappingData.mappingOutput.getStepname();
            StepInterface outputStepInterface = simpleMappingData.mappingTrans.findStepInterface(mappingOutputStepname, 0);
            RowOutputDataMapper outputDataMapper = new RowOutputDataMapper(meta.getInputMapping(), meta.getOutputMapping(), new PutRowInterface() {

                @Override
                public void putRow(RowMetaInterface rowMeta, Object[] rowData) throws KettleStepException {
                    SimpleMapping.this.putRow(rowMeta, rowData);
                }
            });
            outputStepInterface.addRowListener(outputDataMapper);
            // Start the mapping/sub-transformation threads
            // 
            simpleMappingData.mappingTrans.startThreads();
        }
        // The data we read we pass to the mapping
        // 
        Object[] row = getRow();
        boolean rowWasPut = false;
        if (row != null) {
            while (!(data.mappingTrans.isFinishedOrStopped() || rowWasPut)) {
                rowWasPut = data.rowDataInputMapper.putRow(getInputRowMeta(), row);
            }
        }
        if (!rowWasPut) {
            simpleMappingData.rowDataInputMapper.finished();
            simpleMappingData.mappingTrans.waitUntilFinished();
            setOutputDone();
            return false;
        }
        return true;
    } catch (Throwable t) {
        // 
        if (simpleMappingData.mappingTrans != null) {
            simpleMappingData.mappingTrans.stopAll();
        }
        // 
        throw new KettleException(t);
    }
}

Example 37 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class SynchronizeAfterMergeIT method processRow.

private void processRow(TransProcessControl control) throws Exception {
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    generateData(rp);
    rp.finished();
    StepInterface si = trans.getStepInterface(synchronizeAfterMergeStepname, 0);
    switch(control) {
        case ITTERUPT:
            trans.stopAll();
            while (!si.getStatus().equals(StepExecutionStatus.STATUS_STOPPED)) {
            // wait until transformation does not stopped
            }
            ;
            break;
        case WAIT:
        default:
            trans.waitUntilFinished();
            assertEquals("Step still started", StepExecutionStatus.STATUS_FINISHED, si.getStatus());
            break;
    }
    assertEquals("Unexpected error occurred", 0, si.getErrors());
    Field field = SynchronizeAfterMerge.class.getDeclaredField("data");
    field.setAccessible(true);
    SynchronizeAfterMergeData data = (SynchronizeAfterMergeData) field.get(si);
    // should be closed and set null after finish transformation
    assertNull(data.db.getConnection());
}

Example 38 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class GetXMLDataTest method testGetXMLDataSimple1.

/**
 * Test case for Get XML Data step, very simple example.
 *
 * @throws Exception
 *           Upon any exception
 */
public void testGetXMLDataSimple1() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("getxmldata1");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a Get XML Data step
    // 
    String getXMLDataName = "get xml data step";
    GetXMLDataMeta gxdm = new GetXMLDataMeta();
    String getXMLDataPid = registry.getPluginId(StepPluginType.class, gxdm);
    StepMeta getXMLDataStep = new StepMeta(getXMLDataPid, getXMLDataName, gxdm);
    transMeta.addStep(getXMLDataStep);
    GetXMLDataField[] fields = new GetXMLDataField[5];
    for (int idx = 0; idx < fields.length; idx++) {
        fields[idx] = new GetXMLDataField();
    }
    fields[0].setName("objectid");
    fields[0].setXPath("ObjectID");
    fields[0].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
    fields[0].setType(ValueMetaInterface.TYPE_STRING);
    fields[0].setFormat("");
    fields[0].setLength(-1);
    fields[0].setPrecision(-1);
    fields[0].setCurrencySymbol("");
    fields[0].setDecimalSymbol("");
    fields[0].setGroupSymbol("");
    fields[0].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
    fields[1].setName("sapident");
    fields[1].setXPath("SAPIDENT");
    fields[1].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
    fields[1].setType(ValueMetaInterface.TYPE_STRING);
    fields[1].setFormat("");
    fields[1].setLength(-1);
    fields[1].setPrecision(-1);
    fields[1].setCurrencySymbol("");
    fields[1].setDecimalSymbol("");
    fields[1].setGroupSymbol("");
    fields[1].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
    fields[2].setName("quantity");
    fields[2].setXPath("Quantity");
    fields[2].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
    fields[2].setType(ValueMetaInterface.TYPE_STRING);
    fields[2].setFormat("");
    fields[2].setLength(-1);
    fields[2].setPrecision(-1);
    fields[2].setCurrencySymbol("");
    fields[2].setDecimalSymbol("");
    fields[2].setGroupSymbol("");
    fields[2].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
    fields[3].setName("merkmalname");
    fields[3].setXPath("Merkmalname");
    fields[3].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
    fields[3].setType(ValueMetaInterface.TYPE_STRING);
    fields[3].setFormat("");
    fields[3].setLength(-1);
    fields[3].setPrecision(-1);
    fields[3].setCurrencySymbol("");
    fields[3].setDecimalSymbol("");
    fields[3].setGroupSymbol("");
    fields[3].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
    fields[4].setName("merkmalswert");
    fields[4].setXPath("Merkmalswert");
    fields[4].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
    fields[4].setType(ValueMetaInterface.TYPE_STRING);
    fields[4].setFormat("");
    fields[4].setLength(-1);
    fields[4].setPrecision(-1);
    fields[4].setCurrencySymbol("");
    fields[4].setDecimalSymbol("");
    fields[4].setGroupSymbol("");
    fields[4].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
    gxdm.setEncoding("UTF-8");
    gxdm.setIsAFile(false);
    gxdm.setInFields(true);
    gxdm.setLoopXPath("Level1/Level2/Props");
    gxdm.setXMLField("field1");
    gxdm.setInputFields(fields);
    TransHopMeta hi = new TransHopMeta(injectorStep, getXMLDataStep);
    transMeta.addTransHop(hi);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi1 = new TransHopMeta(getXMLDataStep, dummyStep1);
    transMeta.addTransHop(hi1);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData1();
    checkRows(goldenImageRows, resultRows);
}

Example 39 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class MetaInject method processRow.

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (MetaInjectMeta) smi;
    data = (MetaInjectData) sdi;
    // Read the data from all input steps and keep it in memory...
    // Skip the step from which we stream data. Keep that available for runtime action.
    // 
    data.rowMap = new HashMap<String, List<RowMetaAndData>>();
    for (String prevStepName : getTransMeta().getPrevStepNames(getStepMeta())) {
        // 
        if (!data.streaming || !prevStepName.equalsIgnoreCase(data.streamingSourceStepname)) {
            List<RowMetaAndData> list = new ArrayList<RowMetaAndData>();
            RowSet rowSet = findInputRowSet(prevStepName);
            Object[] row = getRowFrom(rowSet);
            while (row != null) {
                RowMetaAndData rd = new RowMetaAndData();
                rd.setRowMeta(rowSet.getRowMeta());
                rd.setData(row);
                list.add(rd);
                row = getRowFrom(rowSet);
            }
            if (!list.isEmpty()) {
                data.rowMap.put(prevStepName, list);
            }
        }
    }
    List<StepMeta> steps = data.transMeta.getSteps();
    for (Map.Entry<String, StepMetaInterface> en : data.stepInjectionMetasMap.entrySet()) {
        newInjection(en.getKey(), en.getValue());
    }
    /*
     * constants injection should be executed after steps, because if constant should be inserted into target with array
     * in path, constants should be inserted into all arrays items
     */
    for (Map.Entry<String, StepMetaInterface> en : data.stepInjectionMetasMap.entrySet()) {
        newInjectionConstants(en.getKey(), en.getValue());
    }
    for (Map.Entry<String, StepMetaInterface> en : data.stepInjectionMetasMap.entrySet()) {
        en.getValue().searchInfoAndTargetSteps(steps);
    }
    for (String targetStepName : data.stepInjectionMap.keySet()) {
        if (!data.stepInjectionMetasMap.containsKey(targetStepName)) {
            oldInjection(targetStepName);
            StepMeta targetStep = StepMeta.findStep(steps, targetStepName);
            if (targetStep != null) {
                targetStep.getStepMetaInterface().searchInfoAndTargetSteps(steps);
            }
        }
    }
    if (!meta.isNoExecution()) {
        // Now we can execute this modified transformation metadata.
        // 
        final Trans injectTrans = createInjectTrans();
        injectTrans.setMetaStore(getMetaStore());
        if (getTrans().getParentJob() != null) {
            // See PDI-13224
            injectTrans.setParentJob(getTrans().getParentJob());
        }
        getTrans().addTransStoppedListener(new TransStoppedListener() {

            public void transStopped(Trans parentTrans) {
                injectTrans.stopAll();
            }
        });
        injectTrans.prepareExecution(null);
        // See if we need to stream some data over...
        // 
        RowProducer rowProducer = null;
        if (data.streaming) {
            rowProducer = injectTrans.addRowProducer(data.streamingTargetStepname, 0);
        }
        // Finally, add the mapping transformation to the active sub-transformations
        // map in the parent transformation
        // 
        getTrans().addActiveSubTransformation(getStepname(), injectTrans);
        if (!Utils.isEmpty(meta.getSourceStepName())) {
            StepInterface stepInterface = injectTrans.getStepInterface(meta.getSourceStepName(), 0);
            if (stepInterface == null) {
                throw new KettleException("Unable to find step '" + meta.getSourceStepName() + "' to read from.");
            }
            stepInterface.addRowListener(new RowAdapter() {

                @Override
                public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException {
                    // Just pass along the data as output of this step...
                    // 
                    MetaInject.this.putRow(rowMeta, row);
                }
            });
        }
        injectTrans.startThreads();
        if (data.streaming) {
            // Deplete all the rows from the parent transformation into the modified transformation
            // 
            RowSet rowSet = findInputRowSet(data.streamingSourceStepname);
            if (rowSet == null) {
                throw new KettleException("Unable to find step '" + data.streamingSourceStepname + "' to stream data from");
            }
            Object[] row = getRowFrom(rowSet);
            while (row != null && !isStopped()) {
                rowProducer.putRow(rowSet.getRowMeta(), row);
                row = getRowFrom(rowSet);
            }
            rowProducer.finished();
        }
        // 
        while (!injectTrans.isFinished() && !injectTrans.isStopped() && !isStopped()) {
            copyResult(injectTrans);
            // Wait a little bit.
            try {
                Thread.sleep(50);
            } catch (Exception e) {
            // Ignore errors
            }
        }
        copyResult(injectTrans);
        waitUntilFinished(injectTrans);
    }
    // let the transformation complete it's execution to allow for any customizations to MDI to happen in the init methods of steps
    if (log.isDetailed()) {
        logDetailed("XML of transformation after injection: " + data.transMeta.getXML());
    }
    String targetFile = environmentSubstitute(meta.getTargetFile());
    if (!Utils.isEmpty(targetFile)) {
        writeInjectedKtr(targetFile);
    }
    // All done!
    setOutputDone();
    return false;
}

Example 40 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class SimpleMappingTest method setup.

@Before
public void setup() throws Exception {
    stepMockHelper = new StepMockHelper<SimpleMappingMeta, SimpleMappingData>("SIMPLE_MAPPING_TEST", SimpleMappingMeta.class, SimpleMappingData.class);
    when(stepMockHelper.logChannelInterfaceFactory.create(any(), any(LoggingObjectInterface.class))).thenReturn(stepMockHelper.logChannelInterface);
    when(stepMockHelper.trans.isRunning()).thenReturn(true);
    // Mock for MappingInput
    MappingInput mpInputMock = mock(MappingInput.class);
    when(mpInputMock.getStepname()).thenReturn(MAPPING_INPUT_STEP_NAME);
    // Mock for MappingOutput
    MappingOutput mpOutputMock = mock(MappingOutput.class);
    when(mpOutputMock.getStepname()).thenReturn(MAPPING_OUTPUT_STEP_NAME);
    // Mock for RowDataInputMapper
    RowDataInputMapper rdInputMpMock = mock(RowDataInputMapper.class);
    RowMetaInterface rwMetaInMock = mock(RowMeta.class);
    doReturn(Boolean.TRUE).when(rdInputMpMock).putRow(rwMetaInMock, new Object[] {});
    // Mock for RowProducer
    RowProducer rProducerMock = mock(RowProducer.class);
    when(rProducerMock.putRow(any(RowMetaInterface.class), any(Object[].class), anyBoolean())).thenReturn(true);
    // Mock for MappingIODefinition
    MappingIODefinition mpIODefMock = mock(MappingIODefinition.class);
    // Set up real SimpleMappingData with some mocked elements
    simpleMpData.mappingInput = mpInputMock;
    simpleMpData.mappingOutput = mpOutputMock;
    simpleMpData.rowDataInputMapper = rdInputMpMock;
    simpleMpData.mappingTrans = stepMockHelper.trans;
    when(stepMockHelper.trans.findStepInterface(MAPPING_OUTPUT_STEP_NAME, 0)).thenReturn(mpOutputMock);
    when(stepMockHelper.trans.addRowProducer(MAPPING_INPUT_STEP_NAME, 0)).thenReturn(rProducerMock);
    when(stepMockHelper.processRowsStepMetaInterface.getInputMapping()).thenReturn(mpIODefMock);
}