use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.
the class GetXMLDataTest method testInit.
public void testInit() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("getxmldata1");
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// Create a Get XML Data step
//
String getXMLDataName = "get xml data step";
GetXMLDataMeta gxdm = new GetXMLDataMeta();
String getXMLDataPid = registry.getPluginId(StepPluginType.class, gxdm);
StepMeta getXMLDataStep = new StepMeta(getXMLDataPid, getXMLDataName, gxdm);
transMeta.addStep(getXMLDataStep);
GetXMLDataField[] fields = new GetXMLDataField[5];
for (int idx = 0; idx < fields.length; idx++) {
fields[idx] = new GetXMLDataField();
}
fields[0].setName("objectid");
fields[0].setXPath("${xml_path}");
fields[0].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
fields[0].setType(ValueMetaInterface.TYPE_STRING);
fields[0].setFormat("");
fields[0].setLength(-1);
fields[0].setPrecision(-1);
fields[0].setCurrencySymbol("");
fields[0].setDecimalSymbol("");
fields[0].setGroupSymbol("");
fields[0].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
gxdm.setEncoding("UTF-8");
gxdm.setIsAFile(false);
gxdm.setInFields(true);
gxdm.setLoopXPath("Level1/Level2/Props");
gxdm.setXMLField("field1");
gxdm.setInputFields(fields);
TransHopMeta hi = new TransHopMeta(injectorStep, getXMLDataStep);
transMeta.addTransHop(hi);
//
// Create a dummy step 1
//
String dummyStepname1 = "dummy step 1";
DummyTransMeta dm1 = new DummyTransMeta();
String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
transMeta.addStep(dummyStep1);
TransHopMeta hi1 = new TransHopMeta(getXMLDataStep, dummyStep1);
transMeta.addTransHop(hi1);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(dummyStepname1, 0);
RowStepCollector dummyRc1 = new RowStepCollector();
si.addRowListener(dummyRc1);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createData();
Iterator<RowMetaAndData> it = inputList.iterator();
while (it.hasNext()) {
RowMetaAndData rm = it.next();
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
// Compare the results
List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
List<RowMetaAndData> goldenImageRows = createResultData1();
GetXMLDataData getXMLDataData = new GetXMLDataData();
GetXMLData getXmlData = new GetXMLData(dummyStep1, getXMLDataData, 0, transMeta, trans);
getXmlData.setVariable("xml_path", "data/owner");
getXmlData.init(gxdm, getXMLDataData);
assertEquals(gxdm.getInputFields()[0].getXPath(), "${xml_path}");
}
use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.
the class XsltTest method runTestWithParams.
public void runTestWithParams(String xmlFieldname, String resultFieldname, boolean xslInField, boolean xslFileInField, String xslFileField, String xslFilename, String xslFactory) throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("xslt");
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// Create a XSLT step
//
String xsltName = "xslt step";
XsltMeta xm = new XsltMeta();
String xsltPid = registry.getPluginId(StepPluginType.class, xm);
StepMeta xsltStep = new StepMeta(xsltPid, xsltName, xm);
transMeta.addStep(xsltStep);
TextFileInputField[] fields = new TextFileInputField[3];
for (int idx = 0; idx < fields.length; idx++) {
fields[idx] = new TextFileInputField();
}
fields[0].setName("XML");
fields[0].setType(ValueMetaInterface.TYPE_STRING);
fields[0].setFormat("");
fields[0].setLength(-1);
fields[0].setPrecision(-1);
fields[0].setCurrencySymbol("");
fields[0].setDecimalSymbol("");
fields[0].setGroupSymbol("");
fields[0].setTrimType(ValueMetaInterface.TRIM_TYPE_NONE);
fields[1].setName("XSL");
fields[1].setType(ValueMetaInterface.TYPE_STRING);
fields[1].setFormat("");
fields[1].setLength(-1);
fields[1].setPrecision(-1);
fields[1].setCurrencySymbol("");
fields[1].setDecimalSymbol("");
fields[1].setGroupSymbol("");
fields[1].setTrimType(ValueMetaInterface.TRIM_TYPE_NONE);
fields[2].setName("filename");
fields[2].setType(ValueMetaInterface.TYPE_STRING);
fields[2].setFormat("");
fields[2].setLength(-1);
fields[2].setPrecision(-1);
fields[2].setCurrencySymbol("");
fields[2].setDecimalSymbol("");
fields[2].setGroupSymbol("");
fields[2].setTrimType(ValueMetaInterface.TRIM_TYPE_NONE);
xm.setFieldname(xmlFieldname);
xm.setResultfieldname(resultFieldname);
xm.setXSLField(xslInField);
xm.setXSLFileField(xslFileField);
xm.setXSLFieldIsAFile(xslFileInField);
xm.setXslFilename(xslFilename);
xm.setXSLFactory(xslFactory);
TransHopMeta hi = new TransHopMeta(injectorStep, xsltStep);
transMeta.addTransHop(hi);
//
// Create a dummy step 1
//
String dummyStepname1 = "dummy step 1";
DummyTransMeta dm1 = new DummyTransMeta();
String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
transMeta.addStep(dummyStep1);
TransHopMeta hi1 = new TransHopMeta(xsltStep, dummyStep1);
transMeta.addTransHop(hi1);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(dummyStepname1, 0);
RowStepCollector dummyRc1 = new RowStepCollector();
si.addRowListener(dummyRc1);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createData(xslFilename);
Iterator<RowMetaAndData> it = inputList.iterator();
while (it.hasNext()) {
RowMetaAndData rm = it.next();
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
// Compare the results
List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
List<RowMetaAndData> goldenImageRows = createResultData1();
checkRows(goldenImageRows, resultRows, 2);
}
use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.
the class CheckSumTest method executeHexTest.
/**
* Create, execute, and return the row listener attached to the output step with complete results from the execution.
*
* @param checkSumType
* Type of checksum to use (the array index of {@link CheckSumMeta#checksumtypeCodes})
* @param compatibilityMode
* Use compatibility mode for CheckSum
* @param input
* String to calculate checksum for
* @param meta
* meta to be used
* @return RowListener with results.
*/
private MockRowListener executeHexTest(int checkSumType, boolean compatibilityMode, Object input, ValueMetaInterface meta, boolean oldChecksumBehaviour) throws Exception {
Trans trans = buildHexadecimalChecksumTrans(checkSumType, compatibilityMode, oldChecksumBehaviour);
trans.prepareExecution(null);
StepInterface output = trans.getRunThread("Output", 0);
MockRowListener listener = new MockRowListener();
output.addRowListener(listener);
RowProducer rp = trans.addRowProducer("CheckSum", 0);
RowMeta inputRowMeta = createStringRowMeta(meta);
((BaseStep) trans.getRunThread("CheckSum", 0)).setInputRowMeta(inputRowMeta);
trans.startThreads();
rp.putRow(inputRowMeta, new Object[] { input });
rp.finished();
trans.waitUntilFinished();
trans.stopAll();
trans.cleanup();
return listener;
}
use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.
the class TableInputIT method testTableInputWithParam.
/**
* Test case for table input which is taking its input from a hop. This is a regression test case for JIRA PDI-588.
*
* The query in the table input step has one '?' and this parameter is filled by values read from an input hop.
*/
public void testTableInputWithParam() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("transname");
// Add the database connections
for (int i = 0; i < databasesXML.length; i++) {
DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
transMeta.addDatabase(databaseMeta);
}
DatabaseMeta dbInfo = transMeta.findDatabase("db");
// Execute our setup SQLs in the database.
Database database = new Database(transMeta, dbInfo);
database.connect();
createTables(database);
createData(database);
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// create the source step...
//
String fromstepname = "read from [" + source_table + "]";
TableInputMeta tii = new TableInputMeta();
tii.setDatabaseMeta(transMeta.findDatabase("db"));
tii.setLookupFromStep(injectorStep);
tii.setExecuteEachInputRow(true);
String selectSQL = "SELECT " + Const.CR;
selectSQL += "ID, CODE ";
selectSQL += "FROM " + source_table + " WHERE CODE = ? ORDER BY ID, CODE;";
tii.setSQL(selectSQL);
String fromstepid = registry.getPluginId(StepPluginType.class, tii);
StepMeta fromstep = new StepMeta(fromstepid, fromstepname, tii);
fromstep.setDescription("Reads information from table [" + source_table + "] on database [" + dbInfo + "]");
transMeta.addStep(fromstep);
TransHopMeta hi = new TransHopMeta(injectorStep, fromstep);
transMeta.addTransHop(hi);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(fromstepname, 0);
RowStepCollector rc = new RowStepCollector();
si.addRowListener(rc);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createDataRows();
for (RowMetaAndData rm : inputList) {
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
List<RowMetaAndData> resultRows = rc.getRowsWritten();
List<RowMetaAndData> goldRows = createResultDataRows();
checkRows(goldRows, resultRows);
}
use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.
the class TableOutputIT method testTableOutputNormal.
/**
* Test case for normal table output case.
*/
@SuppressWarnings("deprecation")
public void testTableOutputNormal() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("table output normal test");
// Add the database connections
for (int i = 0; i < databasesXML.length; i++) {
DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
transMeta.addDatabase(databaseMeta);
}
DatabaseMeta dbInfo = transMeta.findDatabase("db");
// Execute our setup SQLs in the database.
Database database = new Database(transMeta, dbInfo);
database.connect();
createTable(database, target_table, createSourceRowMetaInterface1());
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// create the source step...
//
String outputname = "output to [" + target_table + "]";
TableOutputMeta tom = new TableOutputMeta();
tom.setDatabaseMeta(transMeta.findDatabase("db"));
tom.setTablename(target_table);
String fromid = registry.getPluginId(StepPluginType.class, tom);
StepMeta fromstep = new StepMeta(fromid, outputname, tom);
fromstep.setDescription("write data to table [" + target_table + "] on database [" + dbInfo + "]");
transMeta.addStep(fromstep);
TransHopMeta hi = new TransHopMeta(injectorStep, fromstep);
transMeta.addTransHop(hi);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(outputname, 0);
RowStepCollector rc = new RowStepCollector();
si.addRowListener(rc);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createNormalDataRows();
for (RowMetaAndData rm : inputList) {
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
List<RowMetaAndData> resultRows = rc.getRowsWritten();
List<RowMetaAndData> goldRows = createNormalDataRows();
checkRows(goldRows, resultRows);
checkResultsNormal(database);
}
Aggregations