use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.
the class GetXMLDataTest method testInit.
@Test
public void testInit() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("getxmldata1");
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// Create a Get XML Data step
//
String getXMLDataName = "get xml data step";
GetXMLDataMeta gxdm = new GetXMLDataMeta();
String getXMLDataPid = registry.getPluginId(StepPluginType.class, gxdm);
StepMeta getXMLDataStep = new StepMeta(getXMLDataPid, getXMLDataName, gxdm);
transMeta.addStep(getXMLDataStep);
GetXMLDataField[] fields = new GetXMLDataField[5];
for (int idx = 0; idx < fields.length; idx++) {
fields[idx] = new GetXMLDataField();
}
fields[0].setName("objectid");
fields[0].setXPath("${xml_path}");
fields[0].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
fields[0].setType(ValueMetaInterface.TYPE_STRING);
fields[0].setFormat("");
fields[0].setLength(-1);
fields[0].setPrecision(-1);
fields[0].setCurrencySymbol("");
fields[0].setDecimalSymbol("");
fields[0].setGroupSymbol("");
fields[0].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
gxdm.setEncoding("UTF-8");
gxdm.setIsAFile(false);
gxdm.setInFields(true);
gxdm.setLoopXPath("Level1/Level2/Props");
gxdm.setXMLField("field1");
gxdm.setInputFields(fields);
TransHopMeta hi = new TransHopMeta(injectorStep, getXMLDataStep);
transMeta.addTransHop(hi);
//
// Create a dummy step 1
//
String dummyStepname1 = "dummy step 1";
DummyTransMeta dm1 = new DummyTransMeta();
String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
transMeta.addStep(dummyStep1);
TransHopMeta hi1 = new TransHopMeta(getXMLDataStep, dummyStep1);
transMeta.addTransHop(hi1);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(dummyStepname1, 0);
RowStepCollector dummyRc1 = new RowStepCollector();
si.addRowListener(dummyRc1);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createData();
Iterator<RowMetaAndData> it = inputList.iterator();
while (it.hasNext()) {
RowMetaAndData rm = it.next();
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
GetXMLDataData getXMLDataData = new GetXMLDataData();
GetXMLData getXmlData = new GetXMLData(dummyStep1, getXMLDataData, 0, transMeta, trans);
getXmlData.setVariable("xml_path", "data/owner");
getXmlData.init(gxdm, getXMLDataData);
assertEquals("${xml_path}", gxdm.getInputFields()[0].getXPath());
assertEquals("data/owner", gxdm.getInputFields()[0].getResolvedXPath());
}
use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.
the class JsonOutputTest method test.
public String test(boolean compatibilityMode) throws Exception {
KettleEnvironment.init();
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("testJsonOutput");
PluginRegistry registry = PluginRegistry.getInstance();
// create an injector step
String injectorStepName = "injector step";
StepMeta injectorStep = TestUtilities.createInjectorStep(injectorStepName, registry);
transMeta.addStep(injectorStep);
// create a row generator step
StepMeta rowGeneratorStep = createRowGeneratorStep("Create rows for testJsonOutput1", registry);
transMeta.addStep(rowGeneratorStep);
// create a TransHopMeta for injector and add it to the transMeta
TransHopMeta hop_injectory_rowGenerator = new TransHopMeta(injectorStep, rowGeneratorStep);
transMeta.addTransHop(hop_injectory_rowGenerator);
// create the json output step
// but first lets get a filename
String jsonFileName = TestUtilities.createEmptyTempFile("testJsonOutput1_");
StepMeta jsonOutputStep = createJsonOutputStep("json output step", jsonFileName, registry);
((JsonOutputMeta) jsonOutputStep.getStepMetaInterface()).setCompatibilityMode(compatibilityMode);
transMeta.addStep(jsonOutputStep);
// create a TransHopMeta for jsonOutputStep and add it to the transMeta
TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta(rowGeneratorStep, jsonOutputStep);
transMeta.addTransHop(hop_RowGenerator_outputTextFile);
// Create a dummy step and add it to the tranMeta
String dummyStepName = "dummy step";
StepMeta dummyStep = createDummyStep(dummyStepName, registry);
transMeta.addStep(dummyStep);
// create a TransHopMeta for the
TransHopMeta hop_outputJson_dummyStep = new TransHopMeta(jsonOutputStep, dummyStep);
transMeta.addTransHop(hop_outputJson_dummyStep);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
// Create a row collector and add it to the dummy step interface
StepInterface dummyStepInterface = trans.getStepInterface(dummyStepName, 0);
RowStepCollector dummyRowCollector = new RowStepCollector();
dummyStepInterface.addRowListener(dummyRowCollector);
// RowProducer rowProducer = trans.addRowProducer(injectorStepName, 0);
trans.startThreads();
trans.waitUntilFinished();
// get the results and return it
File outputFile = new File(jsonFileName + ".js");
String jsonStructure = FileUtils.readFileToString(outputFile);
return jsonStructure;
}
use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.
the class GetXMLDataTest method testGetXMLDataSimple1.
/**
* Test case for Get XML Data step, very simple example.
*
* @throws Exception
* Upon any exception
*/
public void testGetXMLDataSimple1() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("getxmldata1");
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// Create a Get XML Data step
//
String getXMLDataName = "get xml data step";
GetXMLDataMeta gxdm = new GetXMLDataMeta();
String getXMLDataPid = registry.getPluginId(StepPluginType.class, gxdm);
StepMeta getXMLDataStep = new StepMeta(getXMLDataPid, getXMLDataName, gxdm);
transMeta.addStep(getXMLDataStep);
GetXMLDataField[] fields = new GetXMLDataField[5];
for (int idx = 0; idx < fields.length; idx++) {
fields[idx] = new GetXMLDataField();
}
fields[0].setName("objectid");
fields[0].setXPath("ObjectID");
fields[0].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
fields[0].setType(ValueMetaInterface.TYPE_STRING);
fields[0].setFormat("");
fields[0].setLength(-1);
fields[0].setPrecision(-1);
fields[0].setCurrencySymbol("");
fields[0].setDecimalSymbol("");
fields[0].setGroupSymbol("");
fields[0].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
fields[1].setName("sapident");
fields[1].setXPath("SAPIDENT");
fields[1].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
fields[1].setType(ValueMetaInterface.TYPE_STRING);
fields[1].setFormat("");
fields[1].setLength(-1);
fields[1].setPrecision(-1);
fields[1].setCurrencySymbol("");
fields[1].setDecimalSymbol("");
fields[1].setGroupSymbol("");
fields[1].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
fields[2].setName("quantity");
fields[2].setXPath("Quantity");
fields[2].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
fields[2].setType(ValueMetaInterface.TYPE_STRING);
fields[2].setFormat("");
fields[2].setLength(-1);
fields[2].setPrecision(-1);
fields[2].setCurrencySymbol("");
fields[2].setDecimalSymbol("");
fields[2].setGroupSymbol("");
fields[2].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
fields[3].setName("merkmalname");
fields[3].setXPath("Merkmalname");
fields[3].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
fields[3].setType(ValueMetaInterface.TYPE_STRING);
fields[3].setFormat("");
fields[3].setLength(-1);
fields[3].setPrecision(-1);
fields[3].setCurrencySymbol("");
fields[3].setDecimalSymbol("");
fields[3].setGroupSymbol("");
fields[3].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
fields[4].setName("merkmalswert");
fields[4].setXPath("Merkmalswert");
fields[4].setElementType(GetXMLDataField.ELEMENT_TYPE_NODE);
fields[4].setType(ValueMetaInterface.TYPE_STRING);
fields[4].setFormat("");
fields[4].setLength(-1);
fields[4].setPrecision(-1);
fields[4].setCurrencySymbol("");
fields[4].setDecimalSymbol("");
fields[4].setGroupSymbol("");
fields[4].setTrimType(GetXMLDataField.TYPE_TRIM_NONE);
gxdm.setEncoding("UTF-8");
gxdm.setIsAFile(false);
gxdm.setInFields(true);
gxdm.setLoopXPath("Level1/Level2/Props");
gxdm.setXMLField("field1");
gxdm.setInputFields(fields);
TransHopMeta hi = new TransHopMeta(injectorStep, getXMLDataStep);
transMeta.addTransHop(hi);
//
// Create a dummy step 1
//
String dummyStepname1 = "dummy step 1";
DummyTransMeta dm1 = new DummyTransMeta();
String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
transMeta.addStep(dummyStep1);
TransHopMeta hi1 = new TransHopMeta(getXMLDataStep, dummyStep1);
transMeta.addTransHop(hi1);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(dummyStepname1, 0);
RowStepCollector dummyRc1 = new RowStepCollector();
si.addRowListener(dummyRc1);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createData();
Iterator<RowMetaAndData> it = inputList.iterator();
while (it.hasNext()) {
RowMetaAndData rm = it.next();
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
// Compare the results
List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
List<RowMetaAndData> goldenImageRows = createResultData1();
checkRows(goldenImageRows, resultRows);
}
use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.
the class CsvInputBase method createAndTestTrans.
protected Trans createAndTestTrans(PluginRegistry registry, TransMeta transMeta, StepMeta injectorStep, StepMeta csvInputStep, String fileName, int numRows) throws KettleException {
TransHopMeta hi = new TransHopMeta(injectorStep, csvInputStep);
transMeta.addTransHop(hi);
//
// Create a dummy step 1
//
String dummyStepname1 = "dummy step 1";
DummyTransMeta dm1 = new DummyTransMeta();
String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
transMeta.addStep(dummyStep1);
TransHopMeta hi1 = new TransHopMeta(csvInputStep, dummyStep1);
transMeta.addTransHop(hi1);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(dummyStepname1, 0);
RowStepCollector dummyRc1 = new RowStepCollector();
si.addRowListener(dummyRc1);
RowProducer rp = trans.addRowProducer(injectorStep.getName(), 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createData(fileName);
Iterator<RowMetaAndData> it = inputList.iterator();
while (it.hasNext()) {
RowMetaAndData rm = it.next();
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
// Compare the results
List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
List<RowMetaAndData> goldenImageRows = createResultData1();
checkRows(goldenImageRows, resultRows, numRows);
return trans;
}
use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.
the class DatabaseLookupIT method basicDatabaseLookup.
/**
* Basic Test case for database lookup.
*/
@Test
public void basicDatabaseLookup() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("transname");
// Add the database connections
for (int i = 0; i < databasesXML.length; i++) {
DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
transMeta.addDatabase(databaseMeta);
}
DatabaseMeta dbInfo = transMeta.findDatabase("db");
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// create the lookup step...
//
String lookupName = "look up from [" + lookup_table + "]";
DatabaseLookupMeta dbl = new DatabaseLookupMeta();
dbl.setDatabaseMeta(transMeta.findDatabase("db"));
dbl.setTablename(lookup_table);
dbl.setCached(false);
dbl.setEatingRowOnLookupFailure(false);
dbl.setFailingOnMultipleResults(false);
dbl.setOrderByClause("");
dbl.setTableKeyField(new String[] { "ID" });
dbl.setKeyCondition(new String[] { "=" });
dbl.setStreamKeyField1(new String[] { "int_field" });
dbl.setStreamKeyField2(new String[] { "" });
dbl.setReturnValueField(new String[] { "CODE", "STRING" });
dbl.setReturnValueDefaultType(new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING });
dbl.setReturnValueDefault(new String[] { "-1", "UNDEF" });
dbl.setReturnValueNewName(new String[] { "RET_CODE", "RET_STRING" });
String lookupId = registry.getPluginId(StepPluginType.class, dbl);
StepMeta lookupStep = new StepMeta(lookupId, lookupName, dbl);
lookupStep.setDescription("Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]");
transMeta.addStep(lookupStep);
TransHopMeta hi = new TransHopMeta(injectorStep, lookupStep);
transMeta.addTransHop(hi);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(lookupName, 0);
RowStepCollector rc = new RowStepCollector();
si.addRowListener(rc);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createDataRows();
for (RowMetaAndData rm : inputList) {
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
List<RowMetaAndData> resultRows = rc.getRowsWritten();
List<RowMetaAndData> goldRows = createResultDataRows();
checkRows(goldRows, resultRows);
}
Aggregations