use of org.pentaho.di.core.plugins.PluginRegistry in project pentaho-kettle by pentaho.
the class TableOutputIT method testTableOutputJIRA897.
/**
* Test case for normal table output where the table is included in the instream, but the tablename is not stored in
* the table.
*/
public void testTableOutputJIRA897() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("table output JIRA897 test");
// Add the database connections
for (int i = 0; i < databasesXML.length; i++) {
DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
transMeta.addDatabase(databaseMeta);
}
DatabaseMeta dbInfo = transMeta.findDatabase("db");
// Execute our setup SQLs in the database.
Database database = new Database(transMeta, dbInfo);
database.connect();
createTable(database, target_table1, createSourceRowMetaInterface1());
createTable(database, target_table2, createSourceRowMetaInterface1());
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// create the source step...
//
String outputname = "output to [" + target_table1 + "] and [" + target_table2 + "]";
TableOutputMeta tom = new TableOutputMeta();
tom.setDatabaseMeta(transMeta.findDatabase("db"));
tom.setTableNameInField(true);
tom.setTableNameField("TABLE");
tom.setTableNameInTable(false);
String fromid = registry.getPluginId(StepPluginType.class, tom);
StepMeta fromstep = new StepMeta(fromid, outputname, tom);
fromstep.setDescription("write data to tables on database [" + dbInfo + "]");
transMeta.addStep(fromstep);
TransHopMeta hi = new TransHopMeta(injectorStep, fromstep);
transMeta.addTransHop(hi);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(outputname, 0);
RowStepCollector rc = new RowStepCollector();
si.addRowListener(rc);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createJIRA897DataRows();
for (RowMetaAndData rm : inputList) {
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
List<RowMetaAndData> resultRows = rc.getRowsWritten();
// The name of the table should still be in here.
List<RowMetaAndData> goldRows = createJIRA897DataRows();
checkRows(goldRows, resultRows);
checkResultsJIRA897(database);
}
use of org.pentaho.di.core.plugins.PluginRegistry in project pentaho-kettle by pentaho.
the class TableOutputIT method testTableOutputNormal.
/**
* Test case for normal table output case.
*/
@SuppressWarnings("deprecation")
public void testTableOutputNormal() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("table output normal test");
// Add the database connections
for (int i = 0; i < databasesXML.length; i++) {
DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
transMeta.addDatabase(databaseMeta);
}
DatabaseMeta dbInfo = transMeta.findDatabase("db");
// Execute our setup SQLs in the database.
Database database = new Database(transMeta, dbInfo);
database.connect();
createTable(database, target_table, createSourceRowMetaInterface1());
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// create the source step...
//
String outputname = "output to [" + target_table + "]";
TableOutputMeta tom = new TableOutputMeta();
tom.setDatabaseMeta(transMeta.findDatabase("db"));
tom.setTablename(target_table);
String fromid = registry.getPluginId(StepPluginType.class, tom);
StepMeta fromstep = new StepMeta(fromid, outputname, tom);
fromstep.setDescription("write data to table [" + target_table + "] on database [" + dbInfo + "]");
transMeta.addStep(fromstep);
TransHopMeta hi = new TransHopMeta(injectorStep, fromstep);
transMeta.addTransHop(hi);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(outputname, 0);
RowStepCollector rc = new RowStepCollector();
si.addRowListener(rc);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createNormalDataRows();
for (RowMetaAndData rm : inputList) {
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
List<RowMetaAndData> resultRows = rc.getRowsWritten();
List<RowMetaAndData> goldRows = createNormalDataRows();
checkRows(goldRows, resultRows);
checkResultsNormal(database);
}
use of org.pentaho.di.core.plugins.PluginRegistry in project pentaho-kettle by pentaho.
the class TableOutputIT method testTableOutputJIRA2733.
/**
* Test case for commitSize see PDI2733 in JIRA.
*/
@SuppressWarnings("deprecation")
public void testTableOutputJIRA2733() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("table output JIRA2733 test");
// Add the database connections
for (int i = 0; i < databasesXML.length; i++) {
DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
transMeta.addDatabase(databaseMeta);
}
DatabaseMeta dbInfo = transMeta.findDatabase("db");
// Execute our setup SQLs in the database.
Database database = new Database(transMeta, dbInfo);
database.connect();
createTable(database, target_table3, createSourceRowMetaInterface1());
// Add "ts" timestamp field to target_table with a default value of NOW()
database.execStatement("ALTER TABLE " + target_table3 + " ADD COLUMN ts TIMESTAMP DEFAULT NOW() ");
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// create the source step...
//
String outputname = "output to [" + target_table3 + "]";
TableOutputMeta tom = new TableOutputMeta();
tom.setDatabaseMeta(transMeta.findDatabase("db"));
tom.setTablename(target_table3);
tom.setTruncateTable(true);
tom.setUseBatchUpdate(true);
String fromid = registry.getPluginId(StepPluginType.class, tom);
StepMeta fromstep = new StepMeta(fromid, outputname, tom);
fromstep.setDescription("write data to table [" + target_table3 + "] on database [" + dbInfo + "]");
transMeta.addStep(fromstep);
TransHopMeta hi = new TransHopMeta(injectorStep, fromstep);
transMeta.addTransHop(hi);
// With seven rows these are the number of commits that need to made
// for "commitSize"s ranging between 0 and 8. (0=auto-commit=no commits)
int[] goldRowCounts = { 1, 8, 4, 3, 2, 2, 2, 2, 1 };
for (int commitSize = 0; commitSize <= 8; commitSize++) {
tom.setCommitSize(commitSize);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(outputname, 0);
RowStepCollector rc = new RowStepCollector();
si.addRowListener(rc);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createNormalDataRows();
for (RowMetaAndData rm : inputList) {
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
// Get the number of commits from the DB connection
// in the table output step...
//
TableOutputData data = (TableOutputData) trans.findDataInterface(outputname);
int exp = goldRowCounts[commitSize];
// remove 1 commit too many in the dispose method.
//
int act = data.db.getNrExecutedCommits() - 1;
assertEquals("Incorrect number of commits with commitSize=" + commitSize + Const.CR, exp, act);
}
dropTable(database, target_table3);
}
use of org.pentaho.di.core.plugins.PluginRegistry in project pentaho-kettle by pentaho.
the class TextFileOutputIT method testTextFileOutput1.
/**
* Tests output rows
*/
@Test
public void testTextFileOutput1() throws Exception {
KettleEnvironment.init();
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("testTextFileOutput1");
PluginRegistry registry = PluginRegistry.getInstance();
// create an injector step
String injectorStepName = "injector step";
StepMeta injectorStep = TestUtilities.createInjectorStep(injectorStepName, registry);
transMeta.addStep(injectorStep);
// create a row generator step
StepMeta rowGeneratorStep = createRowGeneratorStep("Create rows for testTextFileOutput1", registry);
transMeta.addStep(rowGeneratorStep);
// create a TransHopMeta for injector and add it to the transMeta
TransHopMeta hop_injectory_rowGenerator = new TransHopMeta(injectorStep, rowGeneratorStep);
transMeta.addTransHop(hop_injectory_rowGenerator);
// create the text file output step
// but first lets get a filename
String textFileName = TestUtilities.createEmptyTempFile("testTextFileOutput1");
StepMeta textFileOutputStep = createTextFileOutputStep("text file output step", textFileName, "None", registry);
transMeta.addStep(textFileOutputStep);
// create a TransHopMeta for textFileOutputStep and add it to the transMeta
TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta(rowGeneratorStep, textFileOutputStep);
transMeta.addTransHop(hop_RowGenerator_outputTextFile);
// Create a dummy step and add it to the tranMeta
String dummyStepName = "dummy step";
StepMeta dummyStep = createDummyStep(dummyStepName, registry);
transMeta.addStep(dummyStep);
// create a TransHopMeta for the
TransHopMeta hop_outputTextFile_dummyStep = new TransHopMeta(textFileOutputStep, dummyStep);
transMeta.addTransHop(hop_outputTextFile_dummyStep);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
// Create a row collector and add it to the dummy step interface
StepInterface dummyStepInterface = trans.getStepInterface(dummyStepName, 0);
RowStepCollector dummyRowCollector = new RowStepCollector();
dummyStepInterface.addRowListener(dummyRowCollector);
// RowProducer rowProducer = trans.addRowProducer(injectorStepName, 0);
trans.startThreads();
trans.waitUntilFinished();
// Compare the results
List<RowMetaAndData> resultRows = dummyRowCollector.getRowsWritten();
List<RowMetaAndData> goldenImageRows = createResultData1();
try {
TestUtilities.checkRows(goldenImageRows, resultRows);
} catch (TestFailedException tfe) {
fail(tfe.getMessage());
}
}
use of org.pentaho.di.core.plugins.PluginRegistry in project pentaho-kettle by pentaho.
the class TextFileOutputIT method testTextFileOutput4.
/**
* Tests the ZIP output capability of the TextFileOutput step
*
* @throws Exception
*/
@Test
public void testTextFileOutput4() throws Exception {
KettleEnvironment.init();
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("testTextFileOutput4");
PluginRegistry registry = PluginRegistry.getInstance();
// create an injector step
String injectorStepName = "injector step";
StepMeta injectorStep = TestUtilities.createInjectorStep(injectorStepName, registry);
transMeta.addStep(injectorStep);
// create a row generator step
StepMeta rowGeneratorStep = createRowGeneratorStep("Create rows for testTextFileOutput4", registry);
transMeta.addStep(rowGeneratorStep);
// create a TransHopMeta for injector and add it to the transMeta
TransHopMeta hop_injectory_rowGenerator = new TransHopMeta(injectorStep, rowGeneratorStep);
transMeta.addTransHop(hop_injectory_rowGenerator);
// create the text file output step with ZIP compression
// but first lets get a filename
String textFileName = "testTextFileOutput4";
String textFileOutputStepName = "text file output step";
StepMeta textFileOutputStep = createTextFileOutputStep(textFileOutputStepName, textFileName, "Zip", registry);
transMeta.addStep(textFileOutputStep);
// create a TransHopMeta for textFileOutputStep and add it to the transMeta
TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta(rowGeneratorStep, textFileOutputStep);
transMeta.addTransHop(hop_RowGenerator_outputTextFile);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
// Create a row collector and add it to the dummy step interface
StepInterface dummyStepInterface = trans.getStepInterface(textFileOutputStepName, 0);
RowStepCollector dummyRowCollector = new RowStepCollector();
dummyStepInterface.addRowListener(dummyRowCollector);
trans.startThreads();
trans.waitUntilFinished();
// Compare the results
List<RowMetaAndData> resultRows = dummyRowCollector.getRowsWritten();
Object[][] rows = new Object[10][3];
File f = new File(textFileName + ".zip");
f.deleteOnExit();
try {
ZipFile zf = new ZipFile(f);
int zipEntryCount = 0;
Enumeration<? extends ZipEntry> entries = zf.entries();
while (entries.hasMoreElements()) {
ZipEntry ze = entries.nextElement();
zipEntryCount++;
BufferedReader input = new BufferedReader(new InputStreamReader(zf.getInputStream(ze)));
readData1Rows(rows, input);
}
zf.close();
assertEquals(zipEntryCount, 1);
} catch (IOException e) {
fail(e.getLocalizedMessage());
}
List<RowMetaAndData> outFileRows = createResultDataFromObjects(rows);
try {
TestUtilities.checkRows(resultRows, outFileRows);
} catch (TestFailedException tfe) {
fail(tfe.getMessage());
}
}
Aggregations