use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.
the class TableCompareIT method testValueNotExistsReference.
/**
* Test compare table if reference table is empty
*
* @throws IOException
* @throws KettleException
*/
@Test
public void testValueNotExistsReference() throws IOException, KettleException {
executeSqlPrecondition("compare_only.sql");
TableCompareMeta meta = getTableCompareMeta();
List<RowMetaAndData> inputData = new ArrayList<RowMetaAndData>();
inputData.add(new RowMetaAndData(getRowMeta(), getData3()));
TransMeta trMeta = TransTestFactory.generateTestTransformationError(null, meta, "junit");
Map<String, RowStepCollector> result = TransTestFactory.executeTestTransformationError(trMeta, "junit", inputData);
List<RowMetaAndData> read = result.get(TransTestFactory.DUMMY_STEPNAME).getRowsRead();
List<RowMetaAndData> errors = result.get(TransTestFactory.ERROR_STEPNAME).getRowsRead();
Assert.assertEquals("One row passed to positive output", 1, read.size());
RowMetaAndData row = read.get(0);
Assert.assertEquals("Errors reported", 4, row.getInteger(8).intValue());
Assert.assertEquals("Reference table row count", 0, row.getInteger(9).intValue());
Assert.assertEquals("Compare table row count", 4, row.getInteger(10).intValue());
Assert.assertEquals("Number of left joins errors", 0, row.getInteger(11).intValue());
Assert.assertEquals("Number of inner joins errors", 0, row.getInteger(12).intValue());
Assert.assertEquals("Number of right joins errors", 4, row.getInteger(13).intValue());
Assert.assertEquals("4 error rows passed to error output", 4, errors.size());
}
use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.
the class TableInputIT method testTableInputWithParam.
/**
* Test case for table input which is taking its input from a hop. This is a regression test case for JIRA PDI-588.
*
* The query in the table input step has one '?' and this parameter is filled by values read from an input hop.
*/
public void testTableInputWithParam() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("transname");
// Add the database connections
for (int i = 0; i < databasesXML.length; i++) {
DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
transMeta.addDatabase(databaseMeta);
}
DatabaseMeta dbInfo = transMeta.findDatabase("db");
// Execute our setup SQLs in the database.
Database database = new Database(transMeta, dbInfo);
database.connect();
createTables(database);
createData(database);
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// create the source step...
//
String fromstepname = "read from [" + source_table + "]";
TableInputMeta tii = new TableInputMeta();
tii.setDatabaseMeta(transMeta.findDatabase("db"));
tii.setLookupFromStep(injectorStep);
tii.setExecuteEachInputRow(true);
String selectSQL = "SELECT " + Const.CR;
selectSQL += "ID, CODE ";
selectSQL += "FROM " + source_table + " WHERE CODE = ? ORDER BY ID, CODE;";
tii.setSQL(selectSQL);
String fromstepid = registry.getPluginId(StepPluginType.class, tii);
StepMeta fromstep = new StepMeta(fromstepid, fromstepname, tii);
fromstep.setDescription("Reads information from table [" + source_table + "] on database [" + dbInfo + "]");
transMeta.addStep(fromstep);
TransHopMeta hi = new TransHopMeta(injectorStep, fromstep);
transMeta.addTransHop(hi);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(fromstepname, 0);
RowStepCollector rc = new RowStepCollector();
si.addRowListener(rc);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createDataRows();
for (RowMetaAndData rm : inputList) {
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
List<RowMetaAndData> resultRows = rc.getRowsWritten();
List<RowMetaAndData> goldRows = createResultDataRows();
checkRows(goldRows, resultRows);
}
use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.
the class TableOutputIT method testTableOutputNormal.
/**
* Test case for normal table output case.
*/
@SuppressWarnings("deprecation")
public void testTableOutputNormal() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("table output normal test");
// Add the database connections
for (int i = 0; i < databasesXML.length; i++) {
DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
transMeta.addDatabase(databaseMeta);
}
DatabaseMeta dbInfo = transMeta.findDatabase("db");
// Execute our setup SQLs in the database.
Database database = new Database(transMeta, dbInfo);
database.connect();
createTable(database, target_table, createSourceRowMetaInterface1());
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// create the source step...
//
String outputname = "output to [" + target_table + "]";
TableOutputMeta tom = new TableOutputMeta();
tom.setDatabaseMeta(transMeta.findDatabase("db"));
tom.setTablename(target_table);
String fromid = registry.getPluginId(StepPluginType.class, tom);
StepMeta fromstep = new StepMeta(fromid, outputname, tom);
fromstep.setDescription("write data to table [" + target_table + "] on database [" + dbInfo + "]");
transMeta.addStep(fromstep);
TransHopMeta hi = new TransHopMeta(injectorStep, fromstep);
transMeta.addTransHop(hi);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(outputname, 0);
RowStepCollector rc = new RowStepCollector();
si.addRowListener(rc);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createNormalDataRows();
for (RowMetaAndData rm : inputList) {
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
List<RowMetaAndData> resultRows = rc.getRowsWritten();
List<RowMetaAndData> goldRows = createNormalDataRows();
checkRows(goldRows, resultRows);
checkResultsNormal(database);
}
use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.
the class TableOutputIT method testTableOutputJIRA2733.
/**
* Test case for commitSize see PDI2733 in JIRA.
*/
@SuppressWarnings("deprecation")
public void testTableOutputJIRA2733() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("table output JIRA2733 test");
// Add the database connections
for (int i = 0; i < databasesXML.length; i++) {
DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
transMeta.addDatabase(databaseMeta);
}
DatabaseMeta dbInfo = transMeta.findDatabase("db");
// Execute our setup SQLs in the database.
Database database = new Database(transMeta, dbInfo);
database.connect();
createTable(database, target_table3, createSourceRowMetaInterface1());
// Add "ts" timestamp field to target_table with a default value of NOW()
database.execStatement("ALTER TABLE " + target_table3 + " ADD COLUMN ts TIMESTAMP DEFAULT NOW() ");
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// create the source step...
//
String outputname = "output to [" + target_table3 + "]";
TableOutputMeta tom = new TableOutputMeta();
tom.setDatabaseMeta(transMeta.findDatabase("db"));
tom.setTablename(target_table3);
tom.setTruncateTable(true);
tom.setUseBatchUpdate(true);
String fromid = registry.getPluginId(StepPluginType.class, tom);
StepMeta fromstep = new StepMeta(fromid, outputname, tom);
fromstep.setDescription("write data to table [" + target_table3 + "] on database [" + dbInfo + "]");
transMeta.addStep(fromstep);
TransHopMeta hi = new TransHopMeta(injectorStep, fromstep);
transMeta.addTransHop(hi);
// With seven rows these are the number of commits that need to made
// for "commitSize"s ranging between 0 and 8. (0=auto-commit=no commits)
int[] goldRowCounts = { 1, 8, 4, 3, 2, 2, 2, 2, 1 };
for (int commitSize = 0; commitSize <= 8; commitSize++) {
tom.setCommitSize(commitSize);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(outputname, 0);
RowStepCollector rc = new RowStepCollector();
si.addRowListener(rc);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createNormalDataRows();
for (RowMetaAndData rm : inputList) {
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
// Get the number of commits from the DB connection
// in the table output step...
//
TableOutputData data = (TableOutputData) trans.findDataInterface(outputname);
int exp = goldRowCounts[commitSize];
// remove 1 commit too many in the dispose method.
//
int act = data.db.getNrExecutedCommits() - 1;
assertEquals("Incorrect number of commits with commitSize=" + commitSize + Const.CR, exp, act);
}
dropTable(database, target_table3);
}
use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.
the class TableOutputIT method testTableOutputJIRA897.
/**
* Test case for normal table output where the table is included in the instream, but the tablename is not stored in
* the table.
*/
public void testTableOutputJIRA897() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("table output JIRA897 test");
// Add the database connections
for (int i = 0; i < databasesXML.length; i++) {
DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
transMeta.addDatabase(databaseMeta);
}
DatabaseMeta dbInfo = transMeta.findDatabase("db");
// Execute our setup SQLs in the database.
Database database = new Database(transMeta, dbInfo);
database.connect();
createTable(database, target_table1, createSourceRowMetaInterface1());
createTable(database, target_table2, createSourceRowMetaInterface1());
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// create the source step...
//
String outputname = "output to [" + target_table1 + "] and [" + target_table2 + "]";
TableOutputMeta tom = new TableOutputMeta();
tom.setDatabaseMeta(transMeta.findDatabase("db"));
tom.setTableNameInField(true);
tom.setTableNameField("TABLE");
tom.setTableNameInTable(false);
String fromid = registry.getPluginId(StepPluginType.class, tom);
StepMeta fromstep = new StepMeta(fromid, outputname, tom);
fromstep.setDescription("write data to tables on database [" + dbInfo + "]");
transMeta.addStep(fromstep);
TransHopMeta hi = new TransHopMeta(injectorStep, fromstep);
transMeta.addTransHop(hi);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(outputname, 0);
RowStepCollector rc = new RowStepCollector();
si.addRowListener(rc);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createJIRA897DataRows();
for (RowMetaAndData rm : inputList) {
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
List<RowMetaAndData> resultRows = rc.getRowsWritten();
// The name of the table should still be in here.
List<RowMetaAndData> goldRows = createJIRA897DataRows();
checkRows(goldRows, resultRows);
checkResultsJIRA897(database);
}
Aggregations