use of org.pentaho.di.trans.steps.injector.InjectorMeta in project pentaho-kettle by pentaho.
the class UniqueRowsIT method testSortCaseInsensitiveUniqueCaseSensitive.
public void testSortCaseInsensitiveUniqueCaseSensitive() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("uniquerowstest");
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// Create a sort rows step
//
String sortRowsStepname = "sort rows step";
SortRowsMeta srm = new SortRowsMeta();
srm.setFieldName(new String[] { "KEY" });
srm.setAscending(new boolean[] { true });
srm.setCaseSensitive(new boolean[] { false });
srm.setPreSortedField(new boolean[] { false });
srm.setPrefix("SortRowsTest");
srm.setDirectory(".");
String sortRowsStepPid = registry.getPluginId(StepPluginType.class, srm);
StepMeta sortRowsStep = new StepMeta(sortRowsStepPid, sortRowsStepname, srm);
transMeta.addStep(sortRowsStep);
transMeta.addTransHop(new TransHopMeta(injectorStep, sortRowsStep));
//
// Create a unique rows step
//
String uniqueRowsStepname = "unique rows step";
UniqueRowsMeta urm = new UniqueRowsMeta();
urm.setCompareFields(new String[] { "KEY" });
urm.setCaseInsensitive(new boolean[] { false });
String uniqueRowsStepPid = registry.getPluginId(StepPluginType.class, urm);
StepMeta uniqueRowsStep = new StepMeta(uniqueRowsStepPid, uniqueRowsStepname, urm);
transMeta.addStep(uniqueRowsStep);
transMeta.addTransHop(new TransHopMeta(sortRowsStep, uniqueRowsStep));
//
// Create a dummy step
//
String dummyStepname = "dummy step";
DummyTransMeta dm = new DummyTransMeta();
String dummyPid = registry.getPluginId(StepPluginType.class, dm);
StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
transMeta.addStep(dummyStep);
transMeta.addTransHop(new TransHopMeta(uniqueRowsStep, dummyStep));
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(dummyStepname, 0);
RowStepCollector dummyRc = new RowStepCollector();
si.addRowListener(dummyRc);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createData();
for (RowMetaAndData rm : inputList) {
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
checkRows(createResultDataSortCaseInsensitiveUniqueCaseSensitive(), resultRows);
}
use of org.pentaho.di.trans.steps.injector.InjectorMeta in project pentaho-kettle by pentaho.
the class UniqueRowsIT method testAllUnique.
@Test
public void testAllUnique() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("uniquerowstest");
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// Create a unique rows step
//
String uniqueRowsStepname = "unique rows step";
UniqueRowsMeta urm = new UniqueRowsMeta();
urm.setCompareFields(new String[] { "KEY" });
urm.setCaseInsensitive(new boolean[] { true });
urm.setRejectDuplicateRow(true);
String uniqueRowsStepPid = registry.getPluginId(StepPluginType.class, urm);
StepMeta uniqueRowsStep = new StepMeta(uniqueRowsStepPid, uniqueRowsStepname, urm);
uniqueRowsStep.setDistributes(false);
transMeta.addStep(uniqueRowsStep);
transMeta.addTransHop(new TransHopMeta(injectorStep, uniqueRowsStep));
//
// Create a dummy step to receive the unique rows
//
String dummyStepname1 = "dummy step";
DummyTransMeta dm1 = new DummyTransMeta();
String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
transMeta.addStep(dummyStep1);
transMeta.addTransHop(new TransHopMeta(uniqueRowsStep, dummyStep1));
//
// Create a dummy step to receive the duplicate rows (errors)
//
String dummyStepname2 = "dummy step2";
DummyTransMeta dm2 = new DummyTransMeta();
String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
transMeta.addStep(dummyStep2);
// Set up error (aka duplicates) handling info
StepErrorMeta stepErrorMeta = new StepErrorMeta(new Variables(), uniqueRowsStep);
stepErrorMeta.setTargetStep(dummyStep2);
stepErrorMeta.setEnabled(true);
stepErrorMeta.setNrErrorsValuename("numErrors");
stepErrorMeta.setErrorDescriptionsValuename("duplicates");
stepErrorMeta.setErrorFieldsValuename("KEY");
stepErrorMeta.setErrorCodesValuename("errorCodes");
stepErrorMeta.setMaxErrors("9999");
stepErrorMeta.setMaxPercentErrors("");
stepErrorMeta.setMinPercentRows("");
uniqueRowsStep.setStepErrorMeta(stepErrorMeta);
transMeta.addTransHop(new TransHopMeta(uniqueRowsStep, dummyStep2));
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(dummyStepname1, 0);
RowStepCollector dummyRc1 = new RowStepCollector();
si.addRowListener(dummyRc1);
StepInterface si2 = trans.getStepInterface(dummyStepname2, 0);
RowStepCollector dummyRc2 = new RowStepCollector();
si2.addRowListener(dummyRc2);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createDataAllUnique();
for (RowMetaAndData rm : inputList) {
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
checkRows(createResultDataAllUnique(), resultRows);
List<RowMetaAndData> errorRows = dummyRc2.getRowsWritten();
// There should be no duplicates for this test
assertEquals(errorRows.size(), 0);
}
use of org.pentaho.di.trans.steps.injector.InjectorMeta in project pentaho-kettle by pentaho.
the class UpdateIT method setUp.
@Override
@Before
public void setUp() throws Exception {
KettleEnvironment.init();
/* SET UP TRANSFORMATION */
// Create a new transformation...
TransMeta transMeta = new TransMeta();
transMeta.setName("update test");
// Add the database connections
for (int i = 0; i < databasesXML.length; i++) {
DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
transMeta.addDatabase(databaseMeta);
}
DatabaseMeta dbInfo = transMeta.findDatabase("db");
/* SET UP DATABASE */
// Create target table
db = new Database(transMeta, dbInfo);
db.connect();
String source = db.getCreateTableStatement(TARGET_TABLE, getTargetTableRowMeta(), null, false, null, true);
db.execStatement(source);
// populate target table
for (String sql : insertStatement) {
db.execStatement(sql);
}
/* SET UP TRANSFORMATION STEPS */
PluginRegistry registry = PluginRegistry.getInstance();
// create an injector step...
String injectorStepName = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepName, im);
transMeta.addStep(injectorStep);
// create the update step...
String updateStepName = "update [" + TARGET_TABLE + "]";
upd = new UpdateMeta();
upd.setDatabaseMeta(transMeta.findDatabase("db"));
upd.setTableName(TARGET_TABLE);
upd.setUpdateLookup(new String[] { "VALUE" });
upd.setUpdateStream(new String[] { "VALUE" });
upd.setErrorIgnored(true);
String fromid = registry.getPluginId(StepPluginType.class, upd);
StepMeta updateStep = new StepMeta(fromid, updateStepName, upd);
updateStep.setDescription("update data in table [" + TARGET_TABLE + "] on database [" + dbInfo + "]");
transMeta.addStep(updateStep);
TransHopMeta hi = new TransHopMeta(injectorStep, updateStep);
transMeta.addTransHop(hi);
/* PREPARE TRANSFORMATION EXECUTION */
trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(updateStepName, 0);
rc = new RowStepCollector();
si.addRowListener(rc);
rp = trans.addRowProducer(injectorStepName, 0);
}
use of org.pentaho.di.trans.steps.injector.InjectorMeta in project pentaho-kettle by pentaho.
the class ValueMapperIT method testValueMapper6.
/**
* Test case for valuemapper step. Injector step to a valuemapper step to a dummy step. Rows go in and should be
* mapped accordingly.
*
* This test will explicitly test the empty field processing. using a non matching default.
*/
public void testValueMapper6() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("valuemappertest6");
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// Create a dummy step 1
//
String dummyStepname1 = "dummy step 1";
DummyTransMeta dm1 = new DummyTransMeta();
String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
transMeta.addStep(dummyStep1);
TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
transMeta.addTransHop(hi);
//
// Create a ValueMapper step
//
String valueMapperName = "valuemapper step";
ValueMapperMeta vm = new ValueMapperMeta();
vm.setFieldToUse("field1");
vm.setTargetField("");
vm.setNonMatchDefault("default");
vm.setSourceValue(new String[] { "abc", "ABC", "def", "ghijk", null });
vm.setTargetValue(new String[] { "begin", "BEG_LONGER", "test", "blah", "emptyField" });
String valueMapperPid = registry.getPluginId(StepPluginType.class, vm);
StepMeta valueMapperStep = new StepMeta(valueMapperPid, valueMapperName, vm);
transMeta.addStep(valueMapperStep);
TransHopMeta hi2 = new TransHopMeta(dummyStep1, valueMapperStep);
transMeta.addTransHop(hi2);
//
// Create a dummy step 2
//
String dummyStepname2 = "dummy step 2";
DummyTransMeta dm2 = new DummyTransMeta();
String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
transMeta.addStep(dummyStep2);
TransHopMeta hi3 = new TransHopMeta(valueMapperStep, dummyStep2);
transMeta.addTransHop(hi3);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
boolean prepare;
try {
trans.prepareExecution(null);
prepare = true;
} catch (KettleException e) {
prepare = false;
}
assertTrue(prepare);
StepInterface si = trans.getStepInterface(dummyStepname1, 0);
RowStepCollector dummyRc1 = new RowStepCollector();
si.addRowListener(dummyRc1);
si = trans.getStepInterface(valueMapperName, 0);
RowStepCollector valueMapperRc = new RowStepCollector();
si.addRowListener(valueMapperRc);
si = trans.getStepInterface(dummyStepname2, 0);
RowStepCollector dummyRc = new RowStepCollector();
si.addRowListener(dummyRc);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createData();
Iterator<RowMetaAndData> it = inputList.iterator();
while (it.hasNext()) {
RowMetaAndData rm = it.next();
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
// Compare the results
List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
List<RowMetaAndData> goldenImageRows = createResultData6();
checkRows(goldenImageRows, resultRows);
}
use of org.pentaho.di.trans.steps.injector.InjectorMeta in project pentaho-kettle by pentaho.
the class ValueMapperIT method testValueMapper4.
/*----------------- TODO */
/**
* Test case for valuemapper step. Injector step to a valuemapper step to a dummy step. Rows go in and should be
* mapped accordingly.
*
* This test will write the mappings in a new field, using a non matching default.
*/
public void testValueMapper4() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("valuemappertest4");
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// Create a dummy step 1
//
String dummyStepname1 = "dummy step 1";
DummyTransMeta dm1 = new DummyTransMeta();
String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
transMeta.addStep(dummyStep1);
TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
transMeta.addTransHop(hi);
//
// Create a step
//
String valueMapperName = "valuemapper step";
ValueMapperMeta vm = new ValueMapperMeta();
vm.setFieldToUse("field1");
vm.setTargetField("new_field");
vm.setNonMatchDefault("default");
vm.setSourceValue(new String[] { "abc", "ABC", "def", "ghij" });
vm.setTargetValue(new String[] { "begin", "BEG_LONGER", "test", "" });
String valueMapperPid = registry.getPluginId(StepPluginType.class, vm);
StepMeta valueMapperStep = new StepMeta(valueMapperPid, valueMapperName, vm);
transMeta.addStep(valueMapperStep);
TransHopMeta hi2 = new TransHopMeta(dummyStep1, valueMapperStep);
transMeta.addTransHop(hi2);
//
// Create a dummy step 2
//
String dummyStepname2 = "dummy step 2";
DummyTransMeta dm2 = new DummyTransMeta();
String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
transMeta.addStep(dummyStep2);
TransHopMeta hi3 = new TransHopMeta(valueMapperStep, dummyStep2);
transMeta.addTransHop(hi3);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(dummyStepname1, 0);
RowStepCollector dummyRc1 = new RowStepCollector();
si.addRowListener(dummyRc1);
si = trans.getStepInterface(valueMapperName, 0);
RowStepCollector valueMapperRc = new RowStepCollector();
si.addRowListener(valueMapperRc);
si = trans.getStepInterface(dummyStepname2, 0);
RowStepCollector dummyRc = new RowStepCollector();
si.addRowListener(dummyRc);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createData();
Iterator<RowMetaAndData> it = inputList.iterator();
while (it.hasNext()) {
RowMetaAndData rm = it.next();
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
// Compare the results
List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
List<RowMetaAndData> goldenImageRows = createResultData4();
checkRows(goldenImageRows, resultRows);
}
Aggregations