use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.
the class JavaScriptSpecialIT method testLuhnCheck.
/**
* Test case for javascript functionality: ltrim(), rtrim(), trim().
*/
public void testLuhnCheck() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("test javascript LuhnCheck");
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// Create a javascript step
//
String javaScriptStepname = "javascript step";
ScriptValuesMetaMod svm = new ScriptValuesMetaMod();
ScriptValuesScript[] js = new ScriptValuesScript[] { new ScriptValuesScript(ScriptValuesScript.TRANSFORM_SCRIPT, "script", "var str = string;\n" + "var bool = LuhnCheck(str);") };
svm.setJSScripts(js);
svm.setFieldname(new String[] { "bool" });
svm.setRename(new String[] { "" });
svm.setType(new int[] { ValueMetaInterface.TYPE_BOOLEAN });
svm.setLength(new int[] { -1 });
svm.setPrecision(new int[] { -1 });
svm.setReplace(new boolean[] { false });
svm.setCompatible(false);
String javaScriptStepPid = registry.getPluginId(StepPluginType.class, svm);
StepMeta javaScriptStep = new StepMeta(javaScriptStepPid, javaScriptStepname, svm);
transMeta.addStep(javaScriptStep);
TransHopMeta hi1 = new TransHopMeta(injectorStep, javaScriptStep);
transMeta.addTransHop(hi1);
//
// Create a dummy step
//
String dummyStepname = "dummy step";
DummyTransMeta dm = new DummyTransMeta();
String dummyPid = registry.getPluginId(StepPluginType.class, dm);
StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
transMeta.addStep(dummyStep);
TransHopMeta hi2 = new TransHopMeta(javaScriptStep, dummyStep);
transMeta.addTransHop(hi2);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si;
si = trans.getStepInterface(javaScriptStepname, 0);
RowStepCollector javaScriptRc = new RowStepCollector();
si.addRowListener(javaScriptRc);
si = trans.getStepInterface(dummyStepname, 0);
RowStepCollector dummyRc = new RowStepCollector();
si.addRowListener(dummyRc);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createData1();
Iterator<RowMetaAndData> it = inputList.iterator();
while (it.hasNext()) {
RowMetaAndData rm = it.next();
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
List<RowMetaAndData> goldenImageRows = createResultData1();
List<RowMetaAndData> resultRows1 = javaScriptRc.getRowsWritten();
checkRows(resultRows1, goldenImageRows);
List<RowMetaAndData> resultRows2 = dummyRc.getRowsRead();
checkRows(resultRows2, goldenImageRows);
}
use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.
the class JavaScriptStringIT method testStringsPadCase.
/**
* Test case for javascript functionality: lpad(), rpad(), upper(), lower().
*/
public void testStringsPadCase() throws Exception {
KettleEnvironment.init();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("test javascript pad casing");
PluginRegistry registry = PluginRegistry.getInstance();
//
// create an injector step...
//
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
//
// Create a javascript step
//
String javaScriptStepname = "javascript step";
ScriptValuesMetaMod svm = new ScriptValuesMetaMod();
ScriptValuesScript[] js = new ScriptValuesScript[] { new ScriptValuesScript(ScriptValuesScript.TRANSFORM_SCRIPT, "script", "var lpadded1 = lpad(string, \"x\", 10);\n" + "var lpadded2 = lpad(string, \" \", 9);\n" + "var rpadded1 = rpad(string, \"x\", 10);\n" + "var rpadded2 = rpad(string, \" \", 9);\n" + "var upperStr = upper(string);\n" + "var lowerStr = lower(string);\n") };
svm.setJSScripts(js);
svm.setFieldname(new String[] { "lpadded1", "lpadded2", "rpadded1", "rpadded2", "upperStr", "lowerStr" });
svm.setRename(new String[] { "", "", "", "", "", "", "" });
svm.setType(new int[] { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING });
svm.setLength(new int[] { -1, -1, -1, -1, -1, -1, -1 });
svm.setPrecision(new int[] { -1, -1, -1, -1, -1, -1, -1 });
svm.setReplace(new boolean[] { false, false, false, false, false, false, false });
svm.setCompatible(true);
String javaScriptStepPid = registry.getPluginId(StepPluginType.class, svm);
StepMeta javaScriptStep = new StepMeta(javaScriptStepPid, javaScriptStepname, svm);
transMeta.addStep(javaScriptStep);
TransHopMeta hi1 = new TransHopMeta(injectorStep, javaScriptStep);
transMeta.addTransHop(hi1);
//
// Create a dummy step
//
String dummyStepname = "dummy step";
DummyTransMeta dm = new DummyTransMeta();
String dummyPid = registry.getPluginId(StepPluginType.class, dm);
StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
transMeta.addStep(dummyStep);
TransHopMeta hi2 = new TransHopMeta(javaScriptStep, dummyStep);
transMeta.addTransHop(hi2);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si;
si = trans.getStepInterface(javaScriptStepname, 0);
RowStepCollector javaScriptRc = new RowStepCollector();
si.addRowListener(javaScriptRc);
si = trans.getStepInterface(dummyStepname, 0);
RowStepCollector dummyRc = new RowStepCollector();
si.addRowListener(dummyRc);
RowProducer rp = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createData2();
Iterator<RowMetaAndData> it = inputList.iterator();
while (it.hasNext()) {
RowMetaAndData rm = it.next();
rp.putRow(rm.getRowMeta(), rm.getData());
}
rp.finished();
trans.waitUntilFinished();
List<RowMetaAndData> goldenImageRows = createResultData2();
List<RowMetaAndData> resultRows1 = javaScriptRc.getRowsWritten();
checkRows(resultRows1, goldenImageRows);
List<RowMetaAndData> resultRows2 = dummyRc.getRowsRead();
checkRows(resultRows2, goldenImageRows);
}
use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.
the class SplitFieldToRowsIT method splitFieldToRows.
/**
* Splits the "stringToSplit" with the passed "delimiter". The "delimiter" is assumed by this method to be a Kettle
* variable. The parameter "delimiterVariableValue" should contain the variables value.
*
* The "isDelimiterRegex" parameter will process the use regex for pattern matching if true.
*
* @param testName
* @param stringToSplit
* @param isDelimiterRegex
* @param delimiter
* @param delimiterVariableValue
* @return
* @throws Exception
*/
private List<RowMetaAndData> splitFieldToRows(String testName, String stringToSplit, boolean isDelimiterRegex, String delimiter, String delimiterVariableValue) {
RowStepCollector rc = new RowStepCollector();
try {
KettleEnvironment.init();
// Create a new transformation...
TransMeta transMeta = new TransMeta();
transMeta.setName("Split field to rows test");
PluginRegistry registry = PluginRegistry.getInstance();
// create an injector step...
String injectorStepname = "injector step";
InjectorMeta im = new InjectorMeta();
// Set the information of the injector.
String injectorPid = registry.getPluginId(StepPluginType.class, im);
StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
transMeta.addStep(injectorStep);
// Create a Split Field to Rows step
String splitfieldToRowsName = "Split field to rows";
SplitFieldToRowsMeta splitFieldtoRowsMeta = new SplitFieldToRowsMeta();
splitFieldtoRowsMeta.setDelimiter(delimiter);
splitFieldtoRowsMeta.setDelimiterRegex(isDelimiterRegex);
splitFieldtoRowsMeta.setSplitField(FIELD_TO_SPLIT_NAME);
splitFieldtoRowsMeta.setNewFieldname(NEW_FIELD_NAME);
String splitFieldTotRowsPid = registry.getPluginId(StepPluginType.class, splitFieldtoRowsMeta);
StepMeta splitFieldToRows = new StepMeta(splitFieldTotRowsPid, splitfieldToRowsName, splitFieldtoRowsMeta);
transMeta.addStep(splitFieldToRows);
// hop the injector to the split field to rows step
TransHopMeta hop_injector_splitfieldToRows = new TransHopMeta(injectorStep, splitFieldToRows);
transMeta.addTransHop(hop_injector_splitfieldToRows);
// Create a dummy step
String dummyStepname = "dummy step";
DummyTransMeta dm = new DummyTransMeta();
String dummyPid = registry.getPluginId(StepPluginType.class, dm);
StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
transMeta.addStep(dummyStep);
TransHopMeta hop_SplitFieldToRows_Dummy = new TransHopMeta(splitFieldToRows, dummyStep);
transMeta.addTransHop(hop_SplitFieldToRows_Dummy);
if (!Utils.isEmpty(delimiterVariableValue)) {
String delimiterVariableName = delimiter.replace("${", "");
delimiterVariableName = delimiterVariableName.replace("}", "");
transMeta.setVariable(delimiterVariableName, delimiterVariableValue);
}
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(dummyStepname, 0);
si.addRowListener(rc);
RowProducer rowProducer = trans.addRowProducer(injectorStepname, 0);
trans.startThreads();
// add rows
List<RowMetaAndData> inputList = createData(stringToSplit);
for (RowMetaAndData rm : inputList) {
rowProducer.putRow(rm.getRowMeta(), rm.getData());
}
rowProducer.finished();
trans.waitUntilFinished();
} catch (KettleException e) {
fail("KettleEnvironment exception" + e.getMessage());
}
List<RowMetaAndData> resultRows = rc.getRowsWritten();
return resultRows;
}
use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.
the class TableCompareIT method testValueExcludeComparsion.
/**
* Test that step can ignore excluded values during comparison
*
* @throws IOException
* @throws KettleException
*/
@Test
public void testValueExcludeComparsion() throws IOException, KettleException {
executeSqlPrecondition("complex_key_test.sql");
TableCompareMeta meta = getTableCompareMeta();
List<RowMetaAndData> inputData = new ArrayList<RowMetaAndData>();
inputData.add(new RowMetaAndData(getRowMeta(), getData4()));
TransMeta trMeta = TransTestFactory.generateTestTransformationError(null, meta, "junit");
Map<String, RowStepCollector> result = TransTestFactory.executeTestTransformationError(trMeta, "junit", inputData);
List<RowMetaAndData> read = result.get(TransTestFactory.DUMMY_STEPNAME).getRowsRead();
List<RowMetaAndData> errors = result.get(TransTestFactory.ERROR_STEPNAME).getRowsRead();
Assert.assertEquals("There is no errors reported", 0, errors.size());
RowMetaAndData row = read.get(0);
Assert.assertEquals("No errors reported", 0, row.getInteger(8).intValue());
Assert.assertEquals("Reference table row count", 4, row.getInteger(9).intValue());
Assert.assertEquals("Compare table row count", 4, row.getInteger(10).intValue());
Assert.assertEquals("Number of left joins errors", 0, row.getInteger(11).intValue());
Assert.assertEquals("Number of inner joins errors", 0, row.getInteger(12).intValue());
Assert.assertEquals("Number of right joins errors", 0, row.getInteger(13).intValue());
}
use of org.pentaho.di.trans.RowStepCollector in project pentaho-kettle by pentaho.
the class CalculatorIT method testCalculator1.
public void testCalculator1() throws Exception {
KettleEnvironment.init();
PluginRegistry registry = PluginRegistry.getInstance();
//
// Create a new transformation...
//
TransMeta transMeta = new TransMeta();
transMeta.setName("calculatortest1");
//
// create a row generator step...
//
String rowGeneratorStepname = "row generator step";
RowGeneratorMeta rm = new RowGeneratorMeta();
// Set the information of the row generator.
String rowGeneratorPid = registry.getPluginId(StepPluginType.class, rm);
StepMeta rowGeneratorStep = new StepMeta(rowGeneratorPid, rowGeneratorStepname, rm);
transMeta.addStep(rowGeneratorStep);
//
// Generate 1 empty row
//
String[] strDummies = {};
int[] intDummies = {};
rm.setDefault();
rm.setFieldName(strDummies);
rm.setFieldType(strDummies);
rm.setValue(strDummies);
rm.setFieldLength(intDummies);
rm.setFieldPrecision(intDummies);
rm.setRowLimit("1");
rm.setFieldFormat(strDummies);
rm.setGroup(strDummies);
rm.setDecimal(strDummies);
//
// Add calculator step.
//
String calculatorStepname1 = "calculator 1";
CalculatorMeta calc1 = new CalculatorMeta();
CalculatorMetaFunction[] calculations = new CalculatorMetaFunction[] { new CalculatorMetaFunction(// fieldName
"timestamp1", // calctype
CalculatorMetaFunction.CALC_CONSTANT, // fieldA
"1970-01-01 00:00:00.100100", // String fieldB
"", // String fieldC
"", // valueType,
ValueMetaInterface.TYPE_TIMESTAMP, // int valueLength,
0, // int valuePrecision,
0, // boolean removedFromResult,
false, // String conversionMask,
"", // String decimalSymbol,
"", // String groupingSymbol,
"", // String currencySymbol
""), new CalculatorMetaFunction(// fieldName
"int1", // calctype
CalculatorMetaFunction.CALC_CONSTANT, // fieldA
"1", // String fieldB
"", // String fieldC
"", // valueType,
ValueMetaInterface.TYPE_INTEGER, // int valueLength,
0, // int valuePrecision,
0, // boolean removedFromResult,
false, // String conversionMask,
"", // String decimalSymbol,
"", // String groupingSymbol,
"", // String currencySymbol
""), new CalculatorMetaFunction(// fieldName
"timestamp plus 1 day", // calctype
CalculatorMetaFunction.CALC_ADD_DAYS, // fieldA
"timestamp1", // String fieldB
"int1", // String fieldC
"", // valueType,
ValueMetaInterface.TYPE_DATE, // int valueLength,
0, // int valuePrecision,
0, // boolean removedFromResult,
false, // String conversionMask,
"", // String decimalSymbol,
"", // String groupingSymbol,
"", // String currencySymbol
"") };
calc1.setCalculation(calculations);
//
String calculatorPid1 = registry.getPluginId(StepPluginType.class, calc1);
StepMeta calcualtorStep1 = new StepMeta(calculatorPid1, calculatorStepname1, calc1);
transMeta.addStep(calcualtorStep1);
//
TransHopMeta hi1 = new TransHopMeta(rowGeneratorStep, calcualtorStep1);
transMeta.addTransHop(hi1);
// Now execute the transformation...
Trans trans = new Trans(transMeta);
trans.prepareExecution(null);
StepInterface si = trans.getStepInterface(calculatorStepname1, 0);
RowStepCollector endRc = new RowStepCollector();
si.addRowListener(endRc);
trans.startThreads();
trans.waitUntilFinished();
// Now check whether the output is still as we expect.
List<RowMetaAndData> goldenImageRows = createResultData1();
List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
checkRows(resultRows1, goldenImageRows);
}
Aggregations