Examples with TransHopMeta org.pentaho.di.trans.TransHopMeta used on opensource projects

Example 61 with TransHopMeta

use of org.pentaho.di.trans.TransHopMeta in project pentaho-kettle by pentaho.

the class CalculatorIT method testCalculator1.

public void testCalculator1() throws Exception {
    KettleEnvironment.init();
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("calculatortest1");
    // 
    // create a row generator step...
    // 
    String rowGeneratorStepname = "row generator step";
    RowGeneratorMeta rm = new RowGeneratorMeta();
    // Set the information of the row generator.
    String rowGeneratorPid = registry.getPluginId(StepPluginType.class, rm);
    StepMeta rowGeneratorStep = new StepMeta(rowGeneratorPid, rowGeneratorStepname, rm);
    transMeta.addStep(rowGeneratorStep);
    // 
    // Generate 1 empty row
    // 
    String[] strDummies = {};
    int[] intDummies = {};
    rm.setDefault();
    rm.setFieldName(strDummies);
    rm.setFieldType(strDummies);
    rm.setValue(strDummies);
    rm.setFieldLength(intDummies);
    rm.setFieldPrecision(intDummies);
    rm.setRowLimit("1");
    rm.setFieldFormat(strDummies);
    rm.setGroup(strDummies);
    rm.setDecimal(strDummies);
    // 
    // Add calculator step.
    // 
    String calculatorStepname1 = "calculator 1";
    CalculatorMeta calc1 = new CalculatorMeta();
    CalculatorMetaFunction[] calculations = new CalculatorMetaFunction[] { new CalculatorMetaFunction(// fieldName
    "timestamp1", // calctype
    CalculatorMetaFunction.CALC_CONSTANT, // fieldA
    "1970-01-01 00:00:00.100100", // String fieldB
    "", // String fieldC
    "", // valueType,
    ValueMetaInterface.TYPE_TIMESTAMP, // int valueLength,
    0, // int valuePrecision,
    0, // boolean removedFromResult,
    false, // String conversionMask,
    "", // String decimalSymbol,
    "", // String groupingSymbol,
    "", // String currencySymbol
    ""), new CalculatorMetaFunction(// fieldName
    "int1", // calctype
    CalculatorMetaFunction.CALC_CONSTANT, // fieldA
    "1", // String fieldB
    "", // String fieldC
    "", // valueType,
    ValueMetaInterface.TYPE_INTEGER, // int valueLength,
    0, // int valuePrecision,
    0, // boolean removedFromResult,
    false, // String conversionMask,
    "", // String decimalSymbol,
    "", // String groupingSymbol,
    "", // String currencySymbol
    ""), new CalculatorMetaFunction(// fieldName
    "timestamp plus 1 day", // calctype
    CalculatorMetaFunction.CALC_ADD_DAYS, // fieldA
    "timestamp1", // String fieldB
    "int1", // String fieldC
    "", // valueType,
    ValueMetaInterface.TYPE_DATE, // int valueLength,
    0, // int valuePrecision,
    0, // boolean removedFromResult,
    false, // String conversionMask,
    "", // String decimalSymbol,
    "", // String groupingSymbol,
    "", // String currencySymbol
    "") };
    calc1.setCalculation(calculations);
    // 
    String calculatorPid1 = registry.getPluginId(StepPluginType.class, calc1);
    StepMeta calcualtorStep1 = new StepMeta(calculatorPid1, calculatorStepname1, calc1);
    transMeta.addStep(calcualtorStep1);
    // 
    TransHopMeta hi1 = new TransHopMeta(rowGeneratorStep, calcualtorStep1);
    transMeta.addTransHop(hi1);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(calculatorStepname1, 0);
    RowStepCollector endRc = new RowStepCollector();
    si.addRowListener(endRc);
    trans.startThreads();
    trans.waitUntilFinished();
    // Now check whether the output is still as we expect.
    List<RowMetaAndData> goldenImageRows = createResultData1();
    List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
    checkRows(resultRows1, goldenImageRows);
}

Example 62 with TransHopMeta

use of org.pentaho.di.trans.TransHopMeta in project pentaho-kettle by pentaho.

the class ConstantIT method testConstant1.

/**
 * Test case for Constant step. Row generator attached to a constant step.
 */
public void testConstant1() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("constanttest1");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create a row generator step...
    // 
    String rowGeneratorStepname = "row generator step";
    RowGeneratorMeta rm = new RowGeneratorMeta();
    // Set the information of the row generator.
    String rowGeneratorPid = registry.getPluginId(StepPluginType.class, rm);
    StepMeta rowGeneratorStep = new StepMeta(rowGeneratorPid, rowGeneratorStepname, rm);
    transMeta.addStep(rowGeneratorStep);
    // 
    // Generate 1 empty row
    // 
    String[] fieldName = {};
    String[] type = {};
    String[] value = {};
    String[] fieldFormat = {};
    String[] group = {};
    String[] decimal = {};
    int[] intDummies = {};
    rm.setDefault();
    rm.setFieldName(fieldName);
    rm.setFieldType(type);
    rm.setValue(value);
    rm.setFieldLength(intDummies);
    rm.setFieldPrecision(intDummies);
    rm.setRowLimit("1");
    rm.setFieldFormat(fieldFormat);
    rm.setGroup(group);
    rm.setDecimal(decimal);
    // 
    // Add constant step.
    // 
    String constStepname1 = "constant 1";
    ConstantMeta cnst1 = new ConstantMeta();
    String[] fieldName1 = { "boolean1", "boolean2", "boolean3", "boolean4", "boolean5", "boolean6", "boolean7", "string1", "string2", "string3", "integer1", "integer2", "integer3", "integer4", "number1", "number2", "number3", "number4", "timestamp1" };
    String[] type1 = { "boolean", "Boolean", "bOOLEAN", "BOOLEAN", "boolean", "boolean", "boolean", "string", "string", "String", "integer", "integer", "integer", "integer", "number", "number", "number", "number", "timestamp" };
    String[] value1 = { "Y", "T", "a", "TRUE", "0", "9", "", "AAAAAAAAAAAAAA", "   ", "", "-100", "0", "212", "", "-100.2", "0.0", "212.23", "", "1970-01-01 00:00:00.000" };
    String[] fieldFormat1 = { "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" };
    String[] group1 = { "", "", "", "", "", "", "", "", "", "", "", "", "", "", ",", ",", ",", ",", "" };
    String[] decimal1 = { "", "", "", "", "", "", "", "", "", "", "", "", "", "", ".", ".", ".", ".", "" };
    String[] currency = { "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" };
    int[] intDummies1 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
    boolean[] setEmptystring = { false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false };
    cnst1.setFieldName(fieldName1);
    cnst1.setFieldType(type1);
    cnst1.setValue(value1);
    cnst1.setFieldLength(intDummies1);
    cnst1.setFieldPrecision(intDummies1);
    cnst1.setFieldFormat(fieldFormat1);
    cnst1.setGroup(group1);
    cnst1.setDecimal(decimal1);
    cnst1.setCurrency(currency);
    cnst1.setEmptyString(setEmptystring);
    String addSeqPid1 = registry.getPluginId(StepPluginType.class, cnst1);
    StepMeta addSeqStep1 = new StepMeta(addSeqPid1, constStepname1, cnst1);
    transMeta.addStep(addSeqStep1);
    TransHopMeta hi1 = new TransHopMeta(rowGeneratorStep, addSeqStep1);
    transMeta.addTransHop(hi1);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(constStepname1, 0);
    RowStepCollector endRc = new RowStepCollector();
    si.addRowListener(endRc);
    trans.startThreads();
    trans.waitUntilFinished();
    // Now check whether the output is still as we expect.
    List<RowMetaAndData> goldenImageRows = createResultData1();
    List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
    checkRows(resultRows1, goldenImageRows);
}

Example 63 with TransHopMeta

use of org.pentaho.di.trans.TransHopMeta in project pentaho-kettle by pentaho.

the class CubeInputStepIntIT method testPDI12897.

@Test
public void testPDI12897() throws KettleException, IOException {
    File tempOutputFile = File.createTempFile("testPDI11374", ".cube");
    tempOutputFile.deleteOnExit();
    String stepName = "Cube Output";
    CubeOutputMeta meta = new CubeOutputMeta();
    meta.setDefault();
    meta.setFilename(tempOutputFile.getAbsolutePath());
    TransMeta transMeta = TransTestFactory.generateTestTransformation(null, meta, stepName);
    List<RowMetaAndData> inputList = getSampleData();
    TransTestFactory.executeTestTransformation(transMeta, TransTestFactory.INJECTOR_STEPNAME, stepName, TransTestFactory.DUMMY_STEPNAME, inputList);
    try {
        Thread.sleep(1000);
    } catch (InterruptedException ignored) {
    // Continue
    }
    // Now, check the result
    Variables varSpace = new Variables();
    varSpace.setVariable("ROWS", "2");
    String checkStepName = "Cube Input";
    CubeInputMeta inputMeta = new CubeInputMeta();
    inputMeta.setFilename(tempOutputFile.getAbsolutePath());
    inputMeta.setRowLimit("${ROWS}");
    transMeta = TransTestFactory.generateTestTransformation(varSpace, inputMeta, checkStepName);
    // Remove the Injector hop, as it's not needed for this transformation
    TransHopMeta injectHop = transMeta.findTransHop(transMeta.findStep(TransTestFactory.INJECTOR_STEPNAME), transMeta.findStep(stepName));
    transMeta.removeTransHop(transMeta.indexOfTransHop(injectHop));
    inputList = new ArrayList<RowMetaAndData>();
    List<RowMetaAndData> result = TransTestFactory.executeTestTransformation(transMeta, TransTestFactory.INJECTOR_STEPNAME, stepName, TransTestFactory.DUMMY_STEPNAME, inputList);
    assertNotNull(result);
    assertEquals(2, result.size());
    assertEquals(1, result.get(0).getRowMeta().size());
    assertEquals(ValueMetaInterface.TYPE_STRING, result.get(0).getValueMeta(0).getType());
    assertEquals("col1", result.get(0).getValueMeta(0).getName());
    assertEquals("data1", result.get(0).getString(0, "fail"));
    assertEquals("data2", result.get(1).getString(0, "fail"));
}

Example 64 with TransHopMeta

use of org.pentaho.di.trans.TransHopMeta in project pentaho-kettle by pentaho.

the class DatabaseLookupIT method CacheAndLoadAllRowsDatabaseLookup.

/**
 * Test "Load All Rows" version of BasicDatabaseLookup test.
 */
@Test
public void CacheAndLoadAllRowsDatabaseLookup() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("transname");
    // Add the database connections
    for (int i = 0; i < databasesXML.length; i++) {
        DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
        transMeta.addDatabase(databaseMeta);
    }
    DatabaseMeta dbInfo = transMeta.findDatabase("db");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // create the lookup step...
    // 
    String lookupName = "look up from [" + lookup_table + "]";
    DatabaseLookupMeta dbl = new DatabaseLookupMeta();
    dbl.setDatabaseMeta(transMeta.findDatabase("db"));
    dbl.setTablename(lookup_table);
    dbl.setCached(true);
    dbl.setLoadingAllDataInCache(true);
    dbl.setEatingRowOnLookupFailure(false);
    dbl.setFailingOnMultipleResults(false);
    dbl.setOrderByClause("");
    dbl.setTableKeyField(new String[] { "ID" });
    dbl.setKeyCondition(new String[] { "=" });
    dbl.setStreamKeyField1(new String[] { "int_field" });
    dbl.setStreamKeyField2(new String[] { "" });
    dbl.setReturnValueField(new String[] { "CODE", "STRING" });
    dbl.setReturnValueDefaultType(new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING });
    dbl.setReturnValueDefault(new String[] { "-1", "UNDEF" });
    dbl.setReturnValueNewName(new String[] { "RET_CODE", "RET_STRING" });
    String lookupId = registry.getPluginId(StepPluginType.class, dbl);
    StepMeta lookupStep = new StepMeta(lookupId, lookupName, dbl);
    lookupStep.setDescription("Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]");
    transMeta.addStep(lookupStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, lookupStep);
    transMeta.addTransHop(hi);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(lookupName, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createDataRows();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = rc.getRowsWritten();
    List<RowMetaAndData> goldRows = createResultDataRows();
    checkRows(goldRows, resultRows);
}

Example 65 with TransHopMeta

use of org.pentaho.di.trans.TransHopMeta in project pentaho-kettle by pentaho.

the class DatabaseLookupIT method CacheAndLoadAllRowsDatabaseLookupWithLikePredicate.

/**
 * Test "Load All Rows" version of BasicDatabaseLookup test with Like predicate.
 */
@Test
public void CacheAndLoadAllRowsDatabaseLookupWithLikePredicate() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("transname");
    // Add the database connections
    for (int i = 0; i < databasesXML.length; i++) {
        DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
        transMeta.addDatabase(databaseMeta);
    }
    DatabaseMeta dbInfo = transMeta.findDatabase("db");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // create the lookup step...
    // 
    String lookupName = "look up from [" + lookup_table + "] by 'LIKE'";
    DatabaseLookupMeta dbl = new DatabaseLookupMeta();
    dbl.setDatabaseMeta(transMeta.findDatabase("db"));
    dbl.setTablename(lookup_table);
    dbl.setCached(true);
    dbl.setLoadingAllDataInCache(true);
    dbl.setEatingRowOnLookupFailure(false);
    dbl.setFailingOnMultipleResults(false);
    dbl.setOrderByClause("");
    dbl.setTableKeyField(new String[] { "STRING" });
    dbl.setKeyCondition(new String[] { "LIKE" });
    dbl.setStreamKeyField1(new String[] { "str_field" });
    dbl.setStreamKeyField2(new String[] { "" });
    dbl.setReturnValueField(new String[] { "CODE", "STRING" });
    dbl.setReturnValueDefaultType(new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING });
    dbl.setReturnValueDefault(new String[] { "-1", "UNDEF" });
    dbl.setReturnValueNewName(new String[] { "RET_CODE", "RET_STRING" });
    String lookupId = registry.getPluginId(StepPluginType.class, dbl);
    StepMeta lookupStep = new StepMeta(lookupId, lookupName, dbl);
    lookupStep.setDescription("Reads information from table [" + lookup_table + "] on database [" + dbInfo + "] using LIKE condition");
    transMeta.addStep(lookupStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, lookupStep);
    transMeta.addTransHop(hi);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(lookupName, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createDataRows();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = rc.getRowsWritten();
    List<RowMetaAndData> goldRows = createResultDataRows();
    checkRows(goldRows, resultRows);
}