Examples with TransHopMeta org.pentaho.di.trans.TransHopMeta used on opensource projects

Example 51 with TransHopMeta

use of org.pentaho.di.trans.TransHopMeta in project pentaho-kettle by pentaho.

the class InsertUpdateIT method setUp.

@Override
@Before
public void setUp() throws Exception {
    KettleEnvironment.init();
    /* SET UP TRANSFORMATION */
    // Create a new transformation...
    TransMeta transMeta = new TransMeta();
    transMeta.setName("insert/update test");
    // Add the database connections
    for (int i = 0; i < databasesXML.length; i++) {
        DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
        transMeta.addDatabase(databaseMeta);
    }
    DatabaseMeta dbInfo = transMeta.findDatabase("db");
    /* SET UP DATABASE */
    // Create target table
    db = new Database(transMeta, dbInfo);
    db.connect();
    String source = db.getCreateTableStatement(TARGET_TABLE, getTargetTableRowMeta(), null, false, null, true);
    db.execStatement(source);
    // populate target table
    for (String sql : insertStatement) {
        db.execStatement(sql);
    }
    /* SET UP TRANSFORMATION STEPS */
    PluginRegistry registry = PluginRegistry.getInstance();
    // create an injector step...
    String injectorStepName = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepName, im);
    transMeta.addStep(injectorStep);
    // create the update step...
    String updateStepName = "insert/update [" + TARGET_TABLE + "]";
    insupd = new InsertUpdateMeta();
    insupd.setDatabaseMeta(transMeta.findDatabase("db"));
    insupd.setTableName(TARGET_TABLE);
    insupd.setUpdateLookup(new String[] { "VALUE", "ROW_ORDER" });
    insupd.setUpdateStream(new String[] { "VALUE", "ROW_ORDER" });
    insupd.setUpdate(new Boolean[] { true, false });
    String fromid = registry.getPluginId(StepPluginType.class, insupd);
    StepMeta updateStep = new StepMeta(fromid, updateStepName, insupd);
    updateStep.setDescription("insert/update data in table [" + TARGET_TABLE + "] on database [" + dbInfo + "]");
    transMeta.addStep(updateStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, updateStep);
    transMeta.addTransHop(hi);
    /* PREPARE TRANSFORMATION EXECUTION */
    trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(updateStepName, 0);
    rc = new RowStepCollector();
    si.addRowListener(rc);
    rp = trans.addRowProducer(injectorStepName, 0);
}

Example 52 with TransHopMeta

use of org.pentaho.di.trans.TransHopMeta in project pentaho-kettle by pentaho.

the class JaninoStepIT method testJaninoStep.

/**
 * Test case for janino step.
 */
@Test
public void testJaninoStep() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("janino test");
    PluginRegistry registry = PluginRegistry.getInstance();
    // create an injector step...
    String injectorStepName = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepName, im);
    transMeta.addStep(injectorStep);
    // 
    // create a janino step...
    // 
    String stepname = "janino";
    JaninoMeta jm = new JaninoMeta();
    // Set the information of the step
    String janinoPid = registry.getPluginId(StepPluginType.class, jm);
    StepMeta janinoStep = new StepMeta(janinoPid, stepname, jm);
    transMeta.addStep(janinoStep);
    jm.setDefault();
    JaninoMetaFunction[] formulas = { new JaninoMetaFunction("string", "(string==null)?null:\"string-value\"", ValueMetaInterface.TYPE_STRING, -1, -1, "string"), new JaninoMetaFunction("integer", "(integer==null)?null:new Long(42L)", ValueMetaInterface.TYPE_INTEGER, -1, -1, "integer"), new JaninoMetaFunction("number", "(number==null)?null:new Double(23.0)", ValueMetaInterface.TYPE_NUMBER, -1, -1, "number"), new JaninoMetaFunction("bigdecimal", "(bigdecimal==null)?null:new java.math.BigDecimal(11.0)", ValueMetaInterface.TYPE_BIGNUMBER, -1, -1, "bigdecimal"), new JaninoMetaFunction("date", "(date==null)?null:new java.util.Date(10000000)", ValueMetaInterface.TYPE_DATE, -1, -1, "date"), new JaninoMetaFunction("binary", "(binary==null)?null:new byte[]{1,2,3,4,5}", ValueMetaInterface.TYPE_BINARY, -1, -1, "binary"), new JaninoMetaFunction("bool", "(bool==null)?null:Boolean.TRUE", ValueMetaInterface.TYPE_BOOLEAN, -1, -1, "bool"), new JaninoMetaFunction("timestamp", "(timestamp==null)?null:new java.sql.Timestamp(0L)", ValueMetaInterface.TYPE_TIMESTAMP, -1, -1, "timestamp"), new JaninoMetaFunction("inetaddress", "(inetaddress==null)?null:java.net.InetAddress.getByAddress( new byte[]{ 127, 0, 0, 1} )", ValueMetaInterface.TYPE_INET, -1, -1, "inetaddress") };
    jm.setFormula(formulas);
    transMeta.addTransHop(new TransHopMeta(injectorStep, janinoStep));
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    TransHopMeta hi = new TransHopMeta(janinoStep, dummyStep);
    transMeta.addTransHop(hi);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer(injectorStepName, 0);
    trans.startThreads();
    for (RowMetaAndData rm : createInputList()) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> checkList = createExpectedList();
    List<RowMetaAndData> resultRows = rc.getRowsWritten();
    checkRows(resultRows, checkList);
}

Example 53 with TransHopMeta

use of org.pentaho.di.trans.TransHopMeta in project pentaho-kettle by pentaho.

the class MappingIT method testInfoStreams_with_main_data_path.

/**
 * Tests that an input step that is a main data path is not flagged as an info stream
 */
public void testInfoStreams_with_main_data_path() throws Exception {
    KettleEnvironment.init();
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // Create a new transformation with a row generator that feeds a Mapping (Sub-Transformation) Step
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("Mapping Info Test");
    StepMeta rowGenerator = buildRowGeneratorStep(registry, "Generate Rows");
    transMeta.addStep(rowGenerator);
    StepMeta rowGeneratorMain = buildRowGeneratorStep(registry, "Generate Rows Main");
    transMeta.addStep(rowGeneratorMain);
    String mappingName = "mapping";
    MappingMeta mappingMeta = new MappingMeta();
    mappingMeta.setSpecificationMethod(ObjectLocationSpecificationMethod.FILENAME);
    mappingMeta.setFileName("test/org/pentaho/di/trans/steps/mapping/subtrans.ktr");
    List<MappingIODefinition> inputMappings = new ArrayList<MappingIODefinition>();
    String mappingInputStepName = "input";
    inputMappings.add(createMappingDef(rowGenerator.getName(), mappingInputStepName, "string", "a"));
    // Create the main data path mapping
    MappingIODefinition mainMappingDef = createMappingDef(rowGeneratorMain.getName(), mappingInputStepName, "string", "a");
    mainMappingDef.setMainDataPath(true);
    inputMappings.add(mainMappingDef);
    mappingMeta.setInputMappings(inputMappings);
    String mappingPid = registry.getPluginId(StepPluginType.class, mappingMeta);
    StepMeta mapping = new StepMeta(mappingPid, mappingName, mappingMeta);
    transMeta.addStep(mapping);
    TransHopMeta hopGeneratorToMapping = new TransHopMeta(rowGenerator, mapping);
    transMeta.addTransHop(hopGeneratorToMapping);
    hopGeneratorToMapping = new TransHopMeta(rowGeneratorMain, mapping);
    transMeta.addTransHop(hopGeneratorToMapping);
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    // (Copied from TransMeta.loadXML())
    for (int i = 0; i < transMeta.nrSteps(); i++) {
        StepMeta stepMeta = transMeta.getStep(i);
        StepMetaInterface sii = stepMeta.getStepMetaInterface();
        if (sii != null) {
            sii.searchInfoAndTargetSteps(transMeta.getSteps());
        }
    }
    // Verify the transformation was configured properly
    assertEquals("Transformation not initialized properly", 3, transMeta.nrSteps());
    StepMeta meta = transMeta.getStep(2);
    assertTrue("Transformation not initialized properly", meta.getStepMetaInterface() instanceof MappingMeta);
    MappingMeta loadedMappingMeta = (MappingMeta) meta.getStepMetaInterface();
    assertEquals("Expected a two input mapping definition", 2, loadedMappingMeta.getInputMappings().size());
    StepIOMetaInterface ioMeta = loadedMappingMeta.getStepIOMeta();
    assertEquals("Expected a single Info Stream", 1, ioMeta.getInfoStreams().size());
    assertEquals("Expected a single Info Step", 1, loadedMappingMeta.getInfoSteps().length);
}

Example 54 with TransHopMeta

use of org.pentaho.di.trans.TransHopMeta in project pentaho-kettle by pentaho.

the class MappingIT method testInfoStreams_single.

/**
 * Tests that info steps are correctly identified via StepMetaInterface.getStepIOMeta()
 */
public void testInfoStreams_single() throws Exception {
    KettleEnvironment.init();
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // Create a new transformation with a row generator that feeds a Mapping (Sub-Transformation) Step
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("Mapping Info Test");
    StepMeta rowGenerator = buildRowGeneratorStep(registry, "Generate Rows");
    transMeta.addStep(rowGenerator);
    String mappingName = "mapping";
    MappingMeta mappingMeta = new MappingMeta();
    mappingMeta.setSpecificationMethod(ObjectLocationSpecificationMethod.FILENAME);
    mappingMeta.setFileName("test/org/pentaho/di/trans/steps/mapping/subtrans.ktr");
    String mappingInputStepName = "input";
    mappingMeta.setInputMappings(Collections.singletonList(createMappingDef(rowGenerator.getName(), mappingInputStepName, "string", "a")));
    String mappingPid = registry.getPluginId(StepPluginType.class, mappingMeta);
    StepMeta mapping = new StepMeta(mappingPid, mappingName, mappingMeta);
    transMeta.addStep(mapping);
    TransHopMeta hopGeneratorToMapping = new TransHopMeta(rowGenerator, mapping);
    transMeta.addTransHop(hopGeneratorToMapping);
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    // (Copied from TransMeta.loadXML())
    for (int i = 0; i < transMeta.nrSteps(); i++) {
        StepMeta stepMeta = transMeta.getStep(i);
        StepMetaInterface sii = stepMeta.getStepMetaInterface();
        if (sii != null) {
            sii.searchInfoAndTargetSteps(transMeta.getSteps());
        }
    }
    // Verify the transformation was configured properly
    assertEquals("Transformation not initialized properly", 2, transMeta.nrSteps());
    StepMeta meta = transMeta.getStep(1);
    assertTrue("Transformation not initialized properly", meta.getStepMetaInterface() instanceof MappingMeta);
    MappingMeta loadedMappingMeta = (MappingMeta) meta.getStepMetaInterface();
    assertEquals("Expected a single input mapping definition", 1, loadedMappingMeta.getInputMappings().size());
    StepIOMetaInterface ioMeta = loadedMappingMeta.getStepIOMeta();
    assertEquals("Expected a single Info Stream", 1, ioMeta.getInfoStreams().size());
    assertEquals("Expected a single Info Step", 1, loadedMappingMeta.getInfoSteps().length);
    // Verify the transformation can be executed
    StepInterface si = trans.getStepInterface(mappingName, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    trans.startThreads();
    trans.waitUntilFinished();
    assertEquals(1, rc.getRowsRead().size());
    assertEquals(1, rc.getRowsWritten().size());
}

Example 55 with TransHopMeta

use of org.pentaho.di.trans.TransHopMeta in project pentaho-kettle by pentaho.

the class MergeRowsIT method createRowGenerator.

void createRowGenerator(TransMeta transMeta, PluginRegistry registry, String stepName, String[] values, StepMeta mergeRowsStep, MergeRowsMeta mergeRowsMeta, int index) {
    RowGeneratorMeta rowGeneratorMeta = new RowGeneratorMeta();
    String rowGeneratorPid = registry.getPluginId(StepPluginType.class, rowGeneratorMeta);
    StepMeta rowGeneratorStep = new StepMeta(rowGeneratorPid, stepName, rowGeneratorMeta);
    transMeta.addStep(rowGeneratorStep);
    rowGeneratorMeta.setDefault();
    rowGeneratorMeta.setFieldName(fieldName);
    rowGeneratorMeta.setFieldType(type);
    rowGeneratorMeta.setFieldLength(intDummies);
    rowGeneratorMeta.setFieldPrecision(intDummies);
    rowGeneratorMeta.setRowLimit("1");
    rowGeneratorMeta.setFieldFormat(fieldFormat);
    rowGeneratorMeta.setGroup(group);
    rowGeneratorMeta.setDecimal(decimal);
    rowGeneratorMeta.setCurrency(currency);
    rowGeneratorMeta.setEmptyString(setEmptystring);
    rowGeneratorMeta.setValue(values);
    TransHopMeta hi1 = new TransHopMeta(rowGeneratorStep, mergeRowsStep);
    transMeta.addTransHop(hi1);
    List<StreamInterface> infoStreams = mergeRowsMeta.getStepIOMeta().getInfoStreams();
    StreamInterface infoStream = infoStreams.get(index);
    infoStream.setStepMeta(transMeta.findStep(stepName));
}