Examples with InjectorMeta org.pentaho.di.trans.steps.injector.InjectorMeta used on opensource projects

Example 46 with InjectorMeta

use of org.pentaho.di.trans.steps.injector.InjectorMeta in project pentaho-kettle by pentaho.

the class HopIT method testDefaultConfiguration.

/**
 * Test case for hop use.
 *
 * The transformation is as follows: an injector step links to a dummy step, which in turn links to 2 target dummy
 * steps.
 *
 * The default in the GUI of spoon is copy mode, but here it seems to be distribute.
 */
public void testDefaultConfiguration() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("hop test default");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep);
    transMeta.addTransHop(hi);
    // 
    // Create a dummy target step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hop1 = new TransHopMeta(dummyStep, dummyStep1);
    transMeta.addTransHop(hop1);
    // 
    // Create a dummy target step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hop2 = new TransHopMeta(dummyStep, dummyStep2);
    transMeta.addTransHop(hop2);
    // THIS DETERMINES THE COPY OR DISTRIBUTE BEHAVIOUR
    dummyStep.setDistributes(true);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si1 = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector rc1 = new RowStepCollector();
    si1.addRowListener(rc1);
    StepInterface si2 = trans.getStepInterface(dummyStepname2, 0);
    RowStepCollector rc2 = new RowStepCollector();
    si2.addRowListener(rc2);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> compareList1 = new ArrayList<RowMetaAndData>();
    List<RowMetaAndData> compareList2 = new ArrayList<RowMetaAndData>();
    int counter = 1;
    List<RowMetaAndData> inputList = createData();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
        if (counter % 2 == 0) {
            compareList2.add(rm);
        } else {
            compareList1.add(rm);
        }
        counter++;
    }
    rp.finished();
    trans.waitUntilFinished();
    // Dummy1 should get 4 rows: 1 3 5 7
    // Dummy2 should get 3 rows: 2 4 6
    List<RowMetaAndData> resultRows1 = rc1.getRowsWritten();
    checkRows(resultRows1, compareList1);
    List<RowMetaAndData> resultRows2 = rc2.getRowsWritten();
    checkRows(resultRows2, compareList2);
}

Example 47 with InjectorMeta

use of org.pentaho.di.trans.steps.injector.InjectorMeta in project pentaho-kettle by pentaho.

the class HopIT method testDistributeHops.

/**
 * Test case for hop use.
 *
 * The transformation is as follows: an injector step links to a dummy step, which in turn links to 2 target dummy
 * steps.
 *
 * This testcase uses distribute mode, so each hop in turn should get a row.
 */
public void testDistributeHops() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("hop test default");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep);
    transMeta.addTransHop(hi);
    // 
    // Create a dummy target step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hop1 = new TransHopMeta(dummyStep, dummyStep1);
    transMeta.addTransHop(hop1);
    // 
    // Create a dummy target step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hop2 = new TransHopMeta(dummyStep, dummyStep2);
    transMeta.addTransHop(hop2);
    // THIS DETERMINES THE COPY OR DISTRIBUTE BEHAVIOUR
    dummyStep.setDistributes(true);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si1 = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector rc1 = new RowStepCollector();
    si1.addRowListener(rc1);
    StepInterface si2 = trans.getStepInterface(dummyStepname2, 0);
    RowStepCollector rc2 = new RowStepCollector();
    si2.addRowListener(rc2);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> compareList1 = new ArrayList<RowMetaAndData>();
    List<RowMetaAndData> compareList2 = new ArrayList<RowMetaAndData>();
    int counter = 1;
    List<RowMetaAndData> inputList = createData();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
        if (counter % 2 == 0) {
            compareList2.add(rm);
        } else {
            compareList1.add(rm);
        }
        counter++;
    }
    rp.finished();
    trans.waitUntilFinished();
    // Dummy1 should get 4 rows: 1 3 5 7
    // Dummy2 should get 3 rows: 2 4 6
    List<RowMetaAndData> resultRows1 = rc1.getRowsWritten();
    checkRows(resultRows1, compareList1);
    List<RowMetaAndData> resultRows2 = rc2.getRowsWritten();
    checkRows(resultRows2, compareList2);
}

Example 48 with InjectorMeta

use of org.pentaho.di.trans.steps.injector.InjectorMeta in project pentaho-kettle by pentaho.

the class BlockingStepIT method testBlockingStepPassAll.

/**
 * Test case for blocking step step passing all rows. Injector step to a blocking step to a dummy step. rows go in,
 * all rows should be output.
 */
public void testBlockingStepPassAll() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("blockingsteptest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a blocking step
    // 
    String blockingStepname = "blocking step";
    BlockingStepMeta bm = new BlockingStepMeta();
    bm.setPassAllRows(true);
    String blockingStepPid = registry.getPluginId(StepPluginType.class, bm);
    StepMeta blockingStep = new StepMeta(blockingStepPid, blockingStepname, bm);
    transMeta.addStep(blockingStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, blockingStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(blockingStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(blockingStepname, 0);
    RowStepCollector blockingRc = new RowStepCollector();
    si.addRowListener(blockingRc);
    si = trans.getStepInterface(dummyStepname2, 0);
    RowStepCollector dummyRc2 = new RowStepCollector();
    si.addRowListener(dummyRc2);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // The results should be that dummy1 gets all rows.
    // blocking step should receive all rows (but only send the
    // last one through). dummy2 should only get the last row.
    List<RowMetaAndData> resultRows1 = dummyRc1.getRowsWritten();
    checkRows(resultRows1, inputList);
    List<RowMetaAndData> resultRows2 = blockingRc.getRowsWritten();
    checkRows(resultRows2, inputList);
    List<RowMetaAndData> resultRows3 = dummyRc2.getRowsWritten();
    checkRows(resultRows3, inputList);
}

Example 49 with InjectorMeta

use of org.pentaho.di.trans.steps.injector.InjectorMeta in project pentaho-kettle by pentaho.

the class DatabaseLookupIT method basicDatabaseLookup.

/**
 * Basic Test case for database lookup.
 */
@Test
public void basicDatabaseLookup() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("transname");
    // Add the database connections
    for (int i = 0; i < databasesXML.length; i++) {
        DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
        transMeta.addDatabase(databaseMeta);
    }
    DatabaseMeta dbInfo = transMeta.findDatabase("db");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // create the lookup step...
    // 
    String lookupName = "look up from [" + lookup_table + "]";
    DatabaseLookupMeta dbl = new DatabaseLookupMeta();
    dbl.setDatabaseMeta(transMeta.findDatabase("db"));
    dbl.setTablename(lookup_table);
    dbl.setCached(false);
    dbl.setEatingRowOnLookupFailure(false);
    dbl.setFailingOnMultipleResults(false);
    dbl.setOrderByClause("");
    dbl.setTableKeyField(new String[] { "ID" });
    dbl.setKeyCondition(new String[] { "=" });
    dbl.setStreamKeyField1(new String[] { "int_field" });
    dbl.setStreamKeyField2(new String[] { "" });
    dbl.setReturnValueField(new String[] { "CODE", "STRING" });
    dbl.setReturnValueDefaultType(new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING });
    dbl.setReturnValueDefault(new String[] { "-1", "UNDEF" });
    dbl.setReturnValueNewName(new String[] { "RET_CODE", "RET_STRING" });
    String lookupId = registry.getPluginId(StepPluginType.class, dbl);
    StepMeta lookupStep = new StepMeta(lookupId, lookupName, dbl);
    lookupStep.setDescription("Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]");
    transMeta.addStep(lookupStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, lookupStep);
    transMeta.addTransHop(hi);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(lookupName, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createDataRows();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = rc.getRowsWritten();
    List<RowMetaAndData> goldRows = createResultDataRows();
    checkRows(goldRows, resultRows);
}

Example 50 with InjectorMeta

use of org.pentaho.di.trans.steps.injector.InjectorMeta in project pentaho-kettle by pentaho.

the class DatabaseLookupIT method NOTCachedAndLoadAllRowsDatabaseLookup.

/**
 * Test with cache turned off but "Load All Rows" enabled (Load all rows should have no effect) See JIRA PDI-1910
 */
@Test
public void NOTCachedAndLoadAllRowsDatabaseLookup() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("transname");
    // Add the database connections
    for (int i = 0; i < databasesXML.length; i++) {
        DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
        transMeta.addDatabase(databaseMeta);
    }
    DatabaseMeta dbInfo = transMeta.findDatabase("db");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // create the lookup step...
    // 
    String lookupName = "look up from [" + lookup_table + "]";
    DatabaseLookupMeta dbl = new DatabaseLookupMeta();
    dbl.setDatabaseMeta(transMeta.findDatabase("db"));
    dbl.setTablename(lookup_table);
    dbl.setCached(false);
    dbl.setLoadingAllDataInCache(true);
    dbl.setEatingRowOnLookupFailure(false);
    dbl.setFailingOnMultipleResults(false);
    dbl.setOrderByClause("");
    dbl.setTableKeyField(new String[] { "ID" });
    dbl.setKeyCondition(new String[] { "=" });
    dbl.setStreamKeyField1(new String[] { "int_field" });
    dbl.setStreamKeyField2(new String[] { "" });
    dbl.setReturnValueField(new String[] { "CODE", "STRING" });
    dbl.setReturnValueDefaultType(new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING });
    dbl.setReturnValueDefault(new String[] { "-1", "UNDEF" });
    dbl.setReturnValueNewName(new String[] { "RET_CODE", "RET_STRING" });
    String lookupId = registry.getPluginId(StepPluginType.class, dbl);
    StepMeta lookupStep = new StepMeta(lookupId, lookupName, dbl);
    lookupStep.setDescription("Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]");
    transMeta.addStep(lookupStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, lookupStep);
    transMeta.addTransHop(hi);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(lookupName, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createDataRows();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = rc.getRowsWritten();
    List<RowMetaAndData> goldRows = createResultDataRows();
    checkRows(goldRows, resultRows);
}