Examples with RowProducer org.pentaho.di.trans.RowProducer used on opensource projects

Example 51 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class DatabaseLookupIT method basicDatabaseLookup.

/**
 * Basic Test case for database lookup.
 */
@Test
public void basicDatabaseLookup() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("transname");
    // Add the database connections
    for (int i = 0; i < databasesXML.length; i++) {
        DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
        transMeta.addDatabase(databaseMeta);
    }
    DatabaseMeta dbInfo = transMeta.findDatabase("db");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // create the lookup step...
    // 
    String lookupName = "look up from [" + lookup_table + "]";
    DatabaseLookupMeta dbl = new DatabaseLookupMeta();
    dbl.setDatabaseMeta(transMeta.findDatabase("db"));
    dbl.setTablename(lookup_table);
    dbl.setCached(false);
    dbl.setEatingRowOnLookupFailure(false);
    dbl.setFailingOnMultipleResults(false);
    dbl.setOrderByClause("");
    dbl.setTableKeyField(new String[] { "ID" });
    dbl.setKeyCondition(new String[] { "=" });
    dbl.setStreamKeyField1(new String[] { "int_field" });
    dbl.setStreamKeyField2(new String[] { "" });
    dbl.setReturnValueField(new String[] { "CODE", "STRING" });
    dbl.setReturnValueDefaultType(new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING });
    dbl.setReturnValueDefault(new String[] { "-1", "UNDEF" });
    dbl.setReturnValueNewName(new String[] { "RET_CODE", "RET_STRING" });
    String lookupId = registry.getPluginId(StepPluginType.class, dbl);
    StepMeta lookupStep = new StepMeta(lookupId, lookupName, dbl);
    lookupStep.setDescription("Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]");
    transMeta.addStep(lookupStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, lookupStep);
    transMeta.addTransHop(hi);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(lookupName, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createDataRows();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = rc.getRowsWritten();
    List<RowMetaAndData> goldRows = createResultDataRows();
    checkRows(goldRows, resultRows);
}

Example 52 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class DatabaseLookupIT method NOTCachedAndLoadAllRowsDatabaseLookup.

/**
 * Test with cache turned off but "Load All Rows" enabled (Load all rows should have no effect) See JIRA PDI-1910
 */
@Test
public void NOTCachedAndLoadAllRowsDatabaseLookup() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("transname");
    // Add the database connections
    for (int i = 0; i < databasesXML.length; i++) {
        DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
        transMeta.addDatabase(databaseMeta);
    }
    DatabaseMeta dbInfo = transMeta.findDatabase("db");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // create the lookup step...
    // 
    String lookupName = "look up from [" + lookup_table + "]";
    DatabaseLookupMeta dbl = new DatabaseLookupMeta();
    dbl.setDatabaseMeta(transMeta.findDatabase("db"));
    dbl.setTablename(lookup_table);
    dbl.setCached(false);
    dbl.setLoadingAllDataInCache(true);
    dbl.setEatingRowOnLookupFailure(false);
    dbl.setFailingOnMultipleResults(false);
    dbl.setOrderByClause("");
    dbl.setTableKeyField(new String[] { "ID" });
    dbl.setKeyCondition(new String[] { "=" });
    dbl.setStreamKeyField1(new String[] { "int_field" });
    dbl.setStreamKeyField2(new String[] { "" });
    dbl.setReturnValueField(new String[] { "CODE", "STRING" });
    dbl.setReturnValueDefaultType(new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING });
    dbl.setReturnValueDefault(new String[] { "-1", "UNDEF" });
    dbl.setReturnValueNewName(new String[] { "RET_CODE", "RET_STRING" });
    String lookupId = registry.getPluginId(StepPluginType.class, dbl);
    StepMeta lookupStep = new StepMeta(lookupId, lookupName, dbl);
    lookupStep.setDescription("Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]");
    transMeta.addStep(lookupStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, lookupStep);
    transMeta.addTransHop(hi);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(lookupName, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createDataRows();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = rc.getRowsWritten();
    List<RowMetaAndData> goldRows = createResultDataRows();
    checkRows(goldRows, resultRows);
}

Example 53 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class ExecSQLRowIT method testExecSQLRow4.

/**
 * Basic Test case for Exec SQL Row. This tests a commit size of three (i.e. not autocommit but equal to input row
 * size)
 */
@Test
public void testExecSQLRow4() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("transname");
    // Add the database connections
    for (int i = 0; i < databasesXML.length; i++) {
        DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
        transMeta.addDatabase(databaseMeta);
    }
    DatabaseMeta dbInfo = transMeta.findDatabase("db");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // create the Exec SQL Row step...
    // 
    String stepName = "delete from [" + execsqlrow_testtable + "]";
    ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
    execsqlmeta.setDatabaseMeta(transMeta.findDatabase("db"));
    execsqlmeta.setCommitSize(3);
    execsqlmeta.setSqlFieldName("SQL");
    String execSqlRowId = registry.getPluginId(StepPluginType.class, execsqlmeta);
    StepMeta execSqlRowStep = new StepMeta(execSqlRowId, stepName, execsqlmeta);
    execSqlRowStep.setDescription("Deletes information from table [" + execsqlrow_testtable + "] on database [" + dbInfo + "]");
    transMeta.addStep(execSqlRowStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, execSqlRowStep);
    transMeta.addTransHop(hi);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(stepName, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createDataRows();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = rc.getRowsWritten();
    List<RowMetaAndData> goldRows = createResultDataRows();
    checkRows(goldRows, resultRows);
}

Example 54 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class ExecSQLRowIT method testExecSQLRow2.

/**
 * Basic Test case for Exec SQL Row. This tests a commit size of one (i.e. "simulated" autocommit)
 */
@Test
public void testExecSQLRow2() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("transname");
    // Add the database connections
    for (int i = 0; i < databasesXML.length; i++) {
        DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
        transMeta.addDatabase(databaseMeta);
    }
    DatabaseMeta dbInfo = transMeta.findDatabase("db");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // create the Exec SQL Row step...
    // 
    String stepName = "delete from [" + execsqlrow_testtable + "]";
    ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
    execsqlmeta.setDatabaseMeta(transMeta.findDatabase("db"));
    execsqlmeta.setCommitSize(1);
    execsqlmeta.setSqlFieldName("SQL");
    String execSqlRowId = registry.getPluginId(StepPluginType.class, execsqlmeta);
    StepMeta execSqlRowStep = new StepMeta(execSqlRowId, stepName, execsqlmeta);
    execSqlRowStep.setDescription("Deletes information from table [" + execsqlrow_testtable + "] on database [" + dbInfo + "]");
    transMeta.addStep(execSqlRowStep);
    TransHopMeta hi = new TransHopMeta(injectorStep, execSqlRowStep);
    transMeta.addTransHop(hi);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(stepName, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createDataRows();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = rc.getRowsWritten();
    List<RowMetaAndData> goldRows = createResultDataRows();
    checkRows(goldRows, resultRows);
}

Example 55 with RowProducer

use of org.pentaho.di.trans.RowProducer in project pentaho-kettle by pentaho.

the class TextFileInputIT method testTextFileInput1.

public void testTextFileInput1() throws Exception {
    KettleEnvironment.init();
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("testTextFileInput1");
    PluginRegistry registry = PluginRegistry.getInstance();
    // write the data that is to be read in
    // by the step we are testing
    String fileName = writeInputFile(1);
    // create an injector step and add it to the trans meta
    String injectorStepName = "injector step";
    StepMeta injectorStep = TestUtilities.createInjectorStep(injectorStepName, registry);
    transMeta.addStep(injectorStep);
    // Create a Text File Input step
    String testFileInputName = "text file input step";
    StepMeta textFileInputStep = createTextFileInputStep(testFileInputName, fileName, registry);
    transMeta.addStep(textFileInputStep);
    // create a TransHopMeta for textFileInputStep and add it to the transMeta
    TransHopMeta hopInputTextFile = new TransHopMeta(injectorStep, textFileInputStep);
    transMeta.addTransHop(hopInputTextFile);
    // Create a dummy step 1 and add it to the tranMeta
    String dummyStepName = "dummy step";
    StepMeta dummyStep = TestUtilities.createDummyStep(dummyStepName, registry);
    transMeta.addStep(dummyStep);
    // create transHopMeta for the hop from text file input to the dummy step
    TransHopMeta hop_textFileInputStep_dummyStep = new TransHopMeta(textFileInputStep, dummyStep);
    transMeta.addTransHop(hop_textFileInputStep_dummyStep);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    // create a row collector and add it to a row listener for the dummy step
    StepInterface si = trans.getStepInterface(dummyStepName, 0);
    RowStepCollector dummyRowCollector = new RowStepCollector();
    si.addRowListener(dummyRowCollector);
    // Create a row producer for trans
    RowProducer rowProducer = trans.addRowProducer(injectorStepName, 0);
    trans.startThreads();
    // create the filename rows
    List<RowMetaAndData> inputList = createData(fileName);
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rowMetaAndData = it.next();
        rowProducer.putRow(rowMetaAndData.getRowMeta(), rowMetaAndData.getData());
    }
    rowProducer.finished();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRowCollector.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData1();
    try {
        TestUtilities.checkRows(goldenImageRows, resultRows, 5);
    } catch (TestFailedException tfe) {
        fail(tfe.getMessage());
    }
}