Examples with PluginRegistry org.pentaho.di.core.plugins.PluginRegistry used on opensource projects

Example 21 with PluginRegistry

use of org.pentaho.di.core.plugins.PluginRegistry in project pentaho-kettle by pentaho.

the class UniqueRowsIT method testAllUnique.

@Test
public void testAllUnique() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("uniquerowstest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a unique rows step
    // 
    String uniqueRowsStepname = "unique rows step";
    UniqueRowsMeta urm = new UniqueRowsMeta();
    urm.setCompareFields(new String[] { "KEY" });
    urm.setCaseInsensitive(new boolean[] { true });
    urm.setRejectDuplicateRow(true);
    String uniqueRowsStepPid = registry.getPluginId(StepPluginType.class, urm);
    StepMeta uniqueRowsStep = new StepMeta(uniqueRowsStepPid, uniqueRowsStepname, urm);
    uniqueRowsStep.setDistributes(false);
    transMeta.addStep(uniqueRowsStep);
    transMeta.addTransHop(new TransHopMeta(injectorStep, uniqueRowsStep));
    // 
    // Create a dummy step to receive the unique rows
    // 
    String dummyStepname1 = "dummy step";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    transMeta.addTransHop(new TransHopMeta(uniqueRowsStep, dummyStep1));
    // 
    // Create a dummy step to receive the duplicate rows (errors)
    // 
    String dummyStepname2 = "dummy step2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    // Set up error (aka duplicates) handling info
    StepErrorMeta stepErrorMeta = new StepErrorMeta(new Variables(), uniqueRowsStep);
    stepErrorMeta.setTargetStep(dummyStep2);
    stepErrorMeta.setEnabled(true);
    stepErrorMeta.setNrErrorsValuename("numErrors");
    stepErrorMeta.setErrorDescriptionsValuename("duplicates");
    stepErrorMeta.setErrorFieldsValuename("KEY");
    stepErrorMeta.setErrorCodesValuename("errorCodes");
    stepErrorMeta.setMaxErrors("9999");
    stepErrorMeta.setMaxPercentErrors("");
    stepErrorMeta.setMinPercentRows("");
    uniqueRowsStep.setStepErrorMeta(stepErrorMeta);
    transMeta.addTransHop(new TransHopMeta(uniqueRowsStep, dummyStep2));
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    StepInterface si2 = trans.getStepInterface(dummyStepname2, 0);
    RowStepCollector dummyRc2 = new RowStepCollector();
    si2.addRowListener(dummyRc2);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createDataAllUnique();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
    checkRows(createResultDataAllUnique(), resultRows);
    List<RowMetaAndData> errorRows = dummyRc2.getRowsWritten();
    // There should be no duplicates for this test
    assertEquals(errorRows.size(), 0);
}

Example 22 with PluginRegistry

use of org.pentaho.di.core.plugins.PluginRegistry in project pentaho-kettle by pentaho.

the class ValueMapperIT method testValueMapper2.

/**
 * Test case for valuemapper step. Injector step to a valuemapper step to a dummy step. Rows go in and should be
 * mapped accordingly.
 *
 * This test will write the mappings in the same field.
 */
public void testValueMapper2() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("valuemappertest2");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a step
    // 
    String valueMapperName = "valuemapper step";
    ValueMapperMeta vm = new ValueMapperMeta();
    vm.setFieldToUse("field1");
    vm.setTargetField("");
    vm.setSourceValue(new String[] { "abc", "ABC", "def", "ghij" });
    vm.setTargetValue(new String[] { "begin", "BEG_LONGER", "test", "" });
    String valueMapperPid = registry.getPluginId(StepPluginType.class, vm);
    StepMeta valueMapperStep = new StepMeta(valueMapperPid, valueMapperName, vm);
    transMeta.addStep(valueMapperStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, valueMapperStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(valueMapperStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(valueMapperName, 0);
    RowStepCollector valueMapperRc = new RowStepCollector();
    si.addRowListener(valueMapperRc);
    si = trans.getStepInterface(dummyStepname2, 0);
    RowStepCollector dummyRc = new RowStepCollector();
    si.addRowListener(dummyRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData2();
    checkRows(goldenImageRows, resultRows);
}

Example 23 with PluginRegistry

use of org.pentaho.di.core.plugins.PluginRegistry in project pentaho-kettle by pentaho.

the class ValueMapperIT method testValueMapper6.

/**
 * Test case for valuemapper step. Injector step to a valuemapper step to a dummy step. Rows go in and should be
 * mapped accordingly.
 *
 * This test will explicitly test the empty field processing. using a non matching default.
 */
public void testValueMapper6() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("valuemappertest6");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a ValueMapper step
    // 
    String valueMapperName = "valuemapper step";
    ValueMapperMeta vm = new ValueMapperMeta();
    vm.setFieldToUse("field1");
    vm.setTargetField("");
    vm.setNonMatchDefault("default");
    vm.setSourceValue(new String[] { "abc", "ABC", "def", "ghijk", null });
    vm.setTargetValue(new String[] { "begin", "BEG_LONGER", "test", "blah", "emptyField" });
    String valueMapperPid = registry.getPluginId(StepPluginType.class, vm);
    StepMeta valueMapperStep = new StepMeta(valueMapperPid, valueMapperName, vm);
    transMeta.addStep(valueMapperStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, valueMapperStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(valueMapperStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    boolean prepare;
    try {
        trans.prepareExecution(null);
        prepare = true;
    } catch (KettleException e) {
        prepare = false;
    }
    assertTrue(prepare);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(valueMapperName, 0);
    RowStepCollector valueMapperRc = new RowStepCollector();
    si.addRowListener(valueMapperRc);
    si = trans.getStepInterface(dummyStepname2, 0);
    RowStepCollector dummyRc = new RowStepCollector();
    si.addRowListener(dummyRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData6();
    checkRows(goldenImageRows, resultRows);
}

Example 24 with PluginRegistry

use of org.pentaho.di.core.plugins.PluginRegistry in project pentaho-kettle by pentaho.

the class ValueMapperIT method testValueMapper4.

/*----------------- TODO */
/**
 * Test case for valuemapper step. Injector step to a valuemapper step to a dummy step. Rows go in and should be
 * mapped accordingly.
 *
 * This test will write the mappings in a new field, using a non matching default.
 */
public void testValueMapper4() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("valuemappertest4");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a dummy step 1
    // 
    String dummyStepname1 = "dummy step 1";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep1);
    transMeta.addTransHop(hi);
    // 
    // Create a step
    // 
    String valueMapperName = "valuemapper step";
    ValueMapperMeta vm = new ValueMapperMeta();
    vm.setFieldToUse("field1");
    vm.setTargetField("new_field");
    vm.setNonMatchDefault("default");
    vm.setSourceValue(new String[] { "abc", "ABC", "def", "ghij" });
    vm.setTargetValue(new String[] { "begin", "BEG_LONGER", "test", "" });
    String valueMapperPid = registry.getPluginId(StepPluginType.class, vm);
    StepMeta valueMapperStep = new StepMeta(valueMapperPid, valueMapperName, vm);
    transMeta.addStep(valueMapperStep);
    TransHopMeta hi2 = new TransHopMeta(dummyStep1, valueMapperStep);
    transMeta.addTransHop(hi2);
    // 
    // Create a dummy step 2
    // 
    String dummyStepname2 = "dummy step 2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    TransHopMeta hi3 = new TransHopMeta(valueMapperStep, dummyStep2);
    transMeta.addTransHop(hi3);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    si = trans.getStepInterface(valueMapperName, 0);
    RowStepCollector valueMapperRc = new RowStepCollector();
    si.addRowListener(valueMapperRc);
    si = trans.getStepInterface(dummyStepname2, 0);
    RowStepCollector dummyRc = new RowStepCollector();
    si.addRowListener(dummyRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    Iterator<RowMetaAndData> it = inputList.iterator();
    while (it.hasNext()) {
        RowMetaAndData rm = it.next();
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    // Compare the results
    List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
    List<RowMetaAndData> goldenImageRows = createResultData4();
    checkRows(goldenImageRows, resultRows);
}

Example 25 with PluginRegistry

use of org.pentaho.di.core.plugins.PluginRegistry in project pentaho-kettle by pentaho.

the class TableInputIT method testTableInputWithParam.

/**
 * Test case for table input which is taking its input from a hop. This is a regression test case for JIRA PDI-588.
 *
 * The query in the table input step has one '?' and this parameter is filled by values read from an input hop.
 */
public void testTableInputWithParam() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("transname");
    // Add the database connections
    for (int i = 0; i < databasesXML.length; i++) {
        DatabaseMeta databaseMeta = new DatabaseMeta(databasesXML[i]);
        transMeta.addDatabase(databaseMeta);
    }
    DatabaseMeta dbInfo = transMeta.findDatabase("db");
    // Execute our setup SQLs in the database.
    Database database = new Database(transMeta, dbInfo);
    database.connect();
    createTables(database);
    createData(database);
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // create the source step...
    // 
    String fromstepname = "read from [" + source_table + "]";
    TableInputMeta tii = new TableInputMeta();
    tii.setDatabaseMeta(transMeta.findDatabase("db"));
    tii.setLookupFromStep(injectorStep);
    tii.setExecuteEachInputRow(true);
    String selectSQL = "SELECT " + Const.CR;
    selectSQL += "ID, CODE ";
    selectSQL += "FROM " + source_table + " WHERE CODE = ? ORDER BY ID, CODE;";
    tii.setSQL(selectSQL);
    String fromstepid = registry.getPluginId(StepPluginType.class, tii);
    StepMeta fromstep = new StepMeta(fromstepid, fromstepname, tii);
    fromstep.setDescription("Reads information from table [" + source_table + "] on database [" + dbInfo + "]");
    transMeta.addStep(fromstep);
    TransHopMeta hi = new TransHopMeta(injectorStep, fromstep);
    transMeta.addTransHop(hi);
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(fromstepname, 0);
    RowStepCollector rc = new RowStepCollector();
    si.addRowListener(rc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createDataRows();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = rc.getRowsWritten();
    List<RowMetaAndData> goldRows = createResultDataRows();
    checkRows(goldRows, resultRows);
}