Examples with UniqueRowsMeta org.pentaho.di.trans.steps.uniquerows.UniqueRowsMeta used on opensource projects

Example 1 with UniqueRowsMeta

use of org.pentaho.di.trans.steps.uniquerows.UniqueRowsMeta in project pentaho-kettle by pentaho.

the class UniqueRowsIT method testSortCaseSensitiveUniqueCaseSensitive.

public void testSortCaseSensitiveUniqueCaseSensitive() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("uniquerowstest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a sort rows step
    // 
    String sortRowsStepname = "sort rows step";
    SortRowsMeta srm = new SortRowsMeta();
    srm.setFieldName(new String[] { "KEY" });
    srm.setAscending(new boolean[] { true });
    srm.setCaseSensitive(new boolean[] { true });
    srm.setPreSortedField(new boolean[] { false });
    srm.setPrefix("SortRowsTest");
    srm.setDirectory(".");
    String sortRowsStepPid = registry.getPluginId(StepPluginType.class, srm);
    StepMeta sortRowsStep = new StepMeta(sortRowsStepPid, sortRowsStepname, srm);
    transMeta.addStep(sortRowsStep);
    transMeta.addTransHop(new TransHopMeta(injectorStep, sortRowsStep));
    // 
    // Create a unique rows step
    // 
    String uniqueRowsStepname = "unique rows step";
    UniqueRowsMeta urm = new UniqueRowsMeta();
    urm.setCompareFields(new String[] { "KEY" });
    urm.setCaseInsensitive(new boolean[] { false });
    String uniqueRowsStepPid = registry.getPluginId(StepPluginType.class, urm);
    StepMeta uniqueRowsStep = new StepMeta(uniqueRowsStepPid, uniqueRowsStepname, urm);
    transMeta.addStep(uniqueRowsStep);
    transMeta.addTransHop(new TransHopMeta(sortRowsStep, uniqueRowsStep));
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    transMeta.addTransHop(new TransHopMeta(uniqueRowsStep, dummyStep));
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname, 0);
    RowStepCollector dummyRc = new RowStepCollector();
    si.addRowListener(dummyRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
    checkRows(createResultDataSortCaseSensitiveUniqueCaseSensitive(), resultRows);
}

Example 2 with UniqueRowsMeta

use of org.pentaho.di.trans.steps.uniquerows.UniqueRowsMeta in project pentaho-kettle by pentaho.

the class UniqueRowsIT method testSortCaseSensitiveUniqueCaseInsensitive.

public void testSortCaseSensitiveUniqueCaseInsensitive() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("uniquerowstest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a sort rows step
    // 
    String sortRowsStepname = "sort rows step";
    SortRowsMeta srm = new SortRowsMeta();
    srm.setFieldName(new String[] { "KEY" });
    srm.setAscending(new boolean[] { true });
    srm.setCaseSensitive(new boolean[] { true });
    srm.setPreSortedField(new boolean[] { false });
    srm.setPrefix("SortRowsTest");
    srm.setDirectory(".");
    String sortRowsStepPid = registry.getPluginId(StepPluginType.class, srm);
    StepMeta sortRowsStep = new StepMeta(sortRowsStepPid, sortRowsStepname, srm);
    transMeta.addStep(sortRowsStep);
    transMeta.addTransHop(new TransHopMeta(injectorStep, sortRowsStep));
    // 
    // Create a unique rows step
    // 
    String uniqueRowsStepname = "unique rows step";
    UniqueRowsMeta urm = new UniqueRowsMeta();
    urm.setCompareFields(new String[] { "KEY" });
    urm.setCaseInsensitive(new boolean[] { true });
    String uniqueRowsStepPid = registry.getPluginId(StepPluginType.class, urm);
    StepMeta uniqueRowsStep = new StepMeta(uniqueRowsStepPid, uniqueRowsStepname, urm);
    transMeta.addStep(uniqueRowsStep);
    transMeta.addTransHop(new TransHopMeta(sortRowsStep, uniqueRowsStep));
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    transMeta.addTransHop(new TransHopMeta(uniqueRowsStep, dummyStep));
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname, 0);
    RowStepCollector dummyRc = new RowStepCollector();
    si.addRowListener(dummyRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
    checkRows(createResultDataSortCaseSensitiveUniqueCaseInsensitive(), resultRows);
}

Example 3 with UniqueRowsMeta

use of org.pentaho.di.trans.steps.uniquerows.UniqueRowsMeta in project pentaho-kettle by pentaho.

the class UniqueRowsIT method testSortCaseInsensitiveUniqueCaseSensitive.

public void testSortCaseInsensitiveUniqueCaseSensitive() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("uniquerowstest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a sort rows step
    // 
    String sortRowsStepname = "sort rows step";
    SortRowsMeta srm = new SortRowsMeta();
    srm.setFieldName(new String[] { "KEY" });
    srm.setAscending(new boolean[] { true });
    srm.setCaseSensitive(new boolean[] { false });
    srm.setPreSortedField(new boolean[] { false });
    srm.setPrefix("SortRowsTest");
    srm.setDirectory(".");
    String sortRowsStepPid = registry.getPluginId(StepPluginType.class, srm);
    StepMeta sortRowsStep = new StepMeta(sortRowsStepPid, sortRowsStepname, srm);
    transMeta.addStep(sortRowsStep);
    transMeta.addTransHop(new TransHopMeta(injectorStep, sortRowsStep));
    // 
    // Create a unique rows step
    // 
    String uniqueRowsStepname = "unique rows step";
    UniqueRowsMeta urm = new UniqueRowsMeta();
    urm.setCompareFields(new String[] { "KEY" });
    urm.setCaseInsensitive(new boolean[] { false });
    String uniqueRowsStepPid = registry.getPluginId(StepPluginType.class, urm);
    StepMeta uniqueRowsStep = new StepMeta(uniqueRowsStepPid, uniqueRowsStepname, urm);
    transMeta.addStep(uniqueRowsStep);
    transMeta.addTransHop(new TransHopMeta(sortRowsStep, uniqueRowsStep));
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    transMeta.addTransHop(new TransHopMeta(uniqueRowsStep, dummyStep));
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname, 0);
    RowStepCollector dummyRc = new RowStepCollector();
    si.addRowListener(dummyRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
    checkRows(createResultDataSortCaseInsensitiveUniqueCaseSensitive(), resultRows);
}

Example 4 with UniqueRowsMeta

use of org.pentaho.di.trans.steps.uniquerows.UniqueRowsMeta in project pentaho-kettle by pentaho.

the class UniqueRowsIT method testAllUnique.

@Test
public void testAllUnique() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("uniquerowstest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a unique rows step
    // 
    String uniqueRowsStepname = "unique rows step";
    UniqueRowsMeta urm = new UniqueRowsMeta();
    urm.setCompareFields(new String[] { "KEY" });
    urm.setCaseInsensitive(new boolean[] { true });
    urm.setRejectDuplicateRow(true);
    String uniqueRowsStepPid = registry.getPluginId(StepPluginType.class, urm);
    StepMeta uniqueRowsStep = new StepMeta(uniqueRowsStepPid, uniqueRowsStepname, urm);
    uniqueRowsStep.setDistributes(false);
    transMeta.addStep(uniqueRowsStep);
    transMeta.addTransHop(new TransHopMeta(injectorStep, uniqueRowsStep));
    // 
    // Create a dummy step to receive the unique rows
    // 
    String dummyStepname1 = "dummy step";
    DummyTransMeta dm1 = new DummyTransMeta();
    String dummyPid1 = registry.getPluginId(StepPluginType.class, dm1);
    StepMeta dummyStep1 = new StepMeta(dummyPid1, dummyStepname1, dm1);
    transMeta.addStep(dummyStep1);
    transMeta.addTransHop(new TransHopMeta(uniqueRowsStep, dummyStep1));
    // 
    // Create a dummy step to receive the duplicate rows (errors)
    // 
    String dummyStepname2 = "dummy step2";
    DummyTransMeta dm2 = new DummyTransMeta();
    String dummyPid2 = registry.getPluginId(StepPluginType.class, dm2);
    StepMeta dummyStep2 = new StepMeta(dummyPid2, dummyStepname2, dm2);
    transMeta.addStep(dummyStep2);
    // Set up error (aka duplicates) handling info
    StepErrorMeta stepErrorMeta = new StepErrorMeta(new Variables(), uniqueRowsStep);
    stepErrorMeta.setTargetStep(dummyStep2);
    stepErrorMeta.setEnabled(true);
    stepErrorMeta.setNrErrorsValuename("numErrors");
    stepErrorMeta.setErrorDescriptionsValuename("duplicates");
    stepErrorMeta.setErrorFieldsValuename("KEY");
    stepErrorMeta.setErrorCodesValuename("errorCodes");
    stepErrorMeta.setMaxErrors("9999");
    stepErrorMeta.setMaxPercentErrors("");
    stepErrorMeta.setMinPercentRows("");
    uniqueRowsStep.setStepErrorMeta(stepErrorMeta);
    transMeta.addTransHop(new TransHopMeta(uniqueRowsStep, dummyStep2));
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname1, 0);
    RowStepCollector dummyRc1 = new RowStepCollector();
    si.addRowListener(dummyRc1);
    StepInterface si2 = trans.getStepInterface(dummyStepname2, 0);
    RowStepCollector dummyRc2 = new RowStepCollector();
    si2.addRowListener(dummyRc2);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createDataAllUnique();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
    checkRows(createResultDataAllUnique(), resultRows);
    List<RowMetaAndData> errorRows = dummyRc2.getRowsWritten();
    // There should be no duplicates for this test
    assertEquals(errorRows.size(), 0);
}

Example 5 with UniqueRowsMeta

use of org.pentaho.di.trans.steps.uniquerows.UniqueRowsMeta in project pentaho-kettle by pentaho.

the class UniqueRowsIT method testSortCaseInsensitiveUniqueCaseInsensitive.

public void testSortCaseInsensitiveUniqueCaseInsensitive() throws Exception {
    KettleEnvironment.init();
    // 
    // Create a new transformation...
    // 
    TransMeta transMeta = new TransMeta();
    transMeta.setName("uniquerowstest");
    PluginRegistry registry = PluginRegistry.getInstance();
    // 
    // create an injector step...
    // 
    String injectorStepname = "injector step";
    InjectorMeta im = new InjectorMeta();
    // Set the information of the injector.
    String injectorPid = registry.getPluginId(StepPluginType.class, im);
    StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, im);
    transMeta.addStep(injectorStep);
    // 
    // Create a sort rows step
    // 
    String sortRowsStepname = "sort rows step";
    SortRowsMeta srm = new SortRowsMeta();
    srm.setFieldName(new String[] { "KEY" });
    srm.setAscending(new boolean[] { true });
    srm.setCaseSensitive(new boolean[] { false });
    srm.setPreSortedField(new boolean[] { false });
    srm.setPrefix("SortRowsTest");
    srm.setDirectory(".");
    String sortRowsStepPid = registry.getPluginId(StepPluginType.class, srm);
    StepMeta sortRowsStep = new StepMeta(sortRowsStepPid, sortRowsStepname, srm);
    transMeta.addStep(sortRowsStep);
    transMeta.addTransHop(new TransHopMeta(injectorStep, sortRowsStep));
    // 
    // Create a unique rows step
    // 
    String uniqueRowsStepname = "unique rows step";
    UniqueRowsMeta urm = new UniqueRowsMeta();
    urm.setCompareFields(new String[] { "KEY" });
    urm.setCaseInsensitive(new boolean[] { true });
    String uniqueRowsStepPid = registry.getPluginId(StepPluginType.class, urm);
    StepMeta uniqueRowsStep = new StepMeta(uniqueRowsStepPid, uniqueRowsStepname, urm);
    transMeta.addStep(uniqueRowsStep);
    transMeta.addTransHop(new TransHopMeta(sortRowsStep, uniqueRowsStep));
    // 
    // Create a dummy step
    // 
    String dummyStepname = "dummy step";
    DummyTransMeta dm = new DummyTransMeta();
    String dummyPid = registry.getPluginId(StepPluginType.class, dm);
    StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, dm);
    transMeta.addStep(dummyStep);
    transMeta.addTransHop(new TransHopMeta(uniqueRowsStep, dummyStep));
    // Now execute the transformation...
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);
    StepInterface si = trans.getStepInterface(dummyStepname, 0);
    RowStepCollector dummyRc = new RowStepCollector();
    si.addRowListener(dummyRc);
    RowProducer rp = trans.addRowProducer(injectorStepname, 0);
    trans.startThreads();
    // add rows
    List<RowMetaAndData> inputList = createData();
    for (RowMetaAndData rm : inputList) {
        rp.putRow(rm.getRowMeta(), rm.getData());
    }
    rp.finished();
    trans.waitUntilFinished();
    List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
    checkRows(createResultDataSortCaseInsensitiveUniqueCaseInsensitive(), resultRows);
}