Examples with RowCountIndicator org.talend.dataquality.indicators.RowCountIndicator used on opensource projects

Example 21 with RowCountIndicator

use of org.talend.dataquality.indicators.RowCountIndicator in project tdq-studio-se by Talend.

the class ColumnSetMultiValueIndicatorImpl method basicSetRowCountIndicator.

/**
 * <!-- begin-user-doc --> <!-- end-user-doc -->
 * @generated
 */
public NotificationChain basicSetRowCountIndicator(RowCountIndicator newRowCountIndicator, NotificationChain msgs) {
    RowCountIndicator oldRowCountIndicator = rowCountIndicator;
    rowCountIndicator = newRowCountIndicator;
    if (eNotificationRequired()) {
        ENotificationImpl notification = new ENotificationImpl(this, Notification.SET, ColumnsetPackage.COLUMN_SET_MULTI_VALUE_INDICATOR__ROW_COUNT_INDICATOR, oldRowCountIndicator, newRowCountIndicator);
        if (msgs == null)
            msgs = notification;
        else
            msgs.add(notification);
    }
    return msgs;
}

Example 22 with RowCountIndicator

use of org.talend.dataquality.indicators.RowCountIndicator in project tdq-studio-se by Talend.

the class IndicatorEvaluationMain method main.

/**
 * DOC scorreia Comment method "main".
 *
 * @param args
 */
public static void main(String[] args) {
    TypedProperties connectionParams = PropertiesLoader.getProperties(IndicatorEvaluator.class, "db.properties");
    String driverClassName = connectionParams.getProperty("driver");
    String dbUrl = connectionParams.getProperty("url");
    try {
        TimeTracer tt = new TimeTracer("Indicator evaluation", null);
        tt.start();
        // create connection
        Connection connection = ConnectionUtils.createConnection(dbUrl, driverClassName, connectionParams);
        String database = "test";
        String tableName = "my_test";
        // --- columns to analyze
        String[] columnsArray = new String[] { // 0
        "my_int", // 1
        "my_double", // 2
        "my_text", // 4
        "my_date", // 3
        "my_string", // 5
        "my_int_null" };
        List<String> columns = Arrays.asList(columnsArray);
        // store in file
        File file = new File("out/columnTest_0.1.ana");
        EMFUtil util = new EMFUtil();
        Resource resource = util.getResourceSet().createResource(URI.createFileURI(file.getAbsolutePath()));
        rContents = resource.getContents();
        evaluator.setConnection(connection);
        // --- create indicators
        RowCountIndicator rowCountIndicator = IndicatorsFactory.eINSTANCE.createRowCountIndicator();
        NullCountIndicator nullCountIndicator = IndicatorsFactory.eINSTANCE.createNullCountIndicator();
        DistinctCountIndicator distinctCountIndicator = IndicatorsFactory.eINSTANCE.createDistinctCountIndicator();
        DistinctCountIndicator distinctCountIndicator2 = IndicatorsFactory.eINSTANCE.createDistinctCountIndicator();
        UniqueCountIndicator uniqueCountIndicator = IndicatorsFactory.eINSTANCE.createUniqueCountIndicator();
        DuplicateCountIndicator duplicateCountIndicator = IndicatorsFactory.eINSTANCE.createDuplicateCountIndicator();
        BlankCountIndicator blankCountIndicator = IndicatorsFactory.eINSTANCE.createBlankCountIndicator();
        MinLengthIndicator minLengthIndicator = IndicatorsFactory.eINSTANCE.createMinLengthIndicator();
        MaxLengthIndicator maxLengthIndicator = IndicatorsFactory.eINSTANCE.createMaxLengthIndicator();
        AverageLengthIndicator averageLengthIndicator = IndicatorsFactory.eINSTANCE.createAverageLengthIndicator();
        AverageLengthIndicator averageLengthIndicator2 = IndicatorsFactory.eINSTANCE.createAverageLengthIndicator();
        ModeIndicator modeIndicator = IndicatorsFactory.eINSTANCE.createModeIndicator();
        FrequencyIndicator textFrequencyIndicator = IndicatorsFactory.eINSTANCE.createFrequencyIndicator();
        // store in freq indic
        // textFrequencyIndicator.setDistinctCountIndicator(distinctCountIndicator);
        // textFrequencyIndicator.setDistinctCountIndicator(distinctCountIndicator2);
        // textFrequencyIndicator.setUniqueCountIndicator(uniqueCountIndicator);
        // textFrequencyIndicator.setDuplicateCountIndicator(duplicateCountIndicator);
        // textFrequencyIndicator.setModeIndicator(modeIndicator);
        MeanIndicator doubleMeanIndicator = IndicatorsFactory.eINSTANCE.createMeanIndicator();
        MeanIndicator integerMeanIndicator = IndicatorsFactory.eINSTANCE.createMeanIndicator();
        MedianIndicator medianIndicator = IndicatorsFactory.eINSTANCE.createMedianIndicator();
        SumIndicator integerSumIndicator = IndicatorsFactory.eINSTANCE.createSumIndicator();
        addIndicator(columnsArray[0], medianIndicator);
        addIndicator(columnsArray[1], doubleMeanIndicator);
        addIndicator(columnsArray[2], blankCountIndicator);
        addIndicator(columnsArray[5], nullCountIndicator);
        // addIndicator(columnsArray[2], textFrequencyIndicator);
        // addIndicator(columnsArray[2], distinctCountIndicator); // probably not useful?
        // addIndicator(columnsArray[2], uniqueCountIndicator); // probably not useful?
        // addIndicator(columnsArray[2], duplicateCountIndicator); // probably not useful?
        // addIndicator(columnsArray[2], modeIndicator); // probably not useful?
        addIndicator(columnsArray[3], rowCountIndicator);
        addIndicator(columnsArray[5], integerSumIndicator);
        addIndicator(columnsArray[5], integerMeanIndicator);
        addIndicator(columnsArray[2], averageLengthIndicator);
        addIndicator(columnsArray[3], averageLengthIndicator2);
        addIndicator(columnsArray[3], minLengthIndicator);
        addIndicator(columnsArray[3], maxLengthIndicator);
        // build query on columns
        // TODO scorreia add filter somewhere here...
        String selectCols = sqlSelectColumns(database, tableName, columns);
        // --- create a description of the column set
        QueryExpression queryExpression = DatatypesFactory.eINSTANCE.createQueryExpression();
        queryExpression.setBody(selectCols);
        // TODO scorreia externalize this as a constant
        queryExpression.setLanguage("SQL");
        tt.start("compute");
        evaluator.setFetchSize(10000);
        evaluator.evaluateIndicators(selectCols, true);
        tt.end("compute");
        // Print indicators the median
        System.out.println("Median=" + medianIndicator.getMedian());
        System.out.println("# Unique values= " + textFrequencyIndicator.getUniqueValueCount());
        System.out.println("# Distinct values= " + textFrequencyIndicator.getDistinctValueCount());
        for (String col : columns) {
            printIndicators(evaluator.getIndicators(col));
        }
        tt.start("save");
        util.save();
        tt.end("saved in " + file.getAbsolutePath());
        tt.end();
        CwmResource cwmR = (CwmResource) resource;
        String id = cwmR.getID(medianIndicator);
        System.out.println("ecore util.getId= " + EcoreUtil.getID(medianIndicator));
        System.out.println("uuId= " + id);
    // test reload this file
    // LoadSerialData.main(args);
    } catch (SQLException e) {
        // TODO Auto-generated catch block
        log.error(e, e);
    } catch (InstantiationException e) {
        // TODO Auto-generated catch block
        log.error(e, e);
    } catch (IllegalAccessException e) {
        // TODO Auto-generated catch block
        log.error(e, e);
    } catch (ClassNotFoundException e) {
        // TODO Auto-generated catch block
        log.error(e, e);
    }
}

Example 23 with RowCountIndicator

use of org.talend.dataquality.indicators.RowCountIndicator in project tdq-studio-se by Talend.

the class InidcatorUnitTest method testGetFormsCase2.

/**
 * Test method for {@link org.talend.dataprofiler.core.ui.editor.preview.IndicatorUnit#getForms()}.
 *
 * test case 2: Get Forms from TableIndicatorUnit
 */
@Test
public void testGetFormsCase2() {
    RowCountIndicator rowCountIndicator = IndicatorsFactory.eINSTANCE.createRowCountIndicator();
    IRepositoryViewObject rowCountRepositoryViewObject = null;
    try {
        List<IRepositoryViewObject> all = ProxyRepositoryFactory.getInstance().getAll(ERepositoryObjectType.SYSTEM_INDICATORS_SIMPLE_STATISTICS);
        Assert.assertEquals(10, all.size());
        rowCountRepositoryViewObject = all.get(7);
    } catch (PersistenceException e) {
        Assert.fail(e.getMessage());
    }
    Assert.assertNotNull(rowCountRepositoryViewObject);
    TDQIndicatorDefinitionItem rowCountItem = (TDQIndicatorDefinitionItem) rowCountRepositoryViewObject.getProperty().getItem();
    rowCountIndicator.setIndicatorDefinition(rowCountItem.getIndicatorDefinition());
    ColumnIndicatorUnit columnIndicatorUnit = new ColumnIndicatorUnit(IndicatorEnum.RowCountIndicatorEnum, rowCountIndicator, null);
    FormEnum[] forms = new FormEnum[] { FormEnum.IndicatorThresholdsForm };
    FormEnum[] resultforms = columnIndicatorUnit.getForms();
    Assert.assertEquals(forms.length, resultforms.length);
    for (int index = 0; index < forms.length; index++) {
        Assert.assertEquals(forms[index], resultforms[index]);
    }
}

Example 24 with RowCountIndicator

use of org.talend.dataquality.indicators.RowCountIndicator in project tdq-studio-se by Talend.

the class InidcatorUnitTest method testGetIndicatorNameCase1.

/**
 * Test method for {@link org.talend.dataprofiler.core.ui.editor.preview.IndicatorUnit#getIndicatorName()}.
 *
 * test case 1: Get Indicator Name from ColumnSetIndicatorUnit
 */
@Test
public void testGetIndicatorNameCase1() {
    RowCountIndicator rowCountIndicator = IndicatorsFactory.eINSTANCE.createRowCountIndicator();
    IRepositoryViewObject rowCountRepositoryViewObject = null;
    try {
        List<IRepositoryViewObject> all = ProxyRepositoryFactory.getInstance().getAll(ERepositoryObjectType.SYSTEM_INDICATORS_SIMPLE_STATISTICS);
        Assert.assertEquals(10, all.size());
        rowCountRepositoryViewObject = all.get(7);
    } catch (PersistenceException e) {
        Assert.fail(e.getMessage());
    }
    Assert.assertNotNull(rowCountRepositoryViewObject);
    TDQIndicatorDefinitionItem rowCountItem = (TDQIndicatorDefinitionItem) rowCountRepositoryViewObject.getProperty().getItem();
    rowCountIndicator.setIndicatorDefinition(rowCountItem.getIndicatorDefinition());
    ColumnSetIndicatorUnit columnSetIndicatorUnit = new ColumnSetIndicatorUnit(IndicatorEnum.RowCountIndicatorEnum, rowCountIndicator);
    Assert.assertEquals(org.talend.cwm.management.i18n.Messages.getString(rowCountItem.getProperty().getLabel()), columnSetIndicatorUnit.getIndicatorName());
}

Example 25 with RowCountIndicator

use of org.talend.dataquality.indicators.RowCountIndicator in project tdq-studio-se by Talend.

the class InidcatorUnitTest method testGetFormsCase3.

/**
 * Test method for {@link org.talend.dataprofiler.core.ui.editor.preview.IndicatorUnit#getForms()}.
 *
 * test case 3: Get Forms from ColumnIndicatorUnit
 */
@Test
public void testGetFormsCase3() {
    RowCountIndicator rowCountIndicator = IndicatorsFactory.eINSTANCE.createRowCountIndicator();
    IRepositoryViewObject rowCountRepositoryViewObject = null;
    try {
        List<IRepositoryViewObject> all = ProxyRepositoryFactory.getInstance().getAll(ERepositoryObjectType.SYSTEM_INDICATORS_SIMPLE_STATISTICS);
        Assert.assertEquals(10, all.size());
        rowCountRepositoryViewObject = all.get(7);
    } catch (PersistenceException e) {
        Assert.fail(e.getMessage());
    }
    Assert.assertNotNull(rowCountRepositoryViewObject);
    TDQIndicatorDefinitionItem rowCountItem = (TDQIndicatorDefinitionItem) rowCountRepositoryViewObject.getProperty().getItem();
    rowCountIndicator.setIndicatorDefinition(rowCountItem.getIndicatorDefinition());
    TableIndicatorUnit TableIndicatorUnit = new TableIndicatorUnit(IndicatorEnum.RowCountIndicatorEnum, rowCountIndicator, null);
    FormEnum[] forms = new FormEnum[] { FormEnum.IndicatorThresholdsForm };
    FormEnum[] resultforms = TableIndicatorUnit.getForms();
    Assert.assertEquals(forms.length, resultforms.length);
    for (int index = 0; index < forms.length; index++) {
        Assert.assertEquals(forms[index], resultforms[index]);
    }
}