Examples with AllMatchIndicator org.talend.dataquality.indicators.columnset.AllMatchIndicator used on opensource projects

Example 1 with AllMatchIndicator

use of org.talend.dataquality.indicators.columnset.AllMatchIndicator in project tdq-studio-se by Talend.

the class ColumnSetAnalysisDetailsPage method refreshIndicatorsSection.

/**
 * refresh the Indicators Section.
 */
public void refreshIndicatorsSection() {
    // when there have pattern, show allmatchindicator
    ModelElementIndicator[] modelElementIndicator = treeViewer.getModelElementIndicator();
    if (modelElementIndicator != null) {
        for (ModelElementIndicator modelElementInd : modelElementIndicator) {
            Indicator[] inds = modelElementInd.getPatternIndicators();
            for (Indicator ind : inds) {
                if (ind instanceof RegexpMatchingIndicator) {
                    indicatorsViewer.setInput(simpleStatIndicator, allMatchIndicator);
                    return;
                }
            }
        }
    }
    indicatorsViewer.setInput(simpleStatIndicator);
}

Example 2 with AllMatchIndicator

use of org.talend.dataquality.indicators.columnset.AllMatchIndicator in project tdq-studio-se by Talend.

the class ColumnSetAnalysisDetailsPage method recomputeIndicators.

public void recomputeIndicators() {
    columnSetAnalysisHandler = new ColumnSetAnalysisHandler();
    columnSetAnalysisHandler.setAnalysis(getCurrentModelElement());
    stringDataFilter = columnSetAnalysisHandler.getStringDataFilter();
    analyzedColumns = columnSetAnalysisHandler.getAnalyzedColumns();
    if (columnSetAnalysisHandler.getSimpleStatIndicator() == null || columnSetAnalysisHandler.getSimpleStatIndicator().eIsProxy()) {
        ColumnsetFactory columnsetFactory = ColumnsetFactory.eINSTANCE;
        simpleStatIndicator = columnsetFactory.createSimpleStatIndicator();
        simpleStatIndicator.setRowCountIndicator(IndicatorsFactory.eINSTANCE.createRowCountIndicator());
        simpleStatIndicator.setDistinctCountIndicator(IndicatorsFactory.eINSTANCE.createDistinctCountIndicator());
        simpleStatIndicator.setDuplicateCountIndicator(IndicatorsFactory.eINSTANCE.createDuplicateCountIndicator());
        simpleStatIndicator.setUniqueCountIndicator(IndicatorsFactory.eINSTANCE.createUniqueCountIndicator());
    } else {
        simpleStatIndicator = (SimpleStatIndicator) columnSetAnalysisHandler.getSimpleStatIndicator();
    }
    if (columnSetAnalysisHandler.getAllmatchIndicator() == null) {
        ColumnsetFactory columnsetFactory = ColumnsetFactory.eINSTANCE;
        allMatchIndicator = columnsetFactory.createAllMatchIndicator();
        DefinitionHandler.getInstance().setDefaultIndicatorDefinition(allMatchIndicator);
    } else {
        allMatchIndicator = (AllMatchIndicator) columnSetAnalysisHandler.getAllmatchIndicator();
    }
    initializeIndicator(simpleStatIndicator);
    List<ModelElementIndicator> meIndicatorList = new ArrayList<ModelElementIndicator>();
    ModelElementIndicator currentIndicator = null;
    for (ModelElement element : analyzedColumns) {
        // MOD yyi 2011-02-16 17871:delimitefile
        MetadataColumn mdColumn = SwitchHelpers.METADATA_COLUMN_SWITCH.doSwitch(element);
        TdColumn tdColumn = SwitchHelpers.COLUMN_SWITCH.doSwitch(element);
        if (tdColumn == null && mdColumn == null) {
            continue;
        }
        if (tdColumn == null && mdColumn != null) {
            currentIndicator = ModelElementIndicatorHelper.createDFColumnIndicator(RepositoryNodeHelper.recursiveFind(mdColumn));
        } else if (tdColumn != null) {
            RepositoryNode recursiveFind = RepositoryNodeHelper.recursiveFind(tdColumn);
            if (recursiveFind == null) {
                recursiveFind = RepositoryNodeHelper.createRepositoryNode(tdColumn);
            }
            currentIndicator = ModelElementIndicatorHelper.createModelElementIndicator(recursiveFind);
        }
        DataminingType dataminingType = MetadataHelper.getDataminingType(element);
        MetadataHelper.setDataminingType(dataminingType == null ? DataminingType.NOMINAL : dataminingType, element);
        Collection<Indicator> indicatorList = columnSetAnalysisHandler.getRegexMathingIndicators(element);
        if (null != currentIndicator) {
            currentIndicator.setIndicators(indicatorList.toArray(new Indicator[indicatorList.size()]));
            meIndicatorList.add(currentIndicator);
        }
    }
    currentModelElementIndicators = meIndicatorList.toArray(new ModelElementIndicator[meIndicatorList.size()]);
}

Example 3 with AllMatchIndicator

use of org.talend.dataquality.indicators.columnset.AllMatchIndicator in project tdq-studio-se by Talend.

the class IndicatorsComp method setInput.

// input composite indicator
public void setInput(Object... obj) {
    List<ColumnSetIndicatorUnit> indicatortList = new ArrayList<ColumnSetIndicatorUnit>();
    for (Object indicatorObj : obj) {
        // for SimpleStatIndicator, CountAvgNullIndicator, MinMaxDateIndicator, WeakCorrelationIndicator
        if (indicatorObj instanceof SimpleStatIndicator || indicatorObj instanceof CountAvgNullIndicator || indicatorObj instanceof MinMaxDateIndicator || indicatorObj instanceof WeakCorrelationIndicator) {
            columnSetIndicator = (ColumnSetMultiValueIndicator) indicatorObj;
            for (Indicator indicator : IndicatorHelper.getIndicatorLeavesBySingleNode(columnSetIndicator)) {
                IndicatorEnum indicatorEnum = IndicatorEnum.findIndicatorEnum(indicator.eClass());
                indicatortList.add(new ColumnSetIndicatorUnit(indicatorEnum, indicator));
            }
        // MOD msjian TDQ-8860: we always show the allMatchIndicator in the Indicators section
        } else if (indicatorObj instanceof AllMatchIndicator) {
            // for AllMatchIndicator
            AllMatchIndicator allMatchIndicator = (AllMatchIndicator) indicatorObj;
            indicatortList.add(new ColumnSetIndicatorUnit(IndicatorEnum.AllMatchIndicatorEnum, allMatchIndicator));
        // TDQ-8860~
        }
    // ~
    }
    setElements(indicatortList.toArray(new ColumnSetIndicatorUnit[indicatortList.size()]));
}

Example 4 with AllMatchIndicator

use of org.talend.dataquality.indicators.columnset.AllMatchIndicator in project tdq-studio-se by Talend.

the class ColumnSetIndicatorEvaluator method storeDataSet.

/**
 * store data which from 'simpleIndicator.getListRows()' except RowCountIndicator.
 *
 * @param indicToRowMap
 */
private void storeDataSet() {
    EMap<Indicator, AnalyzedDataSet> indicToRowMap = analysis.getResults().getIndicToRowMap();
    for (Indicator indicator : analysis.getResults().getIndicators()) {
        if (indicator instanceof SimpleStatIndicator) {
            SimpleStatIndicator simpleIndicator = (SimpleStatIndicator) indicator;
            if (!analysis.getParameters().isStoreData()) {
                break;
            }
            if (simpleIndicator.isUsedMapDBMode() && AnalysisHelper.isJavaExecutionEngine(analysis)) {
            // nothing need to do
            } else {
                List<Object[]> listRows = simpleIndicator.getListRows();
                if (listRows == null || listRows.isEmpty()) {
                    break;
                }
                for (Indicator leafIndicator : simpleIndicator.getLeafIndicators()) {
                    if (leafIndicator instanceof RowCountIndicator) {
                        continue;
                    }
                    List<Object[]> dataList = new ArrayList<Object[]>();
                    AnalyzedDataSet analyzedDataSet = indicToRowMap.get(leafIndicator);
                    if (analyzedDataSet == null) {
                        analyzedDataSet = AnalysisFactory.eINSTANCE.createAnalyzedDataSet();
                        indicToRowMap.put(leafIndicator, analyzedDataSet);
                        analyzedDataSet.setDataCount(analysis.getParameters().getMaxNumberRows());
                        analyzedDataSet.setRecordSize(0);
                    }
                    for (int i = 0; i < listRows.size(); i++) {
                        // if (dataList.size() >= analyzedDataSet.getDataCount()) {
                        // break;
                        // }
                        Object[] object = listRows.get(i);
                        // the last element store the count value.
                        Object count = object[object.length > 0 ? object.length - 1 : 0];
                        if (leafIndicator instanceof DistinctCountIndicator) {
                            dataList.add(object);
                        } else if (leafIndicator instanceof UniqueCountIndicator) {
                            if (count != null && NumberUtils.isNumber(count + PluginConstant.EMPTY_STRING)) {
                                if (Long.valueOf(count + PluginConstant.EMPTY_STRING).longValue() == 1) {
                                    dataList.add(object);
                                }
                            }
                        } else if (leafIndicator instanceof DuplicateCountIndicator) {
                            if (count != null && NumberUtils.isNumber(count + PluginConstant.EMPTY_STRING)) {
                                if (Long.valueOf(count + PluginConstant.EMPTY_STRING).longValue() > 1) {
                                    dataList.add(object);
                                }
                            }
                        }
                    }
                    analyzedDataSet.setData(dataList);
                }
                // MOD sizhaoliu TDQ-7144 clear the listRows after usage for drill down
                if (!simpleIndicator.isStoreData()) {
                    simpleIndicator.setListRows(new ArrayList<Object[]>());
                }
            }
        }
        if (indicator instanceof AllMatchIndicator) {
            AllMatchIndicator allMatchIndicator = (AllMatchIndicator) indicator;
            if (!allMatchIndicator.isStoreData()) {
                allMatchIndicator.setListRows(new ArrayList<Object[]>());
            }
        }
    }
}

Example 5 with AllMatchIndicator

use of org.talend.dataquality.indicators.columnset.AllMatchIndicator in project tdq-studio-se by Talend.

the class FileSystemImportWriter method updatePatternInAnaParams.

/**
 * if there is a same name pattern in current workspace,update the pattern in imported analysis IndicatorParameters.
 *
 * @param systemSupplyModelElement
 * @param analysis
 */
private void updatePatternInAnaParams(ModelElement systemSupplyModelElement, Analysis analysis) {
    if (analysis.getResults() != null) {
        EList<Indicator> indicators = analysis.getResults().getIndicators();
        IndicatorParameters parameters = null;
        for (Indicator indicator : indicators) {
            // AllMatchIndicator is in column set analysis.
            if (indicator instanceof AllMatchIndicator) {
                EList<RegexpMatchingIndicator> list = ((AllMatchIndicator) indicator).getCompositeRegexMatchingIndicators();
                for (RegexpMatchingIndicator regMatchIndicator : list) {
                    parameters = regMatchIndicator.getParameters();
                    removOldAddSysPatternInAnaParams(parameters, (Pattern) systemSupplyModelElement, analysis);
                }
            } else if (indicator instanceof PatternMatchingIndicator) {
                parameters = ((PatternMatchingIndicator) indicator).getParameters();
                removOldAddSysPatternInAnaParams(parameters, (Pattern) systemSupplyModelElement, analysis);
            }
        }
    }
}