Examples with FrequencyIndicator org.talend.dataquality.indicators.FrequencyIndicator used on opensource projects

Example 1 with FrequencyIndicator

use of org.talend.dataquality.indicators.FrequencyIndicator in project tdq-studio-se by Talend.

the class ModelElementIndicatorImpl method createPlainIndicatorUnit.

/**
 * Create a new IndicatorUnit according to indicatorEnum and indicator, if the parameter indicator is null, will
 * create a new indicator .
 *
 * @param indicatorEnum
 * @param indicator
 * @return
 */
private IndicatorUnit createPlainIndicatorUnit(IndicatorEnum indicatorEnum, Indicator indicator) {
    Indicator tempIndicator = indicator;
    if (tempIndicator == null) {
        EFactoryImpl factory = (EFactoryImpl) indicatorEnum.getIndicatorType().getEPackage().getEFactoryInstance();
        tempIndicator = (Indicator) factory.create(indicatorEnum.getIndicatorType());
        // MOD scorreia 2008-09-18: bug 5131 fixed: set indicator's definition when the indicator is created.
        if (!DefinitionHandler.getInstance().setDefaultIndicatorDefinition(tempIndicator)) {
            // $NON-NLS-1$
            log.error("Could not set the definition of the given indicator :" + tempIndicator.getName());
        }
        // for 4225, the frequency indicator need be initialized
        int sqlType = getJavaType();
        if (tempIndicator instanceof FrequencyIndicator && Java2SqlType.isDateInSQL(sqlType)) {
            IndicatorParameters parameters = tempIndicator.getParameters();
            if (parameters == null) {
                parameters = IndicatorsFactory.eINSTANCE.createIndicatorParameters();
                tempIndicator.setParameters(parameters);
            }
            DateParameters dateParameters = parameters.getDateParameters();
            // "Year Frequency Table" when applying on "Time" type
            if (dateParameters == null && !(indicatorEnum == IndicatorEnum.PatternFreqIndicatorEnum || indicatorEnum == IndicatorEnum.PatternLowFreqIndicatorEnum || indicatorEnum == IndicatorEnum.FrequencyIndicatorEnum || indicatorEnum == IndicatorEnum.LowFrequencyIndicatorEnum)) {
                dateParameters = IndicatorsFactory.eINSTANCE.createDateParameters();
                parameters.setDateParameters(dateParameters);
            }
        // TDQ-5357~
        // MOD scorreia 2008-06-19 default is already set in the model
        // dateParameters.setDateAggregationType(DateGrain.YEAR);
        }
    }
    IndicatorUnit indicatorUnit = new ColumnIndicatorUnit(indicatorEnum, tempIndicator, this);
    this.plainIndicatorUnitMap.put(indicatorEnum, indicatorUnit);
    return indicatorUnit;
}

Example 2 with FrequencyIndicator

use of org.talend.dataquality.indicators.FrequencyIndicator in project tdq-studio-se by Talend.

the class LoadSerialDataMain method main.

/**
 * DOC scorreia Comment method "main".
 *
 * @param args
 */
public static void main(String[] args) {
    EMFUtil util = new EMFUtil();
    File file = new File("out/columnTest_0.1.ana");
    System.out.println("Loading file " + file.getAbsolutePath());
    ResourceSet rs = util.getResourceSet();
    Resource r = rs.getResource(URI.createFileURI(file.getAbsolutePath()), true);
    EList<EObject> contents = r.getContents();
    if (contents.isEmpty()) {
        System.err.println("No content in " + r);
    }
    System.out.println("Nb elements in contents " + contents.size());
    IndicatorsSwitch<FrequencyIndicator> mySwitch = new IndicatorsSwitch<FrequencyIndicator>() {

        @Override
        public FrequencyIndicator caseFrequencyIndicator(FrequencyIndicator object) {
            return object;
        }
    };
    for (EObject object : contents) {
        FrequencyIndicator freqI = mySwitch.doSwitch(object);
        if (freqI != null) {
            Long uniqueValueCount = freqI.getUniqueValueCount();
            System.out.println("nb unique values = " + uniqueValueCount);
            EList<Object> uniqueValues = freqI.getUniqueValues();
            for (Object data : uniqueValues) {
                System.out.println("unique value= " + data + " " + freqI.getCount(data));
            }
        }
    }
}

Example 3 with FrequencyIndicator

use of org.talend.dataquality.indicators.FrequencyIndicator in project tdq-studio-se by Talend.

the class AbstractColumnDropTree method hasIndicatorParameters.

/**
 * DOC msjian Comment method "hasIndicatorParameters".
 *
 * @param indicatorUnit
 * @return
 */
private boolean hasIndicatorParameters(IndicatorUnit indicatorUnit) {
    IndicatorParameters parameters = indicatorUnit.getIndicator().getParameters();
    if (parameters == null) {
        return false;
    }
    if (hideParameters(indicatorUnit)) {
        return false;
    }
    if (indicatorUnit.getIndicator() instanceof FrequencyIndicator) {
        // set on tree
        if (parameters.getBins() == null) {
            return false;
        }
        // ~
        return true;
    }
    TextParameters tParameter = parameters.getTextParameter();
    if (tParameter != null && !hideTextParameters(indicatorUnit)) {
        return true;
    }
    DateParameters dParameters = parameters.getDateParameters();
    if (dParameters != null) {
        return true;
    }
    Domain indicatorValidDomain = parameters.getIndicatorValidDomain();
    if (indicatorValidDomain != null) {
        if (indicatorValidDomain.getRanges() != null && !indicatorValidDomain.getRanges().isEmpty()) {
            return true;
        }
    }
    Domain bins = parameters.getBins();
    if (bins != null) {
        return true;
    }
    return false;
}

Example 4 with FrequencyIndicator

use of org.talend.dataquality.indicators.FrequencyIndicator in project tdq-studio-se by Talend.

the class IndicatorEvaluator method initDataSet.

@SuppressWarnings("unchecked")
protected List<Object[]> initDataSet(Indicator indicator, EMap<Indicator, AnalyzedDataSet> indicToRowMap, Object object) {
    AnalyzedDataSet analyzedDataSet = indicToRowMap.get(indicator);
    List<Object[]> valueObjectList = null;
    if (analyzedDataSet == null) {
        analyzedDataSet = AnalysisFactory.eINSTANCE.createAnalyzedDataSet();
        indicToRowMap.put(indicator, analyzedDataSet);
        analyzedDataSet.setDataCount(analysis.getParameters().getMaxNumberRows());
        analyzedDataSet.setRecordSize(0);
    }
    if (indicator instanceof FrequencyIndicator || indicator instanceof MinLengthIndicator || indicator instanceof MaxLengthIndicator) {
        Map<Object, List<Object[]>> valueObjectListMap = analyzedDataSet.getFrequencyData();
        if (valueObjectListMap == null) {
            valueObjectListMap = new HashMap<Object, List<Object[]>>();
            analyzedDataSet.setFrequencyData(valueObjectListMap);
        }
        String key = null;
        if (object == null) {
            key = SpecialValueDisplay.NULL_FIELD;
        } else if (indicator instanceof MinLengthIndicator || indicator instanceof MaxLengthIndicator) {
            key = String.valueOf(object.toString().length());
        } else if (object.equals(PluginConstant.EMPTY_STRING)) {
            key = SpecialValueDisplay.EMPTY_FIELD;
        } else if (indicator instanceof PatternLowFreqIndicator) {
            key = ((PatternLowFreqIndicator) indicator).convertCharacters(object.toString());
        } else if (indicator instanceof PatternFreqIndicator) {
            key = ((PatternFreqIndicator) indicator).convertCharacters(object.toString());
        } else if (indicator instanceof FormatFreqPieIndicator) {
            // MOD qiongli 2011-8-26,feature TDQ-3253.
            key = ((FormatFreqPieIndicator) indicator).getCurrentKey();
        } else {
            key = object.toString();
        }
        valueObjectList = valueObjectListMap.get(key);
        if (valueObjectList == null) {
            valueObjectList = new ArrayList<Object[]>();
            valueObjectListMap.put(key, valueObjectList);
        }
    } else if (indicator.isInValidRow() || indicator.isValidRow()) {
        List<Object> patternData = analyzedDataSet.getPatternData();
        if (patternData == null) {
            patternData = new ArrayList<Object>();
            // mapping with AnalyzedDataSetImpl.VALID_VALUE
            patternData.add(new ArrayList<Object[]>());
            // mapping with AnalyzedDataSetImpl.INVALID_VALUE
            patternData.add(new ArrayList<Object[]>());
            analyzedDataSet.setPatternData(patternData);
        }
        Object listObject = indicator.isInValidRow() ? patternData.get(AnalyzedDataSetImpl.INVALID_VALUE) : patternData.get(AnalyzedDataSetImpl.VALID_VALUE);
        if (listObject instanceof ArrayList<?>) {
            valueObjectList = (ArrayList<Object[]>) listObject;
        }
    } else {
        valueObjectList = analyzedDataSet.getData();
        if (valueObjectList == null) {
            valueObjectList = new ArrayList<Object[]>();
            analyzedDataSet.setData(valueObjectList);
        }
    }
    return valueObjectList;
}

Example 5 with FrequencyIndicator

use of org.talend.dataquality.indicators.FrequencyIndicator in project tdq-studio-se by Talend.

the class IndicatorCommonUtil method handleFrequency.

/**
 * DOC xqliu Comment method "handleFrequency".
 *
 * @param indicator
 * @return
 */
private static Object handleFrequency(Indicator indicator) {
    FrequencyExt[] frequencyExt = null;
    if (UDIHelper.isUDI(indicator)) {
        UserDefIndicator udi = (UserDefIndicator) indicator;
        Set<Object> valueSet = udi.getDistinctValues();
        if (valueSet == null) {
            return null;
        }
        frequencyExt = new FrequencyExt[valueSet.size()];
        int i = 0;
        for (Object o : valueSet) {
            frequencyExt[i] = new FrequencyExt();
            frequencyExt[i].setKey(o);
            frequencyExt[i].setValue(udi.getCount(o));
            frequencyExt[i].setFrequency(udi.getFrequency(o));
            i++;
        }
    } else if (IndicatorEnum.DatePatternFreqIndicatorEnum.getIndicatorType().isInstance(indicator)) {
        DatePatternFreqIndicator datePatternFrequency = (DatePatternFreqIndicator) indicator;
        Map<String, Long> results = datePatternFrequency.getResult();
        frequencyExt = new FrequencyExt[results.size()];
        int i = 0;
        for (String key : results.keySet()) {
            Long value = results.get(key);
            Double frequency = datePatternFrequency.getFrequency(key);
            frequencyExt[i] = new FrequencyExt();
            frequencyExt[i].setKey(key);
            frequencyExt[i].setValue(value);
            frequencyExt[i].setFrequency(frequency);
            i++;
        }
    } else {
        FrequencyIndicator frequency = (FrequencyIndicator) indicator;
        Set<Object> valueSet = frequency.getDistinctValues();
        if (valueSet == null) {
            return null;
        }
        frequencyExt = new FrequencyExt[valueSet.size()];
        int i = 0;
        for (Object o : valueSet) {
            frequencyExt[i] = new FrequencyExt();
            frequencyExt[i].setKey(o);
            if (IndicatorsPackage.eINSTANCE.getSoundexFreqIndicator().equals(frequency.eClass()) || IndicatorsPackage.eINSTANCE.getSoundexLowFreqIndicator().equals(frequency.eClass())) {
                // MOD scorreia 2009-03-23 display distinct count when working with Soundex
                frequencyExt[i].setValue(((SoundexFreqIndicator) frequency).getDistinctCount(o));
            } else {
                frequencyExt[i].setValue(frequency.getCount(o));
            }
            frequencyExt[i].setFrequency(frequency.getFrequency(o));
            i++;
        }
    }
    return frequencyExt;
}