Search in sources :

Example 1 with DataTypeOccurences

use of org.talend.dataquality.statistics.type.DataTypeOccurences in project data-prep by Talend.

the class StatisticsUtilsTest method adaptColumn.

private void adaptColumn(final ColumnMetadata column, final DataTypeEnum type) {
    Analyzers.Result result = new Analyzers.Result();
    // Data type
    DataTypeOccurences dataType = new DataTypeOccurences();
    dataType.increment(type);
    result.add(dataType);
    // Semantic type
    SemanticType semanticType = new SemanticType();
    CategoryFrequency category1 = new CategoryFrequency("category 1", "category 1");
    category1.setScore(99);
    semanticType.increment(category1, 1);
    result.add(semanticType);
    // Suggested types
    CategoryFrequency category2 = new CategoryFrequency("category 2", "category 2");
    category2.setScore(81);
    semanticType.increment(category2, 1);
    CategoryFrequency category3 = new CategoryFrequency("category 3", "category 3");
    category3.setScore(50);
    semanticType.increment(category3, 1);
    // Value quality
    ValueQualityStatistics valueQualityStatistics = new ValueQualityStatistics();
    valueQualityStatistics.setEmptyCount(10);
    valueQualityStatistics.setInvalidCount(20);
    valueQualityStatistics.setValidCount(30);
    result.add(valueQualityStatistics);
    // Cardinality
    CardinalityStatistics cardinalityStatistics = new CardinalityStatistics();
    cardinalityStatistics.incrementCount();
    cardinalityStatistics.add("distinctValue");
    result.add(cardinalityStatistics);
    // Data frequency
    DataTypeFrequencyStatistics dataFrequencyStatistics = new DataTypeFrequencyStatistics();
    dataFrequencyStatistics.add("frequentValue1");
    dataFrequencyStatistics.add("frequentValue1");
    dataFrequencyStatistics.add("frequentValue2");
    dataFrequencyStatistics.add("frequentValue2");
    result.add(dataFrequencyStatistics);
    // Pattern frequency
    PatternFrequencyStatistics patternFrequencyStatistics = new PatternFrequencyStatistics();
    patternFrequencyStatistics.add("999a999");
    patternFrequencyStatistics.add("999a999");
    patternFrequencyStatistics.add("999aaaa");
    patternFrequencyStatistics.add("999aaaa");
    result.add(patternFrequencyStatistics);
    // Quantiles
    QuantileStatistics quantileStatistics = new QuantileStatistics();
    quantileStatistics.add(1d);
    quantileStatistics.add(2d);
    quantileStatistics.endAddValue();
    result.add(quantileStatistics);
    // Summary
    SummaryStatistics summaryStatistics = new SummaryStatistics();
    summaryStatistics.addData(1d);
    summaryStatistics.addData(2d);
    result.add(summaryStatistics);
    // Histogram
    StreamNumberHistogramStatistics histogramStatistics = new StreamNumberHistogramStatistics();
    histogramStatistics.setNumberOfBins(2);
    histogramStatistics.add(1);
    histogramStatistics.add(2);
    result.add(histogramStatistics);
    // Text length
    TextLengthStatistics textLengthStatistics = new TextLengthStatistics();
    textLengthStatistics.setMaxTextLength(30);
    textLengthStatistics.setMinTextLength(10);
    textLengthStatistics.setSumTextLength(40);
    textLengthStatistics.setCount(5);
    result.add(textLengthStatistics);
    StatisticsAdapter adapter = new StatisticsAdapter(40);
    adapter.adapt(Collections.singletonList(integerColumn), Collections.singletonList(result));
    adapter.adapt(Collections.singletonList(stringColumn), Collections.singletonList(result));
}
Also used : SemanticType(org.talend.dataquality.semantic.statistics.SemanticType) DataTypeFrequencyStatistics(org.talend.dataquality.statistics.frequency.DataTypeFrequencyStatistics) Analyzers(org.talend.dataquality.common.inference.Analyzers) CategoryFrequency(org.talend.dataquality.semantic.recognizer.CategoryFrequency) ValueQualityStatistics(org.talend.dataquality.common.inference.ValueQualityStatistics) SummaryStatistics(org.talend.dataquality.statistics.numeric.summary.SummaryStatistics) QuantileStatistics(org.talend.dataquality.statistics.numeric.quantile.QuantileStatistics) TextLengthStatistics(org.talend.dataquality.statistics.text.TextLengthStatistics) StatisticsAdapter(org.talend.dataprep.dataset.StatisticsAdapter) CardinalityStatistics(org.talend.dataquality.statistics.cardinality.CardinalityStatistics) StreamNumberHistogramStatistics(org.talend.dataprep.api.dataset.statistics.number.StreamNumberHistogramStatistics) DataTypeOccurences(org.talend.dataquality.statistics.type.DataTypeOccurences) PatternFrequencyStatistics(org.talend.dataquality.statistics.frequency.pattern.PatternFrequencyStatistics)

Example 2 with DataTypeOccurences

use of org.talend.dataquality.statistics.type.DataTypeOccurences in project data-prep by Talend.

the class StatisticsAdapter method injectDataTypeAnalysis.

private void injectDataTypeAnalysis(final ColumnMetadata column, final Analyzers.Result result) {
    if (result.exist(DataTypeOccurences.class) && !column.isTypeForced()) {
        final DataTypeOccurences dataType = result.get(DataTypeOccurences.class);
        final DataTypeEnum suggestedEnumType = dataType.getSuggestedType();
        final Type suggestedColumnType = Type.get(suggestedEnumType.name());
        // the suggested type can be modified by #injectValueQuality
        column.setType(suggestedColumnType.getName());
    }
    injectValueQuality(column, result);
}
Also used : SemanticType(org.talend.dataquality.semantic.statistics.SemanticType) Type(org.talend.dataprep.api.type.Type) DataTypeEnum(org.talend.dataquality.statistics.type.DataTypeEnum) DataTypeOccurences(org.talend.dataquality.statistics.type.DataTypeOccurences)

Aggregations

SemanticType (org.talend.dataquality.semantic.statistics.SemanticType)2 DataTypeOccurences (org.talend.dataquality.statistics.type.DataTypeOccurences)2 StreamNumberHistogramStatistics (org.talend.dataprep.api.dataset.statistics.number.StreamNumberHistogramStatistics)1 Type (org.talend.dataprep.api.type.Type)1 StatisticsAdapter (org.talend.dataprep.dataset.StatisticsAdapter)1 Analyzers (org.talend.dataquality.common.inference.Analyzers)1 ValueQualityStatistics (org.talend.dataquality.common.inference.ValueQualityStatistics)1 CategoryFrequency (org.talend.dataquality.semantic.recognizer.CategoryFrequency)1 CardinalityStatistics (org.talend.dataquality.statistics.cardinality.CardinalityStatistics)1 DataTypeFrequencyStatistics (org.talend.dataquality.statistics.frequency.DataTypeFrequencyStatistics)1 PatternFrequencyStatistics (org.talend.dataquality.statistics.frequency.pattern.PatternFrequencyStatistics)1 QuantileStatistics (org.talend.dataquality.statistics.numeric.quantile.QuantileStatistics)1 SummaryStatistics (org.talend.dataquality.statistics.numeric.summary.SummaryStatistics)1 TextLengthStatistics (org.talend.dataquality.statistics.text.TextLengthStatistics)1 DataTypeEnum (org.talend.dataquality.statistics.type.DataTypeEnum)1