use of org.talend.dataquality.statistics.numeric.summary.SummaryStatistics in project data-prep by Talend.
the class StatisticsUtilsTest method adaptColumn.
private void adaptColumn(final ColumnMetadata column, final DataTypeEnum type) {
Analyzers.Result result = new Analyzers.Result();
// Data type
DataTypeOccurences dataType = new DataTypeOccurences();
dataType.increment(type);
result.add(dataType);
// Semantic type
SemanticType semanticType = new SemanticType();
CategoryFrequency category1 = new CategoryFrequency("category 1", "category 1");
category1.setScore(99);
semanticType.increment(category1, 1);
result.add(semanticType);
// Suggested types
CategoryFrequency category2 = new CategoryFrequency("category 2", "category 2");
category2.setScore(81);
semanticType.increment(category2, 1);
CategoryFrequency category3 = new CategoryFrequency("category 3", "category 3");
category3.setScore(50);
semanticType.increment(category3, 1);
// Value quality
ValueQualityStatistics valueQualityStatistics = new ValueQualityStatistics();
valueQualityStatistics.setEmptyCount(10);
valueQualityStatistics.setInvalidCount(20);
valueQualityStatistics.setValidCount(30);
result.add(valueQualityStatistics);
// Cardinality
CardinalityStatistics cardinalityStatistics = new CardinalityStatistics();
cardinalityStatistics.incrementCount();
cardinalityStatistics.add("distinctValue");
result.add(cardinalityStatistics);
// Data frequency
DataTypeFrequencyStatistics dataFrequencyStatistics = new DataTypeFrequencyStatistics();
dataFrequencyStatistics.add("frequentValue1");
dataFrequencyStatistics.add("frequentValue1");
dataFrequencyStatistics.add("frequentValue2");
dataFrequencyStatistics.add("frequentValue2");
result.add(dataFrequencyStatistics);
// Pattern frequency
PatternFrequencyStatistics patternFrequencyStatistics = new PatternFrequencyStatistics();
patternFrequencyStatistics.add("999a999");
patternFrequencyStatistics.add("999a999");
patternFrequencyStatistics.add("999aaaa");
patternFrequencyStatistics.add("999aaaa");
result.add(patternFrequencyStatistics);
// Quantiles
QuantileStatistics quantileStatistics = new QuantileStatistics();
quantileStatistics.add(1d);
quantileStatistics.add(2d);
quantileStatistics.endAddValue();
result.add(quantileStatistics);
// Summary
SummaryStatistics summaryStatistics = new SummaryStatistics();
summaryStatistics.addData(1d);
summaryStatistics.addData(2d);
result.add(summaryStatistics);
// Histogram
StreamNumberHistogramStatistics histogramStatistics = new StreamNumberHistogramStatistics();
histogramStatistics.setNumberOfBins(2);
histogramStatistics.add(1);
histogramStatistics.add(2);
result.add(histogramStatistics);
// Text length
TextLengthStatistics textLengthStatistics = new TextLengthStatistics();
textLengthStatistics.setMaxTextLength(30);
textLengthStatistics.setMinTextLength(10);
textLengthStatistics.setSumTextLength(40);
textLengthStatistics.setCount(5);
result.add(textLengthStatistics);
StatisticsAdapter adapter = new StatisticsAdapter(40);
adapter.adapt(Collections.singletonList(integerColumn), Collections.singletonList(result));
adapter.adapt(Collections.singletonList(stringColumn), Collections.singletonList(result));
}
use of org.talend.dataquality.statistics.numeric.summary.SummaryStatistics in project data-prep by Talend.
the class StatisticsAdapter method injectNumberSummary.
/**
* Injects numerical statistics like max, min to statistics of the specified column metadata.
*
* For columns of type date, min and max values are retrieved from the date histogram
*
* @param column the specified column metadata
* @param result the analyzer result
*/
private void injectNumberSummary(final ColumnMetadata column, final Analyzers.Result result) {
if (result.exist(SummaryStatistics.class)) {
final Statistics statistics = column.getStatistics();
final SummaryStatistics summaryStatistics = result.get(SummaryStatistics.class);
statistics.setMean(summaryStatistics.getMean());
statistics.setVariance(summaryStatistics.getVariance());
// if the column is of type Date
if (DATE.isAssignableFrom(column.getType()) && result.exist(StreamDateHistogramStatistics.class)) {
final DateHistogram histogram = (DateHistogram) result.get(StreamDateHistogramStatistics.class).getHistogram();
statistics.setMax(histogram.getMaxUTCEpochMilliseconds());
statistics.setMin(histogram.getMinUTCEpochMilliseconds());
} else {
statistics.setMax(summaryStatistics.getMax());
statistics.setMin(summaryStatistics.getMin());
}
}
}
Aggregations