Examples with DataQualityWritable co.cask.cdap.dq.DataQualityWritable used on opensource projects

Example 1 with DataQualityWritable

use of co.cask.cdap.dq.DataQualityWritable in project cdap by caskdata.

the class AggregationFunctionsTest method uniqueValuesGenerateAggregationTest.

@Test
public void uniqueValuesGenerateAggregationTest() throws Exception {
    DataQualityWritable val1 = new DataQualityWritable();
    DataQualityWritable val2 = new DataQualityWritable();
    DataQualityWritable val3 = new DataQualityWritable();
    val1.set(new Text("a"));
    val2.set(new Text("a"));
    val3.set(new Text("a"));
    UniqueValues uniqueValues = new UniqueValues();
    uniqueValues.add(val1);
    uniqueValues.add(val2);
    uniqueValues.add(val3);
    byte[] output = uniqueValues.aggregate();
    Assert.assertEquals("[a]", Bytes.toString(output));
}

Example 2 with DataQualityWritable

use of co.cask.cdap.dq.DataQualityWritable in project cdap by caskdata.

the class AggregationFunctionsTest method discreteValuesGenerateAggregationTest.

@Test
public void discreteValuesGenerateAggregationTest() throws Exception {
    DataQualityWritable val1 = new DataQualityWritable();
    DataQualityWritable val2 = new DataQualityWritable();
    DataQualityWritable val3 = new DataQualityWritable();
    val1.set(new Text("a"));
    val2.set(new Text("a"));
    val3.set(new Text("b"));
    DiscreteValuesHistogram discreteValuesHistogram = new DiscreteValuesHistogram();
    discreteValuesHistogram.add(val1);
    discreteValuesHistogram.add(val2);
    discreteValuesHistogram.add(val3);
    Map<String, Integer> expectedMap = Maps.newHashMap();
    expectedMap.put("a", 2);
    expectedMap.put("b", 1);
    byte[] outputVal = discreteValuesHistogram.aggregate();
    Map<String, Integer> outputMap = GSON.fromJson(Bytes.toString(outputVal), TOKEN_TYPE_MAP_STRING_INTEGER);
    Assert.assertEquals(expectedMap, outputMap);
}

Example 3 with DataQualityWritable

use of co.cask.cdap.dq.DataQualityWritable in project cdap by caskdata.

the class AggregationFunctionsTest method standardDeviationGenerateAggregationTest.

@Test
public void standardDeviationGenerateAggregationTest() throws Exception {
    DataQualityWritable val1 = new DataQualityWritable();
    val1.set(new DoubleWritable(2.0));
    DataQualityWritable val2 = new DataQualityWritable();
    val2.set(new DoubleWritable(5.0));
    DataQualityWritable val3 = new DataQualityWritable();
    val3.set(new DoubleWritable(10.0));
    DataQualityWritable val4 = new DataQualityWritable();
    val4.set(new DoubleWritable(52.0));
    StandardDeviation standardDeviation = new StandardDeviation();
    standardDeviation.add(val1);
    standardDeviation.add(val2);
    standardDeviation.add(val3);
    standardDeviation.add(val4);
    byte[] output = standardDeviation.aggregate();
    Assert.assertEquals(20.265426, Bytes.toDouble(output), 0.001);
}

Example 4 with DataQualityWritable

use of co.cask.cdap.dq.DataQualityWritable in project cdap by caskdata.

the class AggregationFunctionsTest method averageGenerateAggregationTest.

@Test
public void averageGenerateAggregationTest() throws Exception {
    DataQualityWritable val1 = new DataQualityWritable();
    val1.set(new DoubleWritable(2.0));
    DataQualityWritable val2 = new DataQualityWritable();
    val2.set(new DoubleWritable(2.0));
    Mean mean = new Mean();
    mean.add(val1);
    mean.add(val2);
    byte[] output = mean.aggregate();
    Assert.assertEquals(2.0, Bytes.toDouble(output), 0);
}

Example 5 with DataQualityWritable

use of co.cask.cdap.dq.DataQualityWritable in project cdap by caskdata.

the class AggregationFunctionsTest method histogramWithBucketingTest.

@Test
public void histogramWithBucketingTest() throws Exception {
    DataQualityWritable val1 = new DataQualityWritable();
    DataQualityWritable val2 = new DataQualityWritable();
    DataQualityWritable val3 = new DataQualityWritable();
    DataQualityWritable val4 = new DataQualityWritable();
    DataQualityWritable val5 = new DataQualityWritable();
    DataQualityWritable val6 = new DataQualityWritable();
    DataQualityWritable val7 = new DataQualityWritable();
    DataQualityWritable val8 = new DataQualityWritable();
    val1.set(new IntWritable(2));
    val2.set(new IntWritable(3));
    val3.set(new IntWritable(4));
    val4.set(new IntWritable(16));
    val5.set(new IntWritable(16));
    val6.set(new IntWritable(26));
    val7.set(new IntWritable(46));
    val8.set(new IntWritable(56));
    HistogramWithBucketing histogramWithBucketing = new HistogramWithBucketing();
    histogramWithBucketing.add(val1);
    histogramWithBucketing.add(val2);
    histogramWithBucketing.add(val3);
    histogramWithBucketing.add(val4);
    histogramWithBucketing.add(val5);
    histogramWithBucketing.add(val6);
    histogramWithBucketing.add(val7);
    histogramWithBucketing.add(val8);
    histogramWithBucketing.aggregate();
    Map<Map.Entry<Double, Double>, Long> expectedMap = new HashMap<>();
    Map.Entry<Double, Double> expectedMapEntry = new AbstractMap.SimpleEntry<>(2.0, 86.0);
    expectedMap.put(expectedMapEntry, new Long(8));
    Assert.assertEquals(histogramWithBucketing.histogram, expectedMap);
}