Examples with TDigest com.facebook.presto.tdigest.TDigest used on opensource projects

Example 21 with TDigest

use of com.facebook.presto.tdigest.TDigest in project presto by prestodb.

the class TestTDigestFunctions method testNormalDistributionHighVariance.

@Test
public void testNormalDistributionHighVariance() {
    TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR);
    List<Double> list = new ArrayList<>();
    NormalDistribution normal = new NormalDistribution(0, 1);
    for (int i = 0; i < NUMBER_OF_ENTRIES; i++) {
        double value = normal.sample();
        tDigest.add(value);
        list.add(value);
    }
    sort(list);
    for (int i = 0; i < quantiles.length; i++) {
        assertContinuousQuantileWithinBound(quantiles[i], STANDARD_ERROR, list, tDigest);
    }
}

Example 22 with TDigest

use of com.facebook.presto.tdigest.TDigest in project presto by prestodb.

the class TestTDigestFunctions method testDestructureTDigest.

@Test
public void testDestructureTDigest() {
    TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR);
    ImmutableList<Double> values = ImmutableList.of(0.0d, 1.0d, 2.0d, 3.0d, 4.0d, 5.0d, 6.0d, 7.0d, 8.0d, 9.0d);
    values.stream().forEach(tDigest::add);
    List<Integer> weights = Collections.nCopies(values.size(), 1);
    double compression = Double.valueOf(STANDARD_COMPRESSION_FACTOR);
    double min = values.stream().reduce(Double.POSITIVE_INFINITY, Double::min);
    double max = values.stream().reduce(Double.NEGATIVE_INFINITY, Double::max);
    double sum = values.stream().reduce(0.0d, Double::sum);
    long count = values.size();
    String sql = format("destructure_tdigest(CAST(X'%s' AS tdigest(%s)))", new SqlVarbinary(tDigest.serialize().getBytes()).toString().replaceAll("\\s+", " "), DOUBLE);
    functionAssertions.assertFunction(sql, TDIGEST_CENTROIDS_ROW_TYPE, ImmutableList.of(values, weights, compression, min, max, sum, count));
    functionAssertions.assertFunction(format("%s.compression", sql), DOUBLE, compression);
    functionAssertions.assertFunction(format("%s.min", sql), DOUBLE, min);
    functionAssertions.assertFunction(format("%s.max", sql), DOUBLE, max);
    functionAssertions.assertFunction(format("%s.sum", sql), DOUBLE, sum);
    functionAssertions.assertFunction(format("%s.count", sql), BIGINT, count);
    functionAssertions.assertFunction(format("%s.centroid_means", sql), new ArrayType(DOUBLE), values);
    functionAssertions.assertFunction(format("%s.centroid_weights", sql), new ArrayType(INTEGER), weights);
}

Example 23 with TDigest

use of com.facebook.presto.tdigest.TDigest in project presto by prestodb.

the class TestTDigestFunctions method testBinomialDistribution.

// disabled because test takes almost 10s
@Test(enabled = false)
public void testBinomialDistribution() {
    int trials = 10;
    for (int k = 1; k < trials; k++) {
        TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR);
        BinomialDistribution binomial = new BinomialDistribution(trials, k * 0.1);
        List<Integer> list = new ArrayList<>();
        for (int i = 0; i < NUMBER_OF_ENTRIES; i++) {
            int sample = binomial.sample();
            tDigest.add(sample);
            list.add(sample);
        }
        Collections.sort(list);
        for (int i = 0; i < quantiles.length; i++) {
            assertDiscreteQuantileWithinBound(quantiles[i], STANDARD_ERROR, list, tDigest);
        }
    }
}

Example 24 with TDigest

use of com.facebook.presto.tdigest.TDigest in project presto by prestodb.

the class TestTDigestFunctions method testAddElementsRandomized.

@Test
public void testAddElementsRandomized() {
    TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR);
    List<Double> list = new ArrayList<>();
    for (int i = 0; i < NUMBER_OF_ENTRIES; i++) {
        double value = Math.random() * NUMBER_OF_ENTRIES;
        tDigest.add(value);
        list.add(value);
    }
    sort(list);
    for (int i = 0; i < quantiles.length; i++) {
        assertContinuousQuantileWithinBound(quantiles[i], STANDARD_ERROR, list, tDigest);
    }
}

Example 25 with TDigest

use of com.facebook.presto.tdigest.TDigest in project presto by prestodb.

the class TestTDigestFunctions method testNormalDistributionHighVarianceQuantileArray.

@Test
public void testNormalDistributionHighVarianceQuantileArray() {
    TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR);
    List<Double> list = new ArrayList<>();
    NormalDistribution normal = new NormalDistribution(0, 1);
    for (int i = 0; i < NUMBER_OF_ENTRIES; i++) {
        double value = normal.sample();
        tDigest.add(value);
        list.add(value);
    }
    sort(list);
    assertBlockQuantiles(quantiles, STANDARD_ERROR, list, tDigest);
}