Examples with QuantileDigest com.facebook.airlift.stats.QuantileDigest used on opensource projects

Example 11 with QuantileDigest

use of com.facebook.airlift.stats.QuantileDigest in project presto by prestodb.

the class TestQuantileDigestFunctions method testScale.

@Test
public void testScale() {
    QuantileDigest qdigest = new QuantileDigest(1);
    addAll(qdigest, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
    // Before scaling.
    assertEquals(qdigest.getHistogram(asList(0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L)), asList(new QuantileDigest.Bucket(0, Double.NaN), new QuantileDigest.Bucket(1, 0), new QuantileDigest.Bucket(1, 1), new QuantileDigest.Bucket(1, 2), new QuantileDigest.Bucket(1, 3), new QuantileDigest.Bucket(1, 4), new QuantileDigest.Bucket(1, 5), new QuantileDigest.Bucket(1, 6), new QuantileDigest.Bucket(1, 7), new QuantileDigest.Bucket(1, 8), new QuantileDigest.Bucket(1, 9)));
    // Scale up.
    SqlVarbinary sqlVarbinary = functionAssertions.selectSingleValue(format("scale_qdigest(CAST(X'%s' AS qdigest(bigint)), 2)", toHexString(qdigest)), QDIGEST_BIGINT, SqlVarbinary.class);
    QuantileDigest scaledQdigest = new QuantileDigest(wrappedBuffer(sqlVarbinary.getBytes()));
    assertEquals(scaledQdigest.getHistogram(asList(0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L)), asList(new QuantileDigest.Bucket(0, Double.NaN), new QuantileDigest.Bucket(2, 0), new QuantileDigest.Bucket(2, 1), new QuantileDigest.Bucket(2, 2), new QuantileDigest.Bucket(2, 3), new QuantileDigest.Bucket(2, 4), new QuantileDigest.Bucket(2, 5), new QuantileDigest.Bucket(2, 6), new QuantileDigest.Bucket(2, 7), new QuantileDigest.Bucket(2, 8), new QuantileDigest.Bucket(2, 9)));
    // Scale down.
    sqlVarbinary = functionAssertions.selectSingleValue(format("scale_qdigest(CAST(X'%s' AS qdigest(bigint)), 0.5)", toHexString(qdigest)), QDIGEST_BIGINT, SqlVarbinary.class);
    scaledQdigest = new QuantileDigest(wrappedBuffer(sqlVarbinary.getBytes()));
    assertEquals(scaledQdigest.getHistogram(asList(0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L)), asList(new QuantileDigest.Bucket(0, Double.NaN), new QuantileDigest.Bucket(0.5, 0), new QuantileDigest.Bucket(0.5, 1), new QuantileDigest.Bucket(0.5, 2), new QuantileDigest.Bucket(0.5, 3), new QuantileDigest.Bucket(0.5, 4), new QuantileDigest.Bucket(0.5, 5), new QuantileDigest.Bucket(0.5, 6), new QuantileDigest.Bucket(0.5, 7), new QuantileDigest.Bucket(0.5, 8), new QuantileDigest.Bucket(0.5, 9)));
}

Example 12 with QuantileDigest

use of com.facebook.airlift.stats.QuantileDigest in project presto by prestodb.

the class TestQuantileDigestFunctions method testQuantileAtValueDouble.

@Test
public void testQuantileAtValueDouble() {
    QuantileDigest qdigest = new QuantileDigest(1);
    ImmutableList.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9).stream().mapToLong(FloatingPointBitsConverterUtil::doubleToSortableLong).forEach(qdigest::add);
    functionAssertions.assertFunction(format("quantile_at_value(CAST(X'%s' AS qdigest(double)), 5.6)", toHexString(qdigest)), DOUBLE, 0.6);
    functionAssertions.assertFunction(format("quantile_at_value(CAST(X'%s' AS qdigest(double)), -1.23)", toHexString(qdigest)), DOUBLE, null);
    functionAssertions.assertFunction(format("quantile_at_value(CAST(X'%s' AS qdigest(double)), 12.3)", toHexString(qdigest)), DOUBLE, null);
    functionAssertions.assertFunction(format("quantile_at_value(CAST(X'%s' AS qdigest(double)), nan())", toHexString(qdigest)), DOUBLE, null);
}

Example 13 with QuantileDigest

use of com.facebook.airlift.stats.QuantileDigest in project presto by prestodb.

the class DigestAndPercentileArrayStateSerializer method deserialize.

@Override
public void deserialize(Block block, int index, DigestAndPercentileArrayState state) {
    SliceInput input = VARBINARY.getSlice(block, index).getInput();
    // read number of percentiles
    int numPercentiles = input.readInt();
    ImmutableList.Builder<Double> percentilesListBuilder = ImmutableList.builder();
    for (int i = 0; i < numPercentiles; i++) {
        percentilesListBuilder.add(input.readDouble());
    }
    state.setPercentiles(percentilesListBuilder.build());
    // read digest
    int length = input.readInt();
    state.setDigest(new QuantileDigest(input.readSlice(length)));
    state.addMemoryUsage(state.getDigest().estimatedInMemorySizeInBytes());
}

Example 14 with QuantileDigest

use of com.facebook.airlift.stats.QuantileDigest in project presto by prestodb.

the class DigestAndPercentileStateSerializer method deserialize.

@Override
public void deserialize(Block block, int index, DigestAndPercentileState state) {
    SliceInput input = VARBINARY.getSlice(block, index).getInput();
    // read percentile
    state.setPercentile(input.readDouble());
    // read digest
    int length = input.readInt();
    QuantileDigest digest = new QuantileDigest(input.readSlice(length));
    state.setDigest(digest);
    state.addMemoryUsage(state.getDigest().estimatedInMemorySizeInBytes());
}

Example 15 with QuantileDigest

use of com.facebook.airlift.stats.QuantileDigest in project presto by prestodb.

the class TestQuantileDigestAggregationFunction method getExpectedValuesFloats.

private Object getExpectedValuesFloats(double maxError, float... values) {
    if (values.length == 0) {
        return null;
    }
    QuantileDigest qdigest = new QuantileDigest(maxError);
    Floats.asList(values).forEach(value -> qdigest.add(floatToSortableInt(value)));
    return new SqlVarbinary(qdigest.serialize().getBytes());
}