Examples with HyperLogLog com.facebook.airlift.stats.cardinality.HyperLogLog used on opensource projects

Example 16 with HyperLogLog

use of com.facebook.airlift.stats.cardinality.HyperLogLog in project presto by prestodb.

the class ApproximateSetAggregation method input.

@InputFunction
@LiteralParameters("x")
public static void input(@AggregationState HyperLogLogState state, @SqlType("varchar(x)") Slice value, @SqlType(StandardTypes.DOUBLE) double maxStandardError) {
    HyperLogLog hll = HyperLogLogUtils.getOrCreateHyperLogLog(state, maxStandardError);
    state.addMemoryUsage(-hll.estimatedInMemorySize());
    hll.add(value);
    state.addMemoryUsage(hll.estimatedInMemorySize());
}

Example 17 with HyperLogLog

use of com.facebook.airlift.stats.cardinality.HyperLogLog in project presto by prestodb.

the class TestHyperLogLogFunctions method getUniqueElements.

private List<HyperLogLog> getUniqueElements(int blockSize, long uniqueElements) {
    ImmutableList.Builder<HyperLogLog> builder = ImmutableList.builder();
    for (int j = 0; j < blockSize; j++) {
        // create a single HyperLogLog column
        HyperLogLog firstHll = HyperLogLog.newInstance(NUMBER_OF_BUCKETS);
        // populate column with even partitions of the unique elements
        for (long i = j * uniqueElements / blockSize; i < j * uniqueElements / blockSize + uniqueElements / blockSize; i++) {
            firstHll.add(i);
        }
        builder.add(firstHll);
    }
    return builder.build();
}

Example 18 with HyperLogLog

use of com.facebook.airlift.stats.cardinality.HyperLogLog in project presto by prestodb.

the class TestHyperLogLogFunctions method getMergeProjection.

private String getMergeProjection(List<HyperLogLog> list) {
    String projection = "merge_hll(ARRAY[";
    Iterator<HyperLogLog> iterator = list.listIterator();
    ImmutableList.Builder<String> casts = ImmutableList.builder();
    for (HyperLogLog current : list) {
        Slice firstSerial = current.serialize();
        byte[] firstBytes = firstSerial.getBytes();
        String firstEncode = BaseEncoding.base16().lowerCase().encode(firstBytes);
        // create an iterable with all our cast statements
        casts.add("CAST(X'" + firstEncode + "' AS HyperLogLog)");
    }
    projection += Joiner.on(", ").join(casts.build());
    projection += "])";
    return projection;
}

Example 19 with HyperLogLog

use of com.facebook.airlift.stats.cardinality.HyperLogLog in project presto by prestodb.

the class TestHyperLogLogFunctions method getCardinalityProjection.

private String getCardinalityProjection(List<HyperLogLog> list) {
    String projection = "cardinality(merge_hll(ARRAY[";
    Iterator<HyperLogLog> iterator = list.listIterator();
    ImmutableList.Builder<String> casts = ImmutableList.builder();
    for (HyperLogLog current : list) {
        Slice firstSerial = current.serialize();
        byte[] firstBytes = firstSerial.getBytes();
        String firstEncode = BaseEncoding.base16().lowerCase().encode(firstBytes);
        // create an iterable with all our cast statements
        casts.add("CAST(X'" + firstEncode + "' AS HyperLogLog)");
    }
    projection += Joiner.on(", ").join(casts.build());
    projection += "]))";
    return projection;
}

Example 20 with HyperLogLog

use of com.facebook.airlift.stats.cardinality.HyperLogLog in project presto by prestodb.

the class HyperLogLogFunctions method scalarMerge.

@ScalarFunction("merge_hll")
@Description("merge the contents of an array of HyperLogLogs")
@SqlType(StandardTypes.HYPER_LOG_LOG)
@SqlNullable
public static Slice scalarMerge(@SqlType("array(HyperLogLog)") Block block) {
    if (block.getPositionCount() == 0) {
        return null;
    }
    HyperLogLog merged = null;
    int firstNonNullIndex = 0;
    while (firstNonNullIndex < block.getPositionCount() && block.isNull(firstNonNullIndex)) {
        firstNonNullIndex++;
    }
    if (firstNonNullIndex == block.getPositionCount()) {
        return null;
    }
    Slice initialSlice = block.getSlice(firstNonNullIndex, 0, block.getSliceLength(firstNonNullIndex));
    merged = HyperLogLog.newInstance(initialSlice);
    for (int i = firstNonNullIndex; i < block.getPositionCount(); i++) {
        Slice currentSlice = block.getSlice(i, 0, block.getSliceLength(i));
        if (!block.isNull(i)) {
            merged.mergeWith(HyperLogLog.newInstance(currentSlice));
        }
    }
    return merged.serialize();
}