Examples with HyperLogLog com.facebook.airlift.stats.cardinality.HyperLogLog used on opensource projects

Example 6 with HyperLogLog

use of com.facebook.airlift.stats.cardinality.HyperLogLog in project presto by prestodb.

the class KHyperLogLog method newInstance.

public static KHyperLogLog newInstance(Slice serialized) {
    requireNonNull(serialized, "serialized is null");
    SliceInput input = serialized.getInput();
    checkArgument(input.readByte() == VERSION_BYTE, "Unexpected version");
    Long2ObjectRBTreeMap<HyperLogLog> minhash = new Long2ObjectRBTreeMap<>();
    int maxSize = input.readInt();
    int hllBuckets = input.readInt();
    int minhashSize = input.readInt();
    int totalHllSize = input.readInt();
    int[] hllSizes = new int[minhashSize];
    long[] keys = new long[minhashSize];
    input.readBytes(wrappedIntArray(hllSizes));
    input.readBytes(wrappedLongArray(keys));
    Slice allSerializedHlls = input.readSlice(totalHllSize);
    int hllLength;
    int index = 0;
    for (int i = 0; i < minhashSize; i++) {
        Slice serializedHll;
        hllLength = hllSizes[i];
        serializedHll = allSerializedHlls.slice(index, hllLength);
        index += hllLength;
        minhash.put(keys[i], HyperLogLog.newInstance(serializedHll));
    }
    return new KHyperLogLog(maxSize, hllBuckets, minhash);
}

Example 7 with HyperLogLog

use of com.facebook.airlift.stats.cardinality.HyperLogLog in project presto by prestodb.

the class KHyperLogLog method mergeWith.

public KHyperLogLog mergeWith(KHyperLogLog other) {
    LongIterator iterator = other.minhash.keySet().iterator();
    while (iterator.hasNext()) {
        long key = iterator.nextLong();
        HyperLogLog thisHll = minhash.get(key);
        HyperLogLog otherHll = other.minhash.get(key);
        if (minhash.containsKey(key)) {
            decreaseTotalHllSize(thisHll);
            thisHll.mergeWith(otherHll);
            increaseTotalHllSize(thisHll);
        } else {
            minhash.put(key, otherHll);
            increaseTotalHllSize(otherHll);
        }
    }
    removeOverflowEntries();
    return this;
}

Example 8 with HyperLogLog

use of com.facebook.airlift.stats.cardinality.HyperLogLog in project presto by prestodb.

the class HyperLogLogOperators method castToP4Hll.

@ScalarOperator(CAST)
@SqlType(StandardTypes.P4_HYPER_LOG_LOG)
public static Slice castToP4Hll(@SqlType(StandardTypes.HYPER_LOG_LOG) Slice slice) {
    HyperLogLog hll = HyperLogLog.newInstance(slice);
    hll.makeDense();
    return hll.serialize();
}

Example 9 with HyperLogLog

use of com.facebook.airlift.stats.cardinality.HyperLogLog in project presto by prestodb.

the class MergeHyperLogLogAggregation method input.

@InputFunction
public static void input(@AggregationState HyperLogLogState state, @SqlType(StandardTypes.HYPER_LOG_LOG) Slice value) {
    HyperLogLog input = HyperLogLog.newInstance(value);
    HyperLogLogUtils.mergeState(state, input);
}

Example 10 with HyperLogLog

use of com.facebook.airlift.stats.cardinality.HyperLogLog in project presto by prestodb.

the class ApproximateSetAggregation method input.

@InputFunction
public static void input(@AggregationState HyperLogLogState state, @SqlType(StandardTypes.BIGINT) long value, @SqlType(StandardTypes.DOUBLE) double maxStandardError) {
    HyperLogLog hll = HyperLogLogUtils.getOrCreateHyperLogLog(state, maxStandardError);
    state.addMemoryUsage(-hll.estimatedInMemorySize());
    hll.add(value);
    state.addMemoryUsage(hll.estimatedInMemorySize());
}