Examples with HyperLogLog com.facebook.airlift.stats.cardinality.HyperLogLog used on opensource projects

Example 1 with HyperLogLog

use of com.facebook.airlift.stats.cardinality.HyperLogLog in project presto by prestodb.

the class TestReadWrite method nextHyperLogLog.

private static Slice nextHyperLogLog(Random random) {
    HyperLogLog hll = HyperLogLog.newInstance(HYPER_LOG_LOG_BUCKETS);
    int size = random.nextInt(MAX_HYPER_LOG_LOG_ELEMENTS);
    for (int i = 0; i < size; i++) {
        hll.add(random.nextLong());
    }
    return hll.serialize();
}

Example 2 with HyperLogLog

use of com.facebook.airlift.stats.cardinality.HyperLogLog in project presto by prestodb.

the class CreateHll method createHll.

@ScalarFunction
@SqlType(StandardTypes.HYPER_LOG_LOG)
public static Slice createHll(@SqlType(StandardTypes.BIGINT) long value) {
    HyperLogLog hll = HyperLogLog.newInstance(standardErrorToBuckets(DEFAULT_STANDARD_ERROR));
    hll.add(value);
    return hll.serialize();
}

Example 3 with HyperLogLog

use of com.facebook.airlift.stats.cardinality.HyperLogLog in project presto by prestodb.

the class TestMergeHyperLogLogAggregation method getExpectedValue.

@Override
public Object getExpectedValue(int start, int length) {
    if (length == 0) {
        return null;
    }
    HyperLogLog hll = HyperLogLog.newInstance(NUMBER_OF_BUCKETS);
    for (int i = start; i < start + length; i++) {
        hll.add(i);
    }
    hll.makeDense();
    return new SqlVarbinary(hll.serialize().getBytes());
}

Example 4 with HyperLogLog

use of com.facebook.airlift.stats.cardinality.HyperLogLog in project presto by prestodb.

the class TestMergeHyperLogLogAggregation method getSequenceBlocks.

// use dense for expected and actual to assure same serialized bytes
@Override
public Block[] getSequenceBlocks(int start, int length) {
    BlockBuilder blockBuilder = HYPER_LOG_LOG.createBlockBuilder(null, length);
    for (int i = start; i < start + length; i++) {
        HyperLogLog hll = HyperLogLog.newInstance(NUMBER_OF_BUCKETS);
        hll.add(i);
        hll.makeDense();
        HYPER_LOG_LOG.writeSlice(blockBuilder, hll.serialize());
    }
    return new Block[] { blockBuilder.build() };
}

Example 5 with HyperLogLog

use of com.facebook.airlift.stats.cardinality.HyperLogLog in project presto by prestodb.

the class KHyperLogLog method serialize.

public Slice serialize() {
    try (SliceOutput output = new DynamicSliceOutput(estimatedSerializedSize())) {
        List<Slice> hllSlices = new ArrayList<>();
        IntList hllSizes = new IntArrayList();
        int totalHllSize = 0;
        for (HyperLogLog hll : minhash.values()) {
            Slice serializedHll = hll.serialize();
            hllSlices.add(serializedHll);
            totalHllSize += serializedHll.length();
            hllSizes.add(serializedHll.length());
        }
        Slice hashesSlice = wrappedLongArray(minhash.keySet().toLongArray());
        Slice hllSizesSlice = wrappedIntArray(hllSizes.toIntArray());
        output.appendByte(VERSION_BYTE);
        output.appendInt(maxSize);
        output.appendInt(hllBuckets);
        output.appendInt(minhash.size());
        output.appendInt(totalHllSize);
        output.appendBytes(hllSizesSlice);
        output.appendBytes(hashesSlice);
        for (Slice hllSlice : hllSlices) {
            output.appendBytes(hllSlice);
        }
        return output.slice();
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}