Examples with HyperLogLogPlus com.clearspring.analytics.stream.cardinality.HyperLogLogPlus used on opensource projects

Example 61 with HyperLogLogPlus

use of com.clearspring.analytics.stream.cardinality.HyperLogLogPlus in project shifu by ShifuML.

the class AutoTypeDistinctCountReducer method reduce.

@Override
protected void reduce(IntWritable key, Iterable<CountAndFrequentItemsWritable> values, Context context) throws IOException, InterruptedException {
    HyperLogLogPlus hyperLogLogPlus = null;
    Set<String> fis = new HashSet<String>();
    long count = 0, invalidCount = 0, validNumCount = 0;
    for (CountAndFrequentItemsWritable cfiw : values) {
        count += cfiw.getCount();
        invalidCount += cfiw.getInvalidCount();
        validNumCount += cfiw.getValidNumCount();
        fis.addAll(cfiw.getFrequetItems());
        if (hyperLogLogPlus == null) {
            hyperLogLogPlus = HyperLogLogPlus.Builder.build(cfiw.getHyperBytes());
        } else {
            try {
                hyperLogLogPlus = (HyperLogLogPlus) hyperLogLogPlus.merge(HyperLogLogPlus.Builder.build(cfiw.getHyperBytes()));
            } catch (CardinalityMergeException e) {
                throw new RuntimeException(e);
            }
        }
    }
    outputValue.set(count + ":" + invalidCount + ":" + validNumCount + ":" + hyperLogLogPlus.cardinality() + ":" + limitedFrequentItems(fis));
    context.write(key, outputValue);
}

Example 62 with HyperLogLogPlus

use of com.clearspring.analytics.stream.cardinality.HyperLogLogPlus in project angel by Tencent.

the class GetHyperLogLog method partitionGet.

@Override
public PartitionGetResult partitionGet(PartitionGetParam partParam) {
    GetHyperLogLogPartParam param = (GetHyperLogLogPartParam) partParam;
    ServerLongAnyRow row = GraphMatrixUtils.getPSLongKeyRow(psContext, param);
    ILongKeyPartOp keyPart = (ILongKeyPartOp) param.getNodes();
    long[] nodes = keyPart.getKeys();
    Long2ObjectOpenHashMap<HyperLogLogPlus> logs = new Long2ObjectOpenHashMap<>(nodes.length);
    row.startRead(20000);
    try {
        for (int i = 0; i < nodes.length; i++) {
            HyperLogLogPlusElement hllElem = (HyperLogLogPlusElement) row.get(nodes[i]);
            if (hllElem.isActive()) {
                logs.put(nodes[i], hllElem.getHyperLogLogPlus());
            }
        }
    } finally {
        row.endRead();
    }
    return new GetHyperLogLogPartResult(logs);
}

Example 63 with HyperLogLogPlus

use of com.clearspring.analytics.stream.cardinality.HyperLogLogPlus in project angel by Tencent.

the class UpdateHyperLogLog method partitionUpdate.

@Override
public void partitionUpdate(PartitionUpdateParam partParam) {
    UpdateHyperLogLogPartParam param = (UpdateHyperLogLogPartParam) partParam;
    ServerLongAnyRow row = GraphMatrixUtils.getPSLongKeyRow(psContext, param);
    ILongKeyAnyValuePartOp split = (ILongKeyAnyValuePartOp) param.getKeyValuePart();
    int p = param.getP();
    int sp = param.getSp();
    long seed = param.getSeed();
    long[] keys = split.getKeys();
    IElement[] values = split.getValues();
    row.startWrite();
    try {
        if (keys != null && keys.length > 0 && values != null && values.length > 0) {
            for (int i = 0; i < keys.length; i++) {
                long key = keys[i];
                HyperLogLogPlus value = ((HLLPlusElement) values[i]).getCounter();
                if (!row.exist(key))
                    row.set(key, new HyperLogLogPlusElement(key, p, sp, seed));
                HyperLogLogPlusElement hllElem = (HyperLogLogPlusElement) row.get(key);
                if (hllElem.isActive()) {
                    hllElem.merge(value);
                }
            }
        }
    } finally {
        row.endWrite();
    }
}