Examples with HyperLogLogCollector org.apache.druid.hll.HyperLogLogCollector used on opensource projects

Example 31 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class SingleValueStringCardinalityVectorProcessor method aggregate.

@Override
public void aggregate(ByteBuffer buf, int numRows, int[] positions, @Nullable int[] rows, int positionOffset) {
    // Save position, limit and restore later instead of allocating a new ByteBuffer object
    final int oldPosition = buf.position();
    final int oldLimit = buf.limit();
    try {
        final int[] vector = selector.getRowVector();
        for (int i = 0; i < numRows; i++) {
            final String s = selector.lookupName(vector[rows != null ? rows[i] : i]);
            if (NullHandling.replaceWithDefault() || s != null) {
                final int position = positions[i] + positionOffset;
                buf.limit(position + HyperLogLogCollector.getLatestNumBytesForDenseStorage());
                buf.position(position);
                final HyperLogLogCollector collector = HyperLogLogCollector.makeCollector(buf);
                StringCardinalityAggregatorColumnSelectorStrategy.addStringToCollector(collector, s);
            }
        }
    } finally {
        buf.limit(oldLimit);
        buf.position(oldPosition);
    }
}

Example 32 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class HyperUniquesSerde method getExtractor.

@Override
public ComplexMetricExtractor<HyperLogLogCollector> getExtractor() {
    return new ComplexMetricExtractor<HyperLogLogCollector>() {

        @Override
        public Class<HyperLogLogCollector> extractedClass() {
            return HyperLogLogCollector.class;
        }

        @Override
        public HyperLogLogCollector extractValue(InputRow inputRow, String metricName) {
            Object rawValue = inputRow.getRaw(metricName);
            if (rawValue instanceof HyperLogLogCollector) {
                return (HyperLogLogCollector) rawValue;
            } else {
                HyperLogLogCollector collector = HyperLogLogCollector.makeLatestCollector();
                List<String> dimValues = inputRow.getDimension(metricName);
                if (dimValues == null) {
                    return collector;
                }
                for (String dimensionValue : dimValues) {
                    collector.add(hyperLogLogHash.hash(dimensionValue));
                }
                return collector;
            }
        }
    };
}

Example 33 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class HyperUniquesSerde method getObjectStrategy.

@Override
public ObjectStrategy getObjectStrategy() {
    return new ObjectStrategy<HyperLogLogCollector>() {

        @Override
        public Class<HyperLogLogCollector> getClazz() {
            return HyperLogLogCollector.class;
        }

        @Override
        public HyperLogLogCollector fromByteBuffer(ByteBuffer buffer, int numBytes) {
            // make a copy of buffer, because the given buffer is not duplicated in HyperLogLogCollector.makeCollector() and
            // stored in a field.
            final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer();
            readOnlyBuffer.limit(readOnlyBuffer.position() + numBytes);
            return HyperLogLogCollector.makeCollector(readOnlyBuffer);
        }

        @Override
        public byte[] toBytes(HyperLogLogCollector collector) {
            if (collector == null) {
                return new byte[] {};
            }
            ByteBuffer val = collector.toByteBuffer();
            byte[] retVal = new byte[val.remaining()];
            val.asReadOnlyBuffer().get(retVal);
            return retVal;
        }

        @Override
        public int compare(HyperLogLogCollector o1, HyperLogLogCollector o2) {
            return comparator.compare(o1, o2);
        }
    };
}

Example 34 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class HyperUniqueExpressionsTest method testLong.

@Test
public void testLong() {
    Expr expr = Parser.parse("hyper_unique_add(1234, hyper_unique())", MACRO_TABLE);
    ExprEval eval = expr.eval(inputBindings);
    Assert.assertEquals(HyperUniqueExpressions.TYPE, eval.type());
    Assert.assertTrue(eval.value() instanceof HyperLogLogCollector);
    Assert.assertEquals(1.0, ((HyperLogLogCollector) eval.value()).estimateCardinality(), 0.01);
    expr = Parser.parse("hyper_unique_add(1234, hyper_unique_add(5678, hyper_unique()))", MACRO_TABLE);
    eval = expr.eval(inputBindings);
    Assert.assertEquals(HyperUniqueExpressions.TYPE, eval.type());
    Assert.assertTrue(eval.value() instanceof HyperLogLogCollector);
    Assert.assertEquals(2.0, ((HyperLogLogCollector) eval.value()).estimateCardinality(), 0.01);
    expr = Parser.parse("hyper_unique_add(long, hyper_unique())", MACRO_TABLE);
    eval = expr.eval(inputBindings);
    Assert.assertEquals(HyperUniqueExpressions.TYPE, eval.type());
    Assert.assertTrue(eval.value() instanceof HyperLogLogCollector);
    Assert.assertEquals(1.0, ((HyperLogLogCollector) eval.value()).estimateCardinality(), 0.01);
    expr = Parser.parse("hyper_unique_add(nullLong, hyper_unique())", MACRO_TABLE);
    eval = expr.eval(inputBindings);
    Assert.assertEquals(HyperUniqueExpressions.TYPE, eval.type());
    Assert.assertTrue(eval.value() instanceof HyperLogLogCollector);
    Assert.assertEquals(NullHandling.replaceWithDefault() ? 1.0 : 0.0, ((HyperLogLogCollector) eval.value()).estimateCardinality(), 0.01);
}

Example 35 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class FloatCardinalityVectorProcessor method aggregate.

@Override
public void aggregate(ByteBuffer buf, int numRows, int[] positions, @Nullable int[] rows, int positionOffset) {
    // Save position, limit and restore later instead of allocating a new ByteBuffer object
    final int oldPosition = buf.position();
    final int oldLimit = buf.limit();
    try {
        final float[] vector = selector.getFloatVector();
        final boolean[] nullVector = selector.getNullVector();
        for (int i = 0; i < numRows; i++) {
            final int idx = rows != null ? rows[i] : i;
            if (NullHandling.replaceWithDefault() || nullVector == null || !nullVector[idx]) {
                final int position = positions[i] + positionOffset;
                buf.limit(position + HyperLogLogCollector.getLatestNumBytesForDenseStorage());
                buf.position(position);
                final HyperLogLogCollector collector = HyperLogLogCollector.makeCollector(buf);
                FloatCardinalityAggregatorColumnSelectorStrategy.addFloatToCollector(collector, vector[idx]);
            }
        }
    } finally {
        buf.limit(oldLimit);
        buf.position(oldPosition);
    }
}