Examples with HyperLogLogCollector org.apache.druid.hll.HyperLogLogCollector used on opensource projects

Example 36 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class MultiValueStringCardinalityVectorProcessor method aggregate.

@Override
public void aggregate(ByteBuffer buf, int numRows, int[] positions, @Nullable int[] rows, int positionOffset) {
    // Save position, limit and restore later instead of allocating a new ByteBuffer object
    final int oldPosition = buf.position();
    final int oldLimit = buf.limit();
    try {
        final IndexedInts[] vector = selector.getRowVector();
        for (int i = 0; i < numRows; i++) {
            final IndexedInts ids = vector[rows != null ? rows[i] : i];
            final int sz = ids.size();
            for (int j = 0; j < sz; j++) {
                final String s = selector.lookupName(ids.get(j));
                if (NullHandling.replaceWithDefault() || s != null) {
                    final int position = positions[i] + positionOffset;
                    buf.limit(position + HyperLogLogCollector.getLatestNumBytesForDenseStorage());
                    buf.position(position);
                    final HyperLogLogCollector collector = HyperLogLogCollector.makeCollector(buf);
                    StringCardinalityAggregatorColumnSelectorStrategy.addStringToCollector(collector, s);
                }
            }
        }
    } finally {
        buf.limit(oldLimit);
        buf.position(oldPosition);
    }
}

Example 37 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class StringObjectCardinalityVectorProcessor method aggregate.

@Override
public void aggregate(ByteBuffer buf, int numRows, int[] positions, @Nullable int[] rows, int positionOffset) {
    // Save position, limit and restore later instead of allocating a new ByteBuffer object
    final int oldPosition = buf.position();
    final int oldLimit = buf.limit();
    try {
        final Object[] vector = selector.getObjectVector();
        for (int i = 0; i < numRows; i++) {
            final Object obj = vector[rows != null ? rows[i] : i];
            if (NullHandling.replaceWithDefault() || obj != null) {
                final int position = positions[i] + positionOffset;
                buf.limit(position + HyperLogLogCollector.getLatestNumBytesForDenseStorage());
                buf.position(position);
                final HyperLogLogCollector collector = HyperLogLogCollector.makeCollector(buf);
                addObjectIfString(collector, obj);
            }
        }
    } finally {
        buf.limit(oldLimit);
        buf.position(oldPosition);
    }
}

Example 38 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class StringObjectCardinalityVectorProcessor method aggregate.

@Override
public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) {
    // Save position, limit and restore later instead of allocating a new ByteBuffer object
    final int oldPosition = buf.position();
    final int oldLimit = buf.limit();
    try {
        final Object[] vector = selector.getObjectVector();
        buf.limit(position + HyperLogLogCollector.getLatestNumBytesForDenseStorage());
        buf.position(position);
        final HyperLogLogCollector collector = HyperLogLogCollector.makeCollector(buf);
        for (int i = startRow; i < endRow; i++) {
            addObjectIfString(collector, vector[i]);
        }
    } finally {
        buf.limit(oldLimit);
        buf.position(oldPosition);
    }
}

Example 39 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class CardinalityVectorAggregatorTest method testAggregateStyle2.

private static void testAggregateStyle2(final CardinalityVectorAggregator aggregator, final int numRows, final double expectedResult) {
    final int positionOffset = 1;
    final int aggregatorSize = HyperLogLogCollector.getLatestNumBytesForDenseStorage();
    final ByteBuffer buf = ByteBuffer.allocate(positionOffset + 2 * aggregatorSize);
    aggregator.init(buf, positionOffset);
    aggregator.init(buf, positionOffset + aggregatorSize);
    final int[] positions = new int[numRows];
    final int[] rows = new int[numRows];
    for (int i = 0; i < numRows; i++) {
        positions[i] = (i % 2) * aggregatorSize;
        rows[i] = (i + 1) % numRows;
    }
    aggregator.aggregate(buf, numRows, positions, rows, positionOffset);
    Assert.assertEquals("style2", expectedResult, ((HyperLogLogCollector) aggregator.get(buf, positionOffset)).fold((HyperLogLogCollector) aggregator.get(buf, positionOffset + aggregatorSize)).estimateCardinality(), 0.01);
}

Example 40 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class MetricManipulatorFnsTest method constructorFeeder.

@Parameterized.Parameters(name = "{0}")
public static Iterable<Object[]> constructorFeeder() {
    final ArrayList<Object[]> constructorArrays = new ArrayList<>();
    final long longVal = 13789;
    LongMinAggregator longMinAggregator = new LongMinAggregator(new TestLongColumnSelector() {

        @Override
        public long getLong() {
            return longVal;
        }

        @Override
        public boolean isNull() {
            return false;
        }
    });
    LongMinAggregatorFactory longMinAggregatorFactory = new LongMinAggregatorFactory(NAME, FIELD);
    constructorArrays.add(new Object[] { longMinAggregatorFactory, longMinAggregator, longMinAggregator, longMinAggregator, longVal, longVal });
    HyperUniquesAggregatorFactory hyperUniquesAggregatorFactory = new HyperUniquesAggregatorFactory(NAME, FIELD);
    HyperLogLogCollector collector = HyperLogLogCollector.makeLatestCollector();
    collector.add((short) 1, (byte) 5);
    constructorArrays.add(new Object[] { hyperUniquesAggregatorFactory, collector, collector, collector.estimateCardinality(), collector.toByteArray(), collector });
    LongSumAggregatorFactory longSumAggregatorFactory = new LongSumAggregatorFactory(NAME, FIELD);
    LongSumAggregator longSumAggregator = new LongSumAggregator(new TestLongColumnSelector() {

        @Override
        public long getLong() {
            return longVal;
        }

        @Override
        public boolean isNull() {
            return false;
        }
    });
    constructorArrays.add(new Object[] { longSumAggregatorFactory, longSumAggregator, longSumAggregator, longSumAggregator, longVal, longVal });
    for (Object[] argList : constructorArrays) {
        Assert.assertEquals(StringUtils.format("Arglist %s is too short. Expected 6 found %d", Arrays.toString(argList), argList.length), 6, argList.length);
    }
    return constructorArrays;
}