Examples with HyperLogLogCollector org.apache.druid.hll.HyperLogLogCollector used on opensource projects

Example 26 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class HyperUniqueFinalizingPostAggregatorTest method testCompute.

@Test
public void testCompute() {
    Random random = new Random(0L);
    HyperUniqueFinalizingPostAggregator postAggregator = new HyperUniqueFinalizingPostAggregator("uniques", "uniques");
    HyperLogLogCollector collector = HyperLogLogCollector.makeLatestCollector();
    for (int i = 0; i < 100; ++i) {
        byte[] hashedVal = fn.hashLong(random.nextLong()).asBytes();
        collector.add(hashedVal);
    }
    double cardinality = (Double) postAggregator.compute(ImmutableMap.of("uniques", collector));
    Assert.assertTrue(cardinality == 99.37233005831612);
}

Example 27 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class HyperUniquesAggregatorFactoryTest method testCompare1.

@Test
public void testCompare1() {
    HyperLogLogCollector collector1 = HyperLogLogCollector.makeLatestCollector();
    HyperLogLogCollector collector2 = HyperLogLogCollector.makeLatestCollector();
    collector1.add(fn.hashLong(0).asBytes());
    HyperUniquesAggregatorFactory factory = new HyperUniquesAggregatorFactory("foo", "bar");
    Comparator comparator = factory.getComparator();
    for (int i = 1; i < 100; i = i + 2) {
        collector1.add(fn.hashLong(i).asBytes());
        collector2.add(fn.hashLong(i + 1).asBytes());
        Assert.assertEquals(1, comparator.compare(collector1, collector2));
        Assert.assertEquals(1, Double.compare(collector1.estimateCardinality(), collector2.estimateCardinality()));
    }
}

Example 28 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class TopNQueryQueryToolChestTest method doTestCacheStrategyOrderByPost.

private void doTestCacheStrategyOrderByPost(final ColumnType valueType, final Object dimValue) throws IOException {
    CacheStrategy<Result<TopNResultValue>, Object, TopNQuery> strategy = new TopNQueryQueryToolChest(null, null).getCacheStrategy(new TopNQuery(new TableDataSource("dummy"), VirtualColumns.EMPTY, new DefaultDimensionSpec("test", "test", valueType), new NumericTopNMetricSpec("post"), 3, new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("2015-01-01/2015-01-02"))), null, Granularities.ALL, ImmutableList.of(new HyperUniquesAggregatorFactory("metric1", "test", false, false), new CountAggregatorFactory("metric2")), ImmutableList.of(new ArithmeticPostAggregator("post", "+", ImmutableList.of(new FinalizingFieldAccessPostAggregator("metric1", "metric1"), new FieldAccessPostAggregator("metric2", "metric2")))), null));
    HyperLogLogCollector collector = getIntermediateHllCollector(valueType.getType(), dimValue);
    final Result<TopNResultValue> result1 = new Result<>(// test timestamps that result in integer size millis
    DateTimes.utc(123L), new TopNResultValue(Collections.singletonList(ImmutableMap.of("test", dimValue, "metric1", collector, "metric2", 2, "post", collector.estimateCardinality() + 2))));
    Object preparedValue = strategy.prepareForSegmentLevelCache().apply(result1);
    ObjectMapper objectMapper = TestHelper.makeJsonMapper();
    Object fromCacheValue = objectMapper.readValue(objectMapper.writeValueAsBytes(preparedValue), strategy.getCacheObjectClazz());
    Result<TopNResultValue> fromCacheResult = strategy.pullFromSegmentLevelCache().apply(fromCacheValue);
    Assert.assertEquals(result1, fromCacheResult);
    final Result<TopNResultValue> resultLevelCacheResult = new Result<>(// test timestamps that result in integer size millis
    DateTimes.utc(123L), new TopNResultValue(Collections.singletonList(ImmutableMap.of("test", dimValue, "metric1", collector.estimateCardinality(), "metric2", 2, "post", collector.estimateCardinality() + 2))));
    Object preparedResultCacheValue = strategy.prepareForCache(true).apply(resultLevelCacheResult);
    Object fromResultCacheValue = objectMapper.readValue(objectMapper.writeValueAsBytes(preparedResultCacheValue), strategy.getCacheObjectClazz());
    Result<TopNResultValue> fromResultCacheResult = strategy.pullFromCache(true).apply(fromResultCacheValue);
    Assert.assertEquals(resultLevelCacheResult, fromResultCacheResult);
}

Example 29 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class DoubleCardinalityVectorProcessor method aggregate.

@Override
public void aggregate(ByteBuffer buf, int numRows, int[] positions, @Nullable int[] rows, int positionOffset) {
    // Save position, limit and restore later instead of allocating a new ByteBuffer object
    final int oldPosition = buf.position();
    final int oldLimit = buf.limit();
    try {
        final double[] vector = selector.getDoubleVector();
        final boolean[] nullVector = selector.getNullVector();
        for (int i = 0; i < numRows; i++) {
            final int idx = rows != null ? rows[i] : i;
            if (NullHandling.replaceWithDefault() || nullVector == null || !nullVector[idx]) {
                final int position = positions[i] + positionOffset;
                buf.limit(position + HyperLogLogCollector.getLatestNumBytesForDenseStorage());
                buf.position(position);
                final HyperLogLogCollector collector = HyperLogLogCollector.makeCollector(buf);
                DoubleCardinalityAggregatorColumnSelectorStrategy.addDoubleToCollector(collector, vector[idx]);
            }
        }
    } finally {
        buf.limit(oldLimit);
        buf.position(oldPosition);
    }
}

Example 30 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class LongCardinalityVectorProcessor method aggregate.

@Override
public void aggregate(ByteBuffer buf, int numRows, int[] positions, @Nullable int[] rows, int positionOffset) {
    // Save position, limit and restore later instead of allocating a new ByteBuffer object
    final int oldPosition = buf.position();
    final int oldLimit = buf.limit();
    try {
        final long[] vector = selector.getLongVector();
        final boolean[] nullVector = selector.getNullVector();
        for (int i = 0; i < numRows; i++) {
            final int idx = rows != null ? rows[i] : i;
            if (NullHandling.replaceWithDefault() || nullVector == null || !nullVector[idx]) {
                final int position = positions[i] + positionOffset;
                buf.limit(position + HyperLogLogCollector.getLatestNumBytesForDenseStorage());
                buf.position(position);
                final HyperLogLogCollector collector = HyperLogLogCollector.makeCollector(buf);
                LongCardinalityAggregatorColumnSelectorStrategy.addLongToCollector(collector, vector[idx]);
            }
        }
    } finally {
        buf.limit(oldLimit);
        buf.position(oldPosition);
    }
}