Examples with HyperLogLogCollector org.apache.druid.hll.HyperLogLogCollector used on opensource projects

Example 21 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class IndexTask method collectIntervalsAndShardSpecs.

private Map<Interval, Optional<HyperLogLogCollector>> collectIntervalsAndShardSpecs(ObjectMapper jsonMapper, IndexIngestionSpec ingestionSchema, InputSource inputSource, File tmpDir, GranularitySpec granularitySpec, @Nonnull PartitionsSpec partitionsSpec, boolean determineIntervals) throws IOException {
    final Map<Interval, Optional<HyperLogLogCollector>> hllCollectors = new TreeMap<>(Comparators.intervalsByStartThenEnd());
    final Granularity queryGranularity = granularitySpec.getQueryGranularity();
    final Predicate<InputRow> rowFilter = inputRow -> {
        if (inputRow == null) {
            return false;
        }
        if (determineIntervals) {
            return true;
        }
        final Optional<Interval> optInterval = granularitySpec.bucketInterval(inputRow.getTimestamp());
        return optInterval.isPresent();
    };
    try (final CloseableIterator<InputRow> inputRowIterator = AbstractBatchIndexTask.inputSourceReader(tmpDir, ingestionSchema.getDataSchema(), inputSource, inputSource.needsFormat() ? getInputFormat(ingestionSchema) : null, rowFilter, determinePartitionsMeters, determinePartitionsParseExceptionHandler)) {
        while (inputRowIterator.hasNext()) {
            final InputRow inputRow = inputRowIterator.next();
            final Interval interval;
            if (determineIntervals) {
                interval = granularitySpec.getSegmentGranularity().bucket(inputRow.getTimestamp());
            } else {
                final Optional<Interval> optInterval = granularitySpec.bucketInterval(inputRow.getTimestamp());
                // this interval must exist since it passed the rowFilter
                assert optInterval.isPresent();
                interval = optInterval.get();
            }
            if (partitionsSpec.needsDeterminePartitions(false)) {
                hllCollectors.computeIfAbsent(interval, intv -> Optional.of(HyperLogLogCollector.makeLatestCollector()));
                List<Object> groupKey = Rows.toGroupKey(queryGranularity.bucketStart(inputRow.getTimestampFromEpoch()), inputRow);
                hllCollectors.get(interval).get().add(HASH_FUNCTION.hashBytes(jsonMapper.writeValueAsBytes(groupKey)).asBytes());
            } else {
                // we don't need to determine partitions but we still need to determine intervals, so add an Optional.absent()
                // for the interval and don't instantiate a HLL collector
                hllCollectors.putIfAbsent(interval, Optional.absent());
            }
            determinePartitionsMeters.incrementProcessed();
        }
    }
    // These metrics are reported in generateAndPublishSegments()
    if (determinePartitionsMeters.getThrownAway() > 0) {
        log.warn("Unable to find a matching interval for [%,d] events", determinePartitionsMeters.getThrownAway());
    }
    if (determinePartitionsMeters.getUnparseable() > 0) {
        log.warn("Unable to parse [%,d] events", determinePartitionsMeters.getUnparseable());
    }
    return hllCollectors;
}

Example 22 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class IndexTask method createShardSpecsFromInput.

private PartitionAnalysis createShardSpecsFromInput(ObjectMapper jsonMapper, IndexIngestionSpec ingestionSchema, InputSource inputSource, File tmpDir, GranularitySpec granularitySpec, @Nonnull PartitionsSpec partitionsSpec, boolean determineIntervals) throws IOException {
    assert partitionsSpec.getType() != SecondaryPartitionType.RANGE;
    long determineShardSpecsStartMillis = System.currentTimeMillis();
    final Map<Interval, Optional<HyperLogLogCollector>> hllCollectors = collectIntervalsAndShardSpecs(jsonMapper, ingestionSchema, inputSource, tmpDir, granularitySpec, partitionsSpec, determineIntervals);
    final PartitionAnalysis<Integer, ?> partitionAnalysis;
    if (partitionsSpec.getType() == SecondaryPartitionType.LINEAR) {
        partitionAnalysis = new LinearPartitionAnalysis((DynamicPartitionsSpec) partitionsSpec);
    } else if (partitionsSpec.getType() == SecondaryPartitionType.HASH) {
        partitionAnalysis = new HashPartitionAnalysis((HashedPartitionsSpec) partitionsSpec);
    } else {
        throw new UOE("%s", partitionsSpec.getClass().getName());
    }
    for (final Map.Entry<Interval, Optional<HyperLogLogCollector>> entry : hllCollectors.entrySet()) {
        final Interval interval = entry.getKey();
        final int numBucketsPerInterval;
        if (partitionsSpec.getType() == SecondaryPartitionType.HASH) {
            final HashedPartitionsSpec hashedPartitionsSpec = (HashedPartitionsSpec) partitionsSpec;
            final HyperLogLogCollector collector = entry.getValue().orNull();
            if (partitionsSpec.needsDeterminePartitions(false)) {
                final long numRows = Preconditions.checkNotNull(collector, "HLL collector").estimateCardinalityRound();
                final int nonNullMaxRowsPerSegment = partitionsSpec.getMaxRowsPerSegment() == null ? PartitionsSpec.DEFAULT_MAX_ROWS_PER_SEGMENT : partitionsSpec.getMaxRowsPerSegment();
                numBucketsPerInterval = (int) Math.ceil((double) numRows / nonNullMaxRowsPerSegment);
                log.info("Estimated [%,d] rows of data for interval [%s], creating [%,d] shards", numRows, interval, numBucketsPerInterval);
            } else {
                numBucketsPerInterval = hashedPartitionsSpec.getNumShards() == null ? 1 : hashedPartitionsSpec.getNumShards();
                log.info("Creating [%,d] buckets for interval [%s]", numBucketsPerInterval, interval);
            }
        } else {
            numBucketsPerInterval = 1;
        }
        partitionAnalysis.updateBucket(interval, numBucketsPerInterval);
    }
    log.info("Found intervals and shardSpecs in %,dms", System.currentTimeMillis() - determineShardSpecsStartMillis);
    return partitionAnalysis;
}

Example 23 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class HyperUniquesSerdeForTest method getObjectStrategy.

@Override
public ObjectStrategy getObjectStrategy() {
    return new ObjectStrategy<HyperLogLogCollector>() {

        @Override
        public Class<? extends HyperLogLogCollector> getClazz() {
            return HyperLogLogCollector.class;
        }

        @Override
        public HyperLogLogCollector fromByteBuffer(ByteBuffer buffer, int numBytes) {
            final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer();
            readOnlyBuffer.limit(readOnlyBuffer.position() + numBytes);
            return HyperLogLogCollector.makeCollector(readOnlyBuffer);
        }

        @Override
        public byte[] toBytes(HyperLogLogCollector collector) {
            if (collector == null) {
                return ByteArrays.EMPTY_ARRAY;
            }
            ByteBuffer val = collector.toByteBuffer();
            byte[] retVal = new byte[val.remaining()];
            val.asReadOnlyBuffer().get(retVal);
            return retVal;
        }

        @Override
        public int compare(HyperLogLogCollector o1, HyperLogLogCollector o2) {
            return comparator.compare(o1, o2);
        }
    };
}

Example 24 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class DimensionCardinalityReportTest method setup.

@Before
public void setup() {
    Interval interval = Intervals.ETERNITY;
    HyperLogLogCollector collector = HyperLogLogCollector.makeLatestCollector();
    Map<Interval, byte[]> intervalToCardinality = Collections.singletonMap(interval, collector.toByteArray());
    String taskId = "abc";
    target = new DimensionCardinalityReport(taskId, intervalToCardinality);
}

Example 25 with HyperLogLogCollector

use of org.apache.druid.hll.HyperLogLogCollector in project druid by druid-io.

the class HyperUniqueFinalizingPostAggregatorTest method testComputeRounded.

@Test
public void testComputeRounded() {
    Random random = new Random(0L);
    HyperUniqueFinalizingPostAggregator postAggregator = new HyperUniqueFinalizingPostAggregator("uniques", "uniques").decorate(ImmutableMap.of("uniques", new CardinalityAggregatorFactory("uniques", null, Collections.singletonList(DefaultDimensionSpec.of("dummy")), false, true)));
    HyperLogLogCollector collector = HyperLogLogCollector.makeLatestCollector();
    for (int i = 0; i < 100; ++i) {
        byte[] hashedVal = fn.hashLong(random.nextLong()).asBytes();
        collector.add(hashedVal);
    }
    Object cardinality = postAggregator.compute(ImmutableMap.of("uniques", collector));
    Assert.assertThat(cardinality, CoreMatchers.instanceOf(Long.class));
    Assert.assertEquals(99L, cardinality);
}