Examples with HLL org.apache.solr.util.hll.HLL used on opensource projects

Example 1 with HLL

use of org.apache.solr.util.hll.HLL in project lucene-solr by apache.

the class HLLSerializationTest method manyValuesHLLSerializationTest.

/**
   * A smoke-test that covers serialization/deserialization of a (single) HLL
   * with random init params with an extremely large number of unique values added to it.
   *
   * @see #monsterHLLSerializationTest
   */
@Test
@Slow
@Monster("may require as much as -Dtests.heapsize=4g depending on random values picked")
public void manyValuesHLLSerializationTest() throws Exception {
    final HLLType[] ALL_TYPES = EnumSet.allOf(HLLType.class).toArray(new HLLType[0]);
    Arrays.sort(ALL_TYPES);
    final int log2m = TestUtil.nextInt(random(), MINIMUM_LOG2M_PARAM, MAXIMUM_LOG2M_PARAM);
    final int regwidth = TestUtil.nextInt(random(), MINIMUM_REGWIDTH_PARAM, MAXIMUM_REGWIDTH_PARAM);
    final int expthresh = TestUtil.nextInt(random(), MINIMUM_EXPTHRESH_PARAM, MAXIMUM_EXPTHRESH_PARAM);
    final boolean sparse = random().nextBoolean();
    final HLLType type = ALL_TYPES[TestUtil.nextInt(random(), 0, ALL_TYPES.length - 1)];
    HLL hll = new HLL(log2m, regwidth, expthresh, sparse, type);
    final long NUM_VALS = TestUtil.nextLong(random(), 150000, 1000000);
    final long MIN_VAL = TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE - NUM_VALS);
    final long MAX_VAL = MIN_VAL + NUM_VALS;
    assert MIN_VAL < MAX_VAL;
    for (long val = MIN_VAL; val < MAX_VAL; val++) {
        hll.addRaw(val);
    }
    final long expectedCardinality = hll.cardinality();
    final HLLType expectedType = hll.getType();
    byte[] serializedData = hll.toBytes();
    // allow some GC
    hll = null;
    HLL copy = HLL.fromBytes(serializedData);
    // allow some GC
    serializedData = null;
    assertEquals(expectedCardinality, copy.cardinality());
    assertEquals(expectedType, copy.getType());
}

Example 2 with HLL

use of org.apache.solr.util.hll.HLL in project lucene-solr by apache.

the class HLLSerializationTest method assertCardinality.

/**
   * Adds all of the items to the specified hll, then does a round trip serialize/deserialize and confirms
   * equality of several properties (including the byte serialization).  Repeats process with a clone.
   */
private static void assertCardinality(HLL hll, final Collection<Long> items) throws CloneNotSupportedException {
    for (final Long item : items) {
        hll.addRaw(item);
    }
    final long hllCardinality = hll.cardinality();
    final HLLType hllType = hll.getType();
    final byte[] hllBytes = hll.toBytes();
    // allow some GC
    hll = null;
    HLL copy = HLL.fromBytes(hllBytes);
    assertEquals(copy.cardinality(), hllCardinality);
    assertEquals(copy.getType(), hllType);
    assertTrue(Arrays.equals(copy.toBytes(), hllBytes));
    HLL clone = copy.clone();
    // allow some GC
    copy = null;
    assertEquals(clone.cardinality(), hllCardinality);
    assertEquals(clone.getType(), hllType);
    assertTrue(Arrays.equals(clone.toBytes(), hllBytes));
}

Example 3 with HLL

use of org.apache.solr.util.hll.HLL in project lucene-solr by apache.

the class HLLSerializationTest method monsterHLLSerializationTest.

/**
   * A smoke-test that covers serialization/deserialization of HLLs
   * under the max possible numeric init parameters, iterating over all possible combinations of 
   * the other params.
   *
   * @see #manyValuesHLLSerializationTest
   */
@Test
@Slow
@Monster("needs roughly -Dtests.heapsize=8g because of the (multiple) massive data structs")
public void monsterHLLSerializationTest() throws Exception {
    final Random random = new Random(randomLong());
    final int randomCount = 250;
    final List<Long> randoms = new ArrayList<Long>(randomCount);
    for (int i = 0; i < randomCount; i++) {
        randoms.add(random.nextLong());
    }
    for (HLLType type : EnumSet.allOf(HLLType.class)) {
        for (boolean sparse : new boolean[] { true, false }) {
            HLL hll = new HLL(MAXIMUM_LOG2M_PARAM, MAXIMUM_REGWIDTH_PARAM, MAXIMUM_EXPTHRESH_PARAM, sparse, type);
            assertCardinality(hll, randoms);
        }
    }
}

Example 4 with HLL

use of org.apache.solr.util.hll.HLL in project lucene-solr by apache.

the class HLLSerializationTest method manyValuesMonsterHLLSerializationTest.

/**
   * A smoke-test that covers serialization/deserialization of a (single) HLL
   * with random the max possible numeric init parameters, with randomized values for the other params.
   *
   * @see #monsterHLLSerializationTest
   */
@Test
@Slow
@Monster("can require as much as -Dtests.heapsize=4g because of the massive data structs")
public void manyValuesMonsterHLLSerializationTest() throws Exception {
    final HLLType[] ALL_TYPES = EnumSet.allOf(HLLType.class).toArray(new HLLType[0]);
    Arrays.sort(ALL_TYPES);
    final boolean sparse = random().nextBoolean();
    final HLLType type = ALL_TYPES[TestUtil.nextInt(random(), 0, ALL_TYPES.length - 1)];
    HLL hll = new HLL(MAXIMUM_LOG2M_PARAM, MAXIMUM_REGWIDTH_PARAM, MAXIMUM_EXPTHRESH_PARAM, sparse, type);
    final long NUM_VALS = TestUtil.nextLong(random(), 150000, 1000000);
    final long MIN_VAL = TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE - NUM_VALS);
    final long MAX_VAL = MIN_VAL + NUM_VALS;
    assert MIN_VAL < MAX_VAL;
    for (long val = MIN_VAL; val < MAX_VAL; val++) {
        hll.addRaw(val);
    }
    final long expectedCardinality = hll.cardinality();
    final HLLType expectedType = hll.getType();
    byte[] serializedData = hll.toBytes();
    // allow some GC
    hll = null;
    HLL copy = HLL.fromBytes(serializedData);
    // allow some GC
    serializedData = null;
    assertEquals(expectedCardinality, copy.cardinality());
    assertEquals(expectedType, copy.getType());
}

Example 5 with HLL

use of org.apache.solr.util.hll.HLL in project lucene-solr by apache.

the class TestJsonFacets method XtestHLL.

public void XtestHLL() {
    HLLAgg.HLLFactory fac = new HLLAgg.HLLFactory();
    HLL hll = fac.getHLL();
    hll.addRaw(123456789);
    hll.addRaw(987654321);
}