Examples with HLL org.apache.solr.util.hll.HLL used on opensource projects

Example 6 with HLL

use of org.apache.solr.util.hll.HLL in project lucene-solr by apache.

the class StatsComponentTest method testIndividualStatLocalParams.

public void testIndividualStatLocalParams() throws Exception {
    final String kpre = ExpectedStat.KPRE;
    assertU(adoc("id", "1", "a_f", "2.3", "b_f", "9.7", "a_i", "9", "foo_t", "how now brown cow"));
    assertU(commit());
    SolrCore core = h.getCore();
    SchemaField field = core.getLatestSchema().getField("a_i");
    HllOptions hllOpts = HllOptions.parseHllOptions(params("cardinality", "true"), field);
    HLL hll = hllOpts.newHLL();
    HashFunction hasher = hllOpts.getHasher();
    AVLTreeDigest tdigest = new AVLTreeDigest(100);
    // some quick sanity check assertions...
    // trivial check that we only get the exact 2 we ask for
    assertQ("ask for and get only 2 stats", req("q", "*:*", "stats", "true", "stats.field", "{!key=k mean=true min=true}a_i"), kpre + "double[@name='mean'][.='9.0']", kpre + "double[@name='min'][.='9.0']", "count(" + kpre + "*)=2");
    // for stats that are true/false, sanity check false does it's job
    assertQ("min=true & max=false: only min should come back", req("q", "*:*", "stats", "true", "stats.field", "{!key=k max=false min=true}a_i"), kpre + "double[@name='min'][.='9.0']", "count(" + kpre + "*)=1");
    assertQ("min=false: localparam stat means ignore default set, " + "but since only local param is false no stats should be returned", req("q", "*:*", "stats", "true", "stats.field", "{!key=k min=false}a_i"), // section of stats for this field should exist ...
    XPRE + "lst[@name='stats_fields']/lst[@name='k']", // ...but be empty 
    "count(" + kpre + "*)=0");
    double sum = 0;
    double sumOfSquares = 0;
    final int count = 20;
    for (int i = 0; i < count; i++) {
        int a_i = i % 10;
        assertU(adoc("id", String.valueOf(i), "a_f", "2.3", "b_f", "9.7", "a_i", String.valueOf(a_i), "foo_t", "how now brown cow"));
        tdigest.add(a_i);
        hll.addRaw(hasher.hashInt(a_i).asLong());
        sum += a_i;
        sumOfSquares += (a_i) * (a_i);
    }
    double stddev = Math.sqrt(((count * sumOfSquares) - (sum * sum)) / (20 * (count - 1.0D)));
    assertU(commit());
    ByteBuffer tdigestBuf = ByteBuffer.allocate(tdigest.smallByteSize());
    tdigest.asSmallBytes(tdigestBuf);
    byte[] hllBytes = hll.toBytes();
    EnumSet<Stat> allStats = EnumSet.allOf(Stat.class);
    final List<ExpectedStat> expected = new ArrayList<ExpectedStat>(allStats.size());
    ExpectedStat.createSimple(Stat.min, "true", "double", "0.0");
    ExpectedStat.createSimple(Stat.max, "true", "double", "9.0");
    ExpectedStat.createSimple(Stat.missing, "true", "long", "0");
    ExpectedStat.createSimple(Stat.sum, "true", "double", String.valueOf(sum));
    ExpectedStat.createSimple(Stat.count, "true", "long", String.valueOf(count));
    ExpectedStat.createSimple(Stat.mean, "true", "double", String.valueOf(sum / count));
    ExpectedStat.createSimple(Stat.sumOfSquares, "true", "double", String.valueOf(sumOfSquares));
    ExpectedStat.createSimple(Stat.stddev, "true", "double", String.valueOf(stddev));
    final String distinctValsXpath = "count(" + kpre + "arr[@name='distinctValues']/*)=10";
    ExpectedStat.create(Stat.distinctValues, "true", Collections.singletonList(distinctValsXpath), Collections.singletonList(distinctValsXpath));
    ExpectedStat.createSimple(Stat.countDistinct, "true", "long", "10");
    final String percentileShardXpath = kpre + "str[@name='percentiles'][.='" + Base64.byteArrayToBase64(tdigestBuf.array(), 0, tdigestBuf.array().length) + "']";
    final String p90 = "" + tdigest.quantile(0.90D);
    final String p99 = "" + tdigest.quantile(0.99D);
    ExpectedStat.create(Stat.percentiles, "'90, 99'", Collections.singletonList(percentileShardXpath), Arrays.asList("count(" + kpre + "lst[@name='percentiles']/*)=2", kpre + "lst[@name='percentiles']/double[@name='90.0'][.=" + p90 + "]", kpre + "lst[@name='percentiles']/double[@name='99.0'][.=" + p99 + "]"));
    final String cardinalityShardXpath = kpre + "str[@name='cardinality'][.='" + Base64.byteArrayToBase64(hllBytes, 0, hllBytes.length) + "']";
    final String cardinalityXpath = kpre + "long[@name='cardinality'][.='10']";
    ExpectedStat.create(Stat.cardinality, "true", Collections.singletonList(cardinalityShardXpath), Collections.singletonList(cardinalityXpath));
    // canary in the coal mine
    assertEquals("num of ExpectedStat doesn't match all known stats; " + "enum was updated w/o updating test?", ExpectedStat.ALL.size(), allStats.size());
    // whitebox test: explicitly ask for isShard=true with each individual stat
    for (ExpectedStat expect : ExpectedStat.ALL.values()) {
        Stat stat = expect.stat;
        StringBuilder exclude = new StringBuilder();
        List<String> testXpaths = new ArrayList<String>(5 + expect.perShardXpaths.size());
        testXpaths.addAll(expect.perShardXpaths);
        int numKeysExpected = 0;
        EnumSet<Stat> distribDeps = stat.getDistribDeps();
        for (Stat perShardDep : distribDeps) {
            numKeysExpected++;
            // the shard should return them since they are a dependency for the requested stat
            if (!stat.equals(perShardDep)) {
                // NOTE: this only works because all the cases where there are distribDeps
                // beyond a self dependency are simple true/false options
                exclude.append(perShardDep + "=false ");
            }
        }
        // we don't want to find anything we aren't expecting
        testXpaths.add("count(" + kpre + "*)=" + numKeysExpected);
        assertQ("ask for only " + stat + ", with isShard=true, and expect only deps: " + distribDeps, req("q", "*:*", "isShard", "true", "stats", "true", "stats.field", "{!key=k " + exclude + stat + "=" + expect.input + "}a_i"), testXpaths.toArray(new String[testXpaths.size()]));
    }
    // test all the possible combinations (of all possible sizes) of stats params
    for (int numParams = 1; numParams <= allStats.size(); numParams++) {
        for (EnumSet<Stat> set : new StatSetCombinations(numParams, allStats)) {
            // EnumSets use natural ordering, we want to randomize the order of the params
            List<Stat> combo = new ArrayList<Stat>(set);
            Collections.shuffle(combo, random());
            StringBuilder paras = new StringBuilder("{!key=k ");
            List<String> testXpaths = new ArrayList<String>(numParams + 5);
            int numKeysExpected = 0;
            for (Stat stat : combo) {
                ExpectedStat expect = ExpectedStat.ALL.get(stat);
                paras.append(stat + "=" + expect.input + " ");
                numKeysExpected++;
                testXpaths.addAll(expect.finalXpaths);
            }
            paras.append("}a_i");
            // we don't want to find anything we aren't expecting
            testXpaths.add("count(" + kpre + "*)=" + numKeysExpected);
            assertQ("ask for and get only: " + combo, req("q", "*:*", "stats", "true", "stats.field", paras.toString()), testXpaths.toArray(new String[testXpaths.size()]));
        }
    }
}

Example 7 with HLL

use of org.apache.solr.util.hll.HLL in project lucene-solr by apache.

the class StringStatsValues method accumulate.

/**
   * {@inheritDoc}
   */
@Override
public void accumulate(NamedList stv) {
    if (computeCount) {
        count += (Long) stv.get("count");
    }
    if (computeMissing) {
        missing += (Long) stv.get("missing");
    }
    if (computeCalcDistinct) {
        distinctValues.addAll((Collection<T>) stv.get("distinctValues"));
        countDistinct = distinctValues.size();
    }
    if (computeMinOrMax) {
        updateMinMax((T) stv.get("min"), (T) stv.get("max"));
    }
    if (computeCardinality) {
        byte[] data = (byte[]) stv.get("cardinality");
        HLL other = HLL.fromBytes(data);
        if (hll.getType().equals(HLLType.EMPTY)) {
            // The HLL.union method goes out of it's way not to modify the "other" HLL.
            // Which means in the case of merging into an "EMPTY" HLL (garunteed to happen at
            // least once in every coordination of shard requests) it always clones all
            // of the internal storage -- but since we're going to throw "other" away after
            // the merge, this just means a short term doubling of RAM that we can skip.
            hll = other;
        } else {
            hll.union(other);
        }
    }
    updateTypeSpecificStats(stv);
    NamedList f = (NamedList) stv.get(FACETS);
    if (f == null) {
        return;
    }
    for (int i = 0; i < f.size(); i++) {
        String field = f.getName(i);
        NamedList vals = (NamedList) f.getVal(i);
        Map<String, StatsValues> addTo = facets.get(field);
        if (addTo == null) {
            addTo = new HashMap<>();
            facets.put(field, addTo);
        }
        for (int j = 0; j < vals.size(); j++) {
            String val = vals.getName(j);
            StatsValues vvals = addTo.get(val);
            if (vvals == null) {
                vvals = StatsValuesFactory.createStatsValues(statsField);
                addTo.put(val, vvals);
            }
            vvals.accumulate((NamedList) vals.getVal(j));
        }
    }
}

Example 8 with HLL

use of org.apache.solr.util.hll.HLL in project lucene-solr by apache.

the class UniqueSlotAcc method getShardHLL.

private Object getShardHLL(int slot) throws IOException {
    FixedBitSet ords = arr[slot];
    if (ords == null)
        return HLLAgg.NO_VALUES;
    HLL hll = factory.getHLL();
    long maxOrd = ords.length();
    Hash.LongPair hashResult = new Hash.LongPair();
    for (int ord = -1; ++ord < maxOrd; ) {
        ord = ords.nextSetBit(ord);
        if (ord == DocIdSetIterator.NO_MORE_DOCS)
            break;
        BytesRef val = lookupOrd(ord);
        // way to avoid recomputing hash across slots?  Prob not worth space
        Hash.murmurhash3_x64_128(val.bytes, val.offset, val.length, 0, hashResult);
        // idea: if the set is small enough, just send the hashes?  We can add at the top
        // level or even just do a hash table at the top level.
        hll.addRaw(hashResult.val1);
    }
    SimpleOrderedMap map = new SimpleOrderedMap();
    map.add("hll", hll.toBytes());
    return map;
}