Examples with AVLTreeDigest com.tdunning.math.stats.AVLTreeDigest used on opensource projects

Example 1 with AVLTreeDigest

use of com.tdunning.math.stats.AVLTreeDigest in project lucene-solr by apache.

the class TestJsonFacets method XtestTDigest.

/*** test code to ensure TDigest is working as we expect. */
public void XtestTDigest() throws Exception {
    AVLTreeDigest t1 = new AVLTreeDigest(100);
    t1.add(10, 1);
    t1.add(90, 1);
    t1.add(50, 1);
    System.out.println(t1.quantile(0.1));
    System.out.println(t1.quantile(0.5));
    System.out.println(t1.quantile(0.9));
    assertEquals(t1.quantile(0.5), 50.0, 0.01);
    AVLTreeDigest t2 = new AVLTreeDigest(100);
    t2.add(130, 1);
    t2.add(170, 1);
    t2.add(90, 1);
    System.out.println(t2.quantile(0.1));
    System.out.println(t2.quantile(0.5));
    System.out.println(t2.quantile(0.9));
    AVLTreeDigest top = new AVLTreeDigest(100);
    t1.compress();
    // upper bound
    ByteBuffer buf = ByteBuffer.allocate(t1.byteSize());
    t1.asSmallBytes(buf);
    byte[] arr1 = Arrays.copyOf(buf.array(), buf.position());
    ByteBuffer rbuf = ByteBuffer.wrap(arr1);
    top.add(AVLTreeDigest.fromBytes(rbuf));
    System.out.println(top.quantile(0.1));
    System.out.println(top.quantile(0.5));
    System.out.println(top.quantile(0.9));
    t2.compress();
    // upper bound
    ByteBuffer buf2 = ByteBuffer.allocate(t2.byteSize());
    t2.asSmallBytes(buf2);
    byte[] arr2 = Arrays.copyOf(buf2.array(), buf2.position());
    ByteBuffer rbuf2 = ByteBuffer.wrap(arr2);
    top.add(AVLTreeDigest.fromBytes(rbuf2));
    System.out.println(top.quantile(0.1));
    System.out.println(top.quantile(0.5));
    System.out.println(top.quantile(0.9));
}

Example 2 with AVLTreeDigest

use of com.tdunning.math.stats.AVLTreeDigest in project lucene-solr by apache.

the class TestJsonFacets method XtestPercentiles.

public void XtestPercentiles() {
    AVLTreeDigest catA = new AVLTreeDigest(100);
    catA.add(4);
    catA.add(2);
    AVLTreeDigest catB = new AVLTreeDigest(100);
    catB.add(-9);
    catB.add(11);
    catB.add(-5);
    AVLTreeDigest all = new AVLTreeDigest(100);
    all.add(catA);
    all.add(catB);
    System.out.println(str(catA));
    System.out.println(str(catB));
    System.out.println(str(all));
// 2.0 2.2 3.0 3.8 4.0
// -9.0 -8.2 -5.0 7.800000000000001 11.0
// -9.0 -7.3999999999999995 2.0 8.200000000000001 11.0
}

Example 3 with AVLTreeDigest

use of com.tdunning.math.stats.AVLTreeDigest in project lucene-solr by apache.

the class StatsComponentTest method testIndividualStatLocalParams.

public void testIndividualStatLocalParams() throws Exception {
    final String kpre = ExpectedStat.KPRE;
    assertU(adoc("id", "1", "a_f", "2.3", "b_f", "9.7", "a_i", "9", "foo_t", "how now brown cow"));
    assertU(commit());
    SolrCore core = h.getCore();
    SchemaField field = core.getLatestSchema().getField("a_i");
    HllOptions hllOpts = HllOptions.parseHllOptions(params("cardinality", "true"), field);
    HLL hll = hllOpts.newHLL();
    HashFunction hasher = hllOpts.getHasher();
    AVLTreeDigest tdigest = new AVLTreeDigest(100);
    // some quick sanity check assertions...
    // trivial check that we only get the exact 2 we ask for
    assertQ("ask for and get only 2 stats", req("q", "*:*", "stats", "true", "stats.field", "{!key=k mean=true min=true}a_i"), kpre + "double[@name='mean'][.='9.0']", kpre + "double[@name='min'][.='9.0']", "count(" + kpre + "*)=2");
    // for stats that are true/false, sanity check false does it's job
    assertQ("min=true & max=false: only min should come back", req("q", "*:*", "stats", "true", "stats.field", "{!key=k max=false min=true}a_i"), kpre + "double[@name='min'][.='9.0']", "count(" + kpre + "*)=1");
    assertQ("min=false: localparam stat means ignore default set, " + "but since only local param is false no stats should be returned", req("q", "*:*", "stats", "true", "stats.field", "{!key=k min=false}a_i"), // section of stats for this field should exist ...
    XPRE + "lst[@name='stats_fields']/lst[@name='k']", // ...but be empty 
    "count(" + kpre + "*)=0");
    double sum = 0;
    double sumOfSquares = 0;
    final int count = 20;
    for (int i = 0; i < count; i++) {
        int a_i = i % 10;
        assertU(adoc("id", String.valueOf(i), "a_f", "2.3", "b_f", "9.7", "a_i", String.valueOf(a_i), "foo_t", "how now brown cow"));
        tdigest.add(a_i);
        hll.addRaw(hasher.hashInt(a_i).asLong());
        sum += a_i;
        sumOfSquares += (a_i) * (a_i);
    }
    double stddev = Math.sqrt(((count * sumOfSquares) - (sum * sum)) / (20 * (count - 1.0D)));
    assertU(commit());
    ByteBuffer tdigestBuf = ByteBuffer.allocate(tdigest.smallByteSize());
    tdigest.asSmallBytes(tdigestBuf);
    byte[] hllBytes = hll.toBytes();
    EnumSet<Stat> allStats = EnumSet.allOf(Stat.class);
    final List<ExpectedStat> expected = new ArrayList<ExpectedStat>(allStats.size());
    ExpectedStat.createSimple(Stat.min, "true", "double", "0.0");
    ExpectedStat.createSimple(Stat.max, "true", "double", "9.0");
    ExpectedStat.createSimple(Stat.missing, "true", "long", "0");
    ExpectedStat.createSimple(Stat.sum, "true", "double", String.valueOf(sum));
    ExpectedStat.createSimple(Stat.count, "true", "long", String.valueOf(count));
    ExpectedStat.createSimple(Stat.mean, "true", "double", String.valueOf(sum / count));
    ExpectedStat.createSimple(Stat.sumOfSquares, "true", "double", String.valueOf(sumOfSquares));
    ExpectedStat.createSimple(Stat.stddev, "true", "double", String.valueOf(stddev));
    final String distinctValsXpath = "count(" + kpre + "arr[@name='distinctValues']/*)=10";
    ExpectedStat.create(Stat.distinctValues, "true", Collections.singletonList(distinctValsXpath), Collections.singletonList(distinctValsXpath));
    ExpectedStat.createSimple(Stat.countDistinct, "true", "long", "10");
    final String percentileShardXpath = kpre + "str[@name='percentiles'][.='" + Base64.byteArrayToBase64(tdigestBuf.array(), 0, tdigestBuf.array().length) + "']";
    final String p90 = "" + tdigest.quantile(0.90D);
    final String p99 = "" + tdigest.quantile(0.99D);
    ExpectedStat.create(Stat.percentiles, "'90, 99'", Collections.singletonList(percentileShardXpath), Arrays.asList("count(" + kpre + "lst[@name='percentiles']/*)=2", kpre + "lst[@name='percentiles']/double[@name='90.0'][.=" + p90 + "]", kpre + "lst[@name='percentiles']/double[@name='99.0'][.=" + p99 + "]"));
    final String cardinalityShardXpath = kpre + "str[@name='cardinality'][.='" + Base64.byteArrayToBase64(hllBytes, 0, hllBytes.length) + "']";
    final String cardinalityXpath = kpre + "long[@name='cardinality'][.='10']";
    ExpectedStat.create(Stat.cardinality, "true", Collections.singletonList(cardinalityShardXpath), Collections.singletonList(cardinalityXpath));
    // canary in the coal mine
    assertEquals("num of ExpectedStat doesn't match all known stats; " + "enum was updated w/o updating test?", ExpectedStat.ALL.size(), allStats.size());
    // whitebox test: explicitly ask for isShard=true with each individual stat
    for (ExpectedStat expect : ExpectedStat.ALL.values()) {
        Stat stat = expect.stat;
        StringBuilder exclude = new StringBuilder();
        List<String> testXpaths = new ArrayList<String>(5 + expect.perShardXpaths.size());
        testXpaths.addAll(expect.perShardXpaths);
        int numKeysExpected = 0;
        EnumSet<Stat> distribDeps = stat.getDistribDeps();
        for (Stat perShardDep : distribDeps) {
            numKeysExpected++;
            // the shard should return them since they are a dependency for the requested stat
            if (!stat.equals(perShardDep)) {
                // NOTE: this only works because all the cases where there are distribDeps
                // beyond a self dependency are simple true/false options
                exclude.append(perShardDep + "=false ");
            }
        }
        // we don't want to find anything we aren't expecting
        testXpaths.add("count(" + kpre + "*)=" + numKeysExpected);
        assertQ("ask for only " + stat + ", with isShard=true, and expect only deps: " + distribDeps, req("q", "*:*", "isShard", "true", "stats", "true", "stats.field", "{!key=k " + exclude + stat + "=" + expect.input + "}a_i"), testXpaths.toArray(new String[testXpaths.size()]));
    }
    // test all the possible combinations (of all possible sizes) of stats params
    for (int numParams = 1; numParams <= allStats.size(); numParams++) {
        for (EnumSet<Stat> set : new StatSetCombinations(numParams, allStats)) {
            // EnumSets use natural ordering, we want to randomize the order of the params
            List<Stat> combo = new ArrayList<Stat>(set);
            Collections.shuffle(combo, random());
            StringBuilder paras = new StringBuilder("{!key=k ");
            List<String> testXpaths = new ArrayList<String>(numParams + 5);
            int numKeysExpected = 0;
            for (Stat stat : combo) {
                ExpectedStat expect = ExpectedStat.ALL.get(stat);
                paras.append(stat + "=" + expect.input + " ");
                numKeysExpected++;
                testXpaths.addAll(expect.finalXpaths);
            }
            paras.append("}a_i");
            // we don't want to find anything we aren't expecting
            testXpaths.add("count(" + kpre + "*)=" + numKeysExpected);
            assertQ("ask for and get only: " + combo, req("q", "*:*", "stats", "true", "stats.field", paras.toString()), testXpaths.toArray(new String[testXpaths.size()]));
        }
    }
}