Examples with BulletRecord com.yahoo.bullet.record.BulletRecord used on opensource projects

Example 21 with BulletRecord

use of com.yahoo.bullet.record.BulletRecord in project bullet-core by yahoo.

the class QuantileSketchTest method testOnePointDistribution.

@Test
public void testOnePointDistribution() {
    QuantileSketch sketch = new QuantileSketch(64, 2, Distribution.Type.QUANTILE, 1);
    // Insert 10, 20 ... 100
    IntStream.range(1, 11).forEach(i -> sketch.update(i * 10.0));
    List<BulletRecord> records = sketch.getResult(null, null).getRecords();
    Assert.assertEquals(records.size(), 1);
    BulletRecord actual = records.get(0);
    BulletRecord expected = RecordBox.get().add(QUANTILE_FIELD, 0.0).add(VALUE_FIELD, 10.0).getRecord();
    Assert.assertEquals(actual, expected);
    QuantileSketch anotherSketch = new QuantileSketch(64, Distribution.Type.QUANTILE, makePoints(1.0));
    IntStream.range(1, 11).forEach(i -> anotherSketch.update(i * 10.0));
    records = anotherSketch.getResult(null, null).getRecords();
    Assert.assertEquals(records.size(), 1);
    actual = records.get(0);
    expected = RecordBox.get().add(QUANTILE_FIELD, 1.0).add(VALUE_FIELD, 100.0).getRecord();
    Assert.assertEquals(actual, expected);
}

Example 22 with BulletRecord

use of com.yahoo.bullet.record.BulletRecord in project bullet-core by yahoo.

the class QuantileSketchTest method testNoDataPMFDistribution.

@Test
public void testNoDataPMFDistribution() {
    QuantileSketch sketch = new QuantileSketch(64, 2, Distribution.Type.PMF, 10);
    Clip result = sketch.getResult("meta", ALL_METADATA);
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 7);
    Assert.assertFalse((Boolean) metadata.get("isEst"));
    Assert.assertEquals(metadata.get("n"), 0L);
    Assert.assertEquals(metadata.get("min"), Double.POSITIVE_INFINITY);
    Assert.assertEquals(metadata.get("max"), Double.NEGATIVE_INFINITY);
    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 2);
    BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + Double.POSITIVE_INFINITY + END_EXCLUSIVE).add(PROBABILITY_FIELD, Double.NaN).add(COUNT_FIELD, Double.NaN).getRecord();
    BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, START_INCLUSIVE + Double.POSITIVE_INFINITY + SEPARATOR + POSITIVE_INFINITY_END).add(PROBABILITY_FIELD, Double.NaN).add(COUNT_FIELD, Double.NaN).getRecord();
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);
    Assert.assertEquals(sketch.getRecords(), records);
    Assert.assertEquals(sketch.getMetadata("meta", ALL_METADATA).asMap(), result.getMeta().asMap());
}

Example 23 with BulletRecord

use of com.yahoo.bullet.record.BulletRecord in project bullet-core by yahoo.

the class QuantileSketchTest method testNoDataQuantileDistribution.

@Test
public void testNoDataQuantileDistribution() {
    QuantileSketch sketch = new QuantileSketch(64, Distribution.Type.QUANTILE, new double[] { 0, 0.3, 1 });
    Clip result = sketch.getResult("meta", ALL_METADATA);
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 7);
    Assert.assertFalse((Boolean) metadata.get("isEst"));
    Assert.assertEquals(metadata.get("n"), 0L);
    Assert.assertEquals(metadata.get("min"), Double.POSITIVE_INFINITY);
    Assert.assertEquals(metadata.get("max"), Double.NEGATIVE_INFINITY);
    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 3);
    BulletRecord expectedA = RecordBox.get().add(QUANTILE_FIELD, 0.0).add(VALUE_FIELD, Double.POSITIVE_INFINITY).getRecord();
    BulletRecord expectedB = RecordBox.get().add(QUANTILE_FIELD, 0.3).add(VALUE_FIELD, Double.NaN).getRecord();
    BulletRecord expectedC = RecordBox.get().add(QUANTILE_FIELD, 1.0).add(VALUE_FIELD, Double.NEGATIVE_INFINITY).getRecord();
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);
    Assert.assertEquals(records.get(2), expectedC);
    Assert.assertEquals(sketch.getRecords(), records);
    Assert.assertEquals(sketch.getMetadata("meta", ALL_METADATA).asMap(), result.getMeta().asMap());
}

Example 24 with BulletRecord

use of com.yahoo.bullet.record.BulletRecord in project bullet-core by yahoo.

the class QuantileSketchTest method testApproximateQuantilesWithNumberOfPoints.

@Test
public void testApproximateQuantilesWithNumberOfPoints() {
    QuantileSketch sketch = new QuantileSketch(32, 2, Distribution.Type.QUANTILE, 11);
    IntStream.range(1, 101).forEach(i -> sketch.update(i * 0.1));
    Clip result = sketch.getResult("meta", ALL_METADATA);
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 7);
    Assert.assertTrue((Boolean) metadata.get("isEst"));
    Assert.assertEquals((String) metadata.get("family"), Family.QUANTILES.getFamilyName());
    Assert.assertTrue((Integer) metadata.get("size") >= 100);
    double error = DoublesSketch.getNormalizedRankError(32);
    Assert.assertEquals(metadata.get("nre"), error);
    Assert.assertEquals(metadata.get("n"), 100L);
    assertApproxEquals((Double) metadata.get("min"), 0.1);
    assertApproxEquals((Double) metadata.get("max"), 10.0);
    List<BulletRecord> records = result.getRecords();
    for (BulletRecord record : records) {
        Double quantile = (Double) record.get(QUANTILE_FIELD);
        Double value = (Double) record.get(VALUE_FIELD);
        // We input 100 values: 0.0, 0.1, ... 9.9, and our NRE is ~6.3%. This means, for e.g., that the 50th
        // percentile value is approximate and is between the true 43th and 57th percentile, or between 4.3 and 5.7
        // in our case. The NRE * 10 is the epsilon we should use our comparison with high probability.
        assertApproxEquals(value, quantile * 10, error * 10);
    }
    Assert.assertEquals(sketch.getRecords(), records);
    Assert.assertEquals(sketch.getMetadata("meta", ALL_METADATA).asMap(), result.getMeta().asMap());
}

Example 25 with BulletRecord

use of com.yahoo.bullet.record.BulletRecord in project bullet-core by yahoo.

the class QuantileSketchTest method testNoDataCDFDistribution.

@Test
public void testNoDataCDFDistribution() {
    QuantileSketch sketch = new QuantileSketch(64, 2, Distribution.Type.CDF, 10);
    Clip result = sketch.getResult("meta", ALL_METADATA);
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 7);
    Assert.assertFalse((Boolean) metadata.get("isEst"));
    Assert.assertEquals(metadata.get("n"), 0L);
    Assert.assertEquals(metadata.get("min"), Double.POSITIVE_INFINITY);
    Assert.assertEquals(metadata.get("max"), Double.NEGATIVE_INFINITY);
    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 2);
    BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + Double.POSITIVE_INFINITY + END_EXCLUSIVE).add(PROBABILITY_FIELD, Double.NaN).add(COUNT_FIELD, Double.NaN).getRecord();
    BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + POSITIVE_INFINITY_END).add(PROBABILITY_FIELD, Double.NaN).add(COUNT_FIELD, Double.NaN).getRecord();
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);
    Assert.assertEquals(sketch.getRecords(), records);
    Assert.assertEquals(sketch.getMetadata("meta", ALL_METADATA).asMap(), result.getMeta().asMap());
}