Examples with Distribution com.yahoo.bullet.query.aggregations.Distribution used on opensource projects

Example 1 with Distribution

use of com.yahoo.bullet.query.aggregations.Distribution in project bullet-core by yahoo.

the class QuantileSketchingStrategyTest method testCombining.

@Test
public void testCombining() {
    QuantileSketchingStrategy distribution = makeDistribution(DistributionType.CDF, asList(5.0, 2.5));
    IntStream.range(0, 25).mapToDouble(i -> (i * 0.1)).mapToObj(d -> RecordBox.get().add("field", d).getRecord()).forEach(distribution::consume);
    QuantileSketchingStrategy anotherDistribution = makeDistribution(DistributionType.CDF, asList(5.0, 2.5));
    IntStream.range(50, 100).mapToDouble(i -> (i * 0.1)).mapToObj(d -> RecordBox.get().add("field", d).getRecord()).forEach(anotherDistribution::consume);
    QuantileSketchingStrategy union = makeDistribution(DistributionType.CDF, asList(5.0, 2.5));
    union.combine(distribution.getData());
    union.combine(anotherDistribution.getData());
    Clip result = union.getResult();
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 7);
    Assert.assertFalse((Boolean) metadata.get("isEst"));
    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 3);
    BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 2.5 + END_EXCLUSIVE).add(COUNT_FIELD, 25.0).add(PROBABILITY_FIELD, 1.0 / 3).getRecord();
    BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 5.0 + END_EXCLUSIVE).add(COUNT_FIELD, 25.0).add(PROBABILITY_FIELD, 1.0 / 3).getRecord();
    BulletRecord expectedC = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + POSITIVE_INFINITY_END).add(COUNT_FIELD, 75.0).add(PROBABILITY_FIELD, 1.0).getRecord();
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);
    Assert.assertEquals(records.get(2), expectedC);
    Assert.assertEquals(union.getRecords(), records);
    Assert.assertEquals(union.getMetadata().asMap(), result.getMeta().asMap());
}

Example 2 with Distribution

use of com.yahoo.bullet.query.aggregations.Distribution in project bullet-core by yahoo.

the class QuantileSketchingStrategyTest method testResetting.

@Test
public void testResetting() {
    QuantileSketchingStrategy distribution = makeDistribution(DistributionType.CDF, asList(5.0, 2.5));
    IntStream.range(0, 25).mapToDouble(i -> (i * 0.1)).mapToObj(d -> RecordBox.get().add("field", d).getRecord()).forEach(distribution::consume);
    BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 2.5 + END_EXCLUSIVE).add(COUNT_FIELD, 25.0).add(PROBABILITY_FIELD, 1.0).getRecord();
    BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 5.0 + END_EXCLUSIVE).add(COUNT_FIELD, 25.0).add(PROBABILITY_FIELD, 1.0).getRecord();
    BulletRecord expectedC = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + POSITIVE_INFINITY_END).add(COUNT_FIELD, 25.0).add(PROBABILITY_FIELD, 1.0).getRecord();
    Clip result = distribution.getResult();
    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 3);
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);
    Assert.assertEquals(records.get(2), expectedC);
    Assert.assertEquals(distribution.getRecords(), records);
    Assert.assertEquals(distribution.getMetadata().asMap(), result.getMeta().asMap());
    distribution.reset();
    IntStream.range(50, 100).mapToDouble(i -> (i * 0.1)).mapToObj(d -> RecordBox.get().add("field", d).getRecord()).forEach(distribution::consume);
    expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 2.5 + END_EXCLUSIVE).add(COUNT_FIELD, 0.0).add(PROBABILITY_FIELD, 0.0).getRecord();
    expectedB = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 5.0 + END_EXCLUSIVE).add(COUNT_FIELD, 0.0).add(PROBABILITY_FIELD, 0.0).getRecord();
    expectedC = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + POSITIVE_INFINITY_END).add(COUNT_FIELD, 50.0).add(PROBABILITY_FIELD, 1.0).getRecord();
    result = distribution.getResult();
    records = result.getRecords();
    Assert.assertEquals(records.size(), 3);
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);
    Assert.assertEquals(records.get(2), expectedC);
    Assert.assertEquals(distribution.getRecords(), records);
    Assert.assertEquals(distribution.getMetadata().asMap(), result.getMeta().asMap());
}

Example 3 with Distribution

use of com.yahoo.bullet.query.aggregations.Distribution in project bullet-core by yahoo.

the class QuantileSketchingStrategyTest method testQuantiles.

@Test
public void testQuantiles() {
    QuantileSketchingStrategy distribution = makeDistribution("field", DistributionType.QUANTILE, 3);
    IntStream.range(0, 2000).mapToDouble(i -> (i * 0.1)).mapToObj(d -> RecordBox.get().add("field", d).getRecord()).forEach(distribution::consume);
    Clip result = distribution.getResult();
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 7);
    Assert.assertTrue((Boolean) metadata.get("isEst"));
    Assert.assertEquals((String) metadata.get("family"), Family.QUANTILES.getFamilyName());
    // Size should be at least 512 bytes since we inserted 2K uniques with sketch k set to 512
    Assert.assertTrue((Integer) metadata.get("size") >= 512);
    Assert.assertEquals(metadata.get("nre"), DoublesSketch.getNormalizedRankError(512));
    // 60 items
    Assert.assertEquals(metadata.get("n"), 2000L);
    Assert.assertEquals(metadata.get("min"), 0.0);
    Assert.assertEquals(metadata.get("max"), 199.9);
    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 3);
    BulletRecord expectedA = RecordBox.get().add(QUANTILE_FIELD, 0.0).add(VALUE_FIELD, 0.0).getRecord();
    BulletRecord expectedC = RecordBox.get().add(QUANTILE_FIELD, 1.0).add(VALUE_FIELD, 199.9).getRecord();
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(2), expectedC);
    BulletRecord actualB = records.get(1);
    Assert.assertEquals(actualB.typedGet(QUANTILE_FIELD).getValue(), 0.5);
    Double actualMedian = (Double) actualB.typedGet(VALUE_FIELD).getValue();
    // We insert 0,0.1, ... 199.9. Our median is around 100.0. Our NRE < 1%, so we can be pretty certain the median
    // from the sketch is around this.
    assertApproxEquals(actualMedian, 100.0, 2.0);
    Assert.assertEquals(distribution.getRecords(), result.getRecords());
    Assert.assertEquals(distribution.getMetadata().asMap(), result.getMeta().asMap());
}

Example 4 with Distribution

use of com.yahoo.bullet.query.aggregations.Distribution in project bullet-core by yahoo.

the class QuantileSketchingStrategyTest method testNegativeSize.

@Test
public void testNegativeSize() {
    // MAX_POINTS is configured to -1 and we will use the min BulletConfig.DEFAULT_DISTRIBUTION_AGGREGATION_MAX_POINTS
    // and aggregation size, which is 1
    QuantileSketchingStrategy distribution = makeDistribution(makeConfiguration(-1, 128), 1, "field", DistributionType.PMF, 10);
    IntStream.range(0, 100).mapToDouble(i -> i).mapToObj(d -> RecordBox.get().add("field", d).getRecord()).forEach(distribution::consume);
    Clip result = distribution.getResult();
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 7);
    Assert.assertFalse((Boolean) metadata.get("isEst"));
    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 2);
    BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 0.0 + END_EXCLUSIVE).add(COUNT_FIELD, 0.0).add(PROBABILITY_FIELD, 0.0).getRecord();
    BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, START_INCLUSIVE + 0.0 + SEPARATOR + POSITIVE_INFINITY_END).add(COUNT_FIELD, 100.0).add(PROBABILITY_FIELD, 1.0).getRecord();
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);
    Assert.assertEquals(distribution.getRecords(), result.getRecords());
    Assert.assertEquals(distribution.getMetadata().asMap(), result.getMeta().asMap());
}

Example 5 with Distribution

use of com.yahoo.bullet.query.aggregations.Distribution in project bullet-core by yahoo.

the class QuantileSketchingStrategyTest method testCDF.

@Test
public void testCDF() {
    QuantileSketchingStrategy distribution = makeDistribution(DistributionType.CDF, asList(5.0, 2.5));
    IntStream.range(0, 100).mapToDouble(i -> (i * 0.1)).mapToObj(d -> RecordBox.get().add("field", d).getRecord()).forEach(distribution::consume);
    Clip result = distribution.getResult();
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 7);
    Assert.assertFalse((Boolean) metadata.get("isEst"));
    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 3);
    BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 2.5 + END_EXCLUSIVE).add(COUNT_FIELD, 25.0).add(PROBABILITY_FIELD, 0.25).getRecord();
    BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 5.0 + END_EXCLUSIVE).add(COUNT_FIELD, 50.0).add(PROBABILITY_FIELD, 0.5).getRecord();
    BulletRecord expectedC = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + POSITIVE_INFINITY_END).add(COUNT_FIELD, 100.0).add(PROBABILITY_FIELD, 1.0).getRecord();
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);
    Assert.assertEquals(records.get(2), expectedC);
    Assert.assertEquals(distribution.getRecords(), result.getRecords());
    Assert.assertEquals(distribution.getMetadata().asMap(), result.getMeta().asMap());
}