use of com.yahoo.bullet.aggregations.sketches.QuantileSketch.PROBABILITY_FIELD in project bullet-core by yahoo.
the class DistributionTest method testCasting.
@Test
public void testCasting() {
Distribution distribution = makeDistribution(Distribution.Type.PMF, Collections.singletonList(50.0));
IntStream.range(0, 25).mapToObj(String::valueOf).map(s -> RecordBox.get().add("field", s).getRecord()).forEach(distribution::consume);
distribution.consume(RecordBox.get().add("field", "garbage").getRecord());
distribution.consume(RecordBox.get().add("field", "1.0 garbage").getRecord());
IntStream.range(50, 100).mapToDouble(i -> i).mapToObj(d -> RecordBox.get().add("field", d).getRecord()).forEach(distribution::consume);
Clip result = distribution.getResult();
Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
Assert.assertEquals(metadata.size(), 7);
Assert.assertFalse((Boolean) metadata.get("isEst"));
List<BulletRecord> records = result.getRecords();
Assert.assertEquals(records.size(), 2);
BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 50.0 + END_EXCLUSIVE).add(COUNT_FIELD, 25.0).add(PROBABILITY_FIELD, 1.0 / 3).getRecord();
BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, START_INCLUSIVE + 50.0 + SEPARATOR + POSITIVE_INFINITY_END).add(COUNT_FIELD, 50.0).add(PROBABILITY_FIELD, 2.0 / 3).getRecord();
Assert.assertEquals(records.get(0), expectedA);
Assert.assertEquals(records.get(1), expectedB);
Assert.assertEquals(distribution.getRecords(), result.getRecords());
Assert.assertEquals(distribution.getMetadata().asMap(), result.getMeta().asMap());
}
use of com.yahoo.bullet.aggregations.sketches.QuantileSketch.PROBABILITY_FIELD in project bullet-core by yahoo.
the class DistributionTest method testCombining.
@Test
public void testCombining() {
Distribution distribution = makeDistribution(Distribution.Type.CDF, asList(5.0, 2.5));
IntStream.range(0, 25).mapToDouble(i -> (i * 0.1)).mapToObj(d -> RecordBox.get().add("field", d).getRecord()).forEach(distribution::consume);
Distribution anotherDistribution = makeDistribution(Distribution.Type.CDF, asList(5.0, 2.5));
IntStream.range(50, 100).mapToDouble(i -> (i * 0.1)).mapToObj(d -> RecordBox.get().add("field", d).getRecord()).forEach(anotherDistribution::consume);
Distribution union = makeDistribution(Distribution.Type.CDF, asList(5.0, 2.5));
union.combine(distribution.getData());
union.combine(anotherDistribution.getData());
Clip result = union.getResult();
Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
Assert.assertEquals(metadata.size(), 7);
Assert.assertFalse((Boolean) metadata.get("isEst"));
List<BulletRecord> records = result.getRecords();
Assert.assertEquals(records.size(), 3);
BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 2.5 + END_EXCLUSIVE).add(COUNT_FIELD, 25.0).add(PROBABILITY_FIELD, 1.0 / 3).getRecord();
BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 5.0 + END_EXCLUSIVE).add(COUNT_FIELD, 25.0).add(PROBABILITY_FIELD, 1.0 / 3).getRecord();
BulletRecord expectedC = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + POSITIVE_INFINITY_END).add(COUNT_FIELD, 75.0).add(PROBABILITY_FIELD, 1.0).getRecord();
Assert.assertEquals(records.get(0), expectedA);
Assert.assertEquals(records.get(1), expectedB);
Assert.assertEquals(records.get(2), expectedC);
Assert.assertEquals(union.getRecords(), records);
Assert.assertEquals(union.getMetadata().asMap(), result.getMeta().asMap());
}
use of com.yahoo.bullet.aggregations.sketches.QuantileSketch.PROBABILITY_FIELD in project bullet-core by yahoo.
the class DistributionTest method testResetting.
@Test
public void testResetting() {
Distribution distribution = makeDistribution(Distribution.Type.CDF, asList(5.0, 2.5));
IntStream.range(0, 25).mapToDouble(i -> (i * 0.1)).mapToObj(d -> RecordBox.get().add("field", d).getRecord()).forEach(distribution::consume);
BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 2.5 + END_EXCLUSIVE).add(COUNT_FIELD, 25.0).add(PROBABILITY_FIELD, 1.0).getRecord();
BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 5.0 + END_EXCLUSIVE).add(COUNT_FIELD, 25.0).add(PROBABILITY_FIELD, 1.0).getRecord();
BulletRecord expectedC = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + POSITIVE_INFINITY_END).add(COUNT_FIELD, 25.0).add(PROBABILITY_FIELD, 1.0).getRecord();
Clip result = distribution.getResult();
List<BulletRecord> records = result.getRecords();
Assert.assertEquals(records.size(), 3);
Assert.assertEquals(records.get(0), expectedA);
Assert.assertEquals(records.get(1), expectedB);
Assert.assertEquals(records.get(2), expectedC);
Assert.assertEquals(distribution.getRecords(), records);
Assert.assertEquals(distribution.getMetadata().asMap(), result.getMeta().asMap());
distribution.reset();
IntStream.range(50, 100).mapToDouble(i -> (i * 0.1)).mapToObj(d -> RecordBox.get().add("field", d).getRecord()).forEach(distribution::consume);
expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 2.5 + END_EXCLUSIVE).add(COUNT_FIELD, 0.0).add(PROBABILITY_FIELD, 0.0).getRecord();
expectedB = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 5.0 + END_EXCLUSIVE).add(COUNT_FIELD, 0.0).add(PROBABILITY_FIELD, 0.0).getRecord();
expectedC = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + POSITIVE_INFINITY_END).add(COUNT_FIELD, 50.0).add(PROBABILITY_FIELD, 1.0).getRecord();
result = distribution.getResult();
records = result.getRecords();
Assert.assertEquals(records.size(), 3);
Assert.assertEquals(records.get(0), expectedA);
Assert.assertEquals(records.get(1), expectedB);
Assert.assertEquals(records.get(2), expectedC);
Assert.assertEquals(distribution.getRecords(), records);
Assert.assertEquals(distribution.getMetadata().asMap(), result.getMeta().asMap());
}
use of com.yahoo.bullet.aggregations.sketches.QuantileSketch.PROBABILITY_FIELD in project bullet-core by yahoo.
the class DistributionTest method testPMF.
@Test
public void testPMF() {
Distribution distribution = makeDistribution(Distribution.Type.PMF, asList(5.0, 2.5));
IntStream.range(0, 100).mapToDouble(i -> (i * 0.1)).mapToObj(d -> RecordBox.get().add("field", d).getRecord()).forEach(distribution::consume);
Clip result = distribution.getResult();
Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
Assert.assertEquals(metadata.size(), 7);
Assert.assertFalse((Boolean) metadata.get("isEst"));
List<BulletRecord> records = result.getRecords();
Assert.assertEquals(records.size(), 3);
BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 2.5 + END_EXCLUSIVE).add(COUNT_FIELD, 25.0).add(PROBABILITY_FIELD, 0.25).getRecord();
BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, START_INCLUSIVE + 2.5 + SEPARATOR + 5.0 + END_EXCLUSIVE).add(COUNT_FIELD, 25.0).add(PROBABILITY_FIELD, 0.25).getRecord();
BulletRecord expectedC = RecordBox.get().add(RANGE_FIELD, START_INCLUSIVE + 5.0 + SEPARATOR + POSITIVE_INFINITY_END).add(COUNT_FIELD, 50.0).add(PROBABILITY_FIELD, 0.5).getRecord();
Assert.assertEquals(records.get(0), expectedA);
Assert.assertEquals(records.get(1), expectedB);
Assert.assertEquals(records.get(2), expectedC);
Assert.assertEquals(distribution.getRecords(), result.getRecords());
Assert.assertEquals(distribution.getMetadata().asMap(), result.getMeta().asMap());
}
use of com.yahoo.bullet.aggregations.sketches.QuantileSketch.PROBABILITY_FIELD in project bullet-core by yahoo.
the class DistributionTest method testNegativeSize.
@Test
public void testNegativeSize() {
// MAX_POINTS is configured to -1 and we will use the min BulletConfig.DEFAULT_DISTRIBUTION_AGGREGATION_MAX_POINTS
// and aggregation size, which is 1
Distribution distribution = makeDistribution(makeConfiguration(-1, 128), makeAttributes(Distribution.Type.PMF, 10L), "field", 1, ALL_METADATA);
IntStream.range(0, 100).mapToDouble(i -> i).mapToObj(d -> RecordBox.get().add("field", d).getRecord()).forEach(distribution::consume);
Clip result = distribution.getResult();
Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
Assert.assertEquals(metadata.size(), 7);
Assert.assertFalse((Boolean) metadata.get("isEst"));
List<BulletRecord> records = result.getRecords();
Assert.assertEquals(records.size(), 2);
BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 0.0 + END_EXCLUSIVE).add(COUNT_FIELD, 0.0).add(PROBABILITY_FIELD, 0.0).getRecord();
BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, START_INCLUSIVE + 0.0 + SEPARATOR + POSITIVE_INFINITY_END).add(COUNT_FIELD, 100.0).add(PROBABILITY_FIELD, 1.0).getRecord();
Assert.assertEquals(records.get(0), expectedA);
Assert.assertEquals(records.get(1), expectedB);
Assert.assertEquals(distribution.getRecords(), result.getRecords());
Assert.assertEquals(distribution.getMetadata().asMap(), result.getMeta().asMap());
}
Aggregations