use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.
the class FrequentItemsSketchTest method testExactCounting.
@Test
public void testExactCounting() {
FrequentItemsSketch sketch = new FrequentItemsSketch(ErrorType.NO_FALSE_NEGATIVES, 32, 15);
IntStream.range(0, 10).forEach(i -> IntStream.range(0, 10).forEach(j -> sketch.update(String.valueOf(i))));
sketch.update("foo");
IntStream.range(10, 100).forEach(i -> sketch.update("bar"));
sketch.update("baz");
Clip result = sketch.getResult("meta", ALL_METADATA);
Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
Assert.assertEquals(metadata.size(), 5);
Assert.assertFalse((Boolean) metadata.get("isEst"));
Assert.assertEquals((String) metadata.get("family"), Family.FREQUENCY.getFamilyName());
Assert.assertNull(metadata.get("size"));
Assert.assertEquals(metadata.get("error"), 0L);
Assert.assertEquals(metadata.get("n"), 192L);
Assert.assertEquals(metadata.get("actives"), 13);
List<BulletRecord> records = result.getRecords();
Assert.assertEquals(records.size(), 13);
for (BulletRecord actual : records) {
String item = actual.get(FrequentItemsSketch.ITEM_FIELD).toString();
Assert.assertEquals(actual.fieldCount(), 2);
if ("bar".equals(item)) {
Assert.assertEquals(actual.get(FrequentItemsSketch.COUNT_FIELD), 90L);
} else if ("foo".equals(item) || "baz".equals(item)) {
Assert.assertEquals(actual.get(FrequentItemsSketch.COUNT_FIELD), 1L);
} else if (Integer.valueOf(item) < 10) {
Assert.assertEquals(actual.get(FrequentItemsSketch.COUNT_FIELD), 10L);
} else {
Assert.fail("This should not be a case");
}
}
Assert.assertEquals(sketch.getRecords(), result.getRecords());
Assert.assertEquals(sketch.getMetadata("meta", ALL_METADATA).asMap(), result.getMeta().asMap());
}
use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.
the class FrequentItemsSketchTest method testResetting.
@Test
public void testResetting() {
FrequentItemsSketch sketch = new FrequentItemsSketch(ErrorType.NO_FALSE_NEGATIVES, 32, 32);
IntStream.range(0, 10).forEach(i -> IntStream.range(0, 10).forEach(j -> sketch.update(String.valueOf(i))));
IntStream.range(10, 100).forEach(i -> sketch.update("bar"));
FrequentItemsSketch another = new FrequentItemsSketch(ErrorType.NO_FALSE_NEGATIVES, 32, 32);
another.update("foo");
another.update("bar");
another.update("baz");
sketch.union(another.serialize());
Clip result = sketch.getResult("meta", ALL_METADATA);
Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
Assert.assertFalse((Boolean) metadata.get("isEst"));
Assert.assertEquals(result.getRecords().size(), 13);
Assert.assertEquals(sketch.getRecords(), result.getRecords());
Assert.assertEquals(sketch.getMetadata("meta", ALL_METADATA).asMap(), result.getMeta().asMap());
sketch.reset();
result = sketch.getResult("meta", ALL_METADATA);
metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
Assert.assertEquals((String) metadata.get("family"), Family.FREQUENCY.getFamilyName());
Assert.assertNull(metadata.get("size"));
Assert.assertEquals(metadata.get("error"), 0L);
Assert.assertEquals(metadata.get("n"), 0L);
Assert.assertEquals(metadata.get("actives"), 0);
Assert.assertFalse((Boolean) metadata.get("isEst"));
List<BulletRecord> records = result.getRecords();
Assert.assertEquals(records.size(), 0);
Assert.assertEquals(sketch.getRecords(), result.getRecords());
Assert.assertEquals(sketch.getMetadata("meta", ALL_METADATA).asMap(), result.getMeta().asMap());
}
use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.
the class FrequentItemsSketchTest method testSizeLimiting.
@Test
public void testSizeLimiting() {
FrequentItemsSketch sketch = new FrequentItemsSketch(ErrorType.NO_FALSE_NEGATIVES, 32, 10);
// For i from 1 to 13, update the sketch i times
IntStream.range(1, 13).forEach(i -> IntStream.range(0, i).forEach(j -> sketch.update(String.valueOf(i))));
Clip result = sketch.getResult("meta", ALL_METADATA);
Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
Assert.assertEquals(metadata.size(), 5);
Assert.assertFalse((Boolean) metadata.get("isEst"));
List<BulletRecord> records = result.getRecords();
Assert.assertEquals(records.size(), 10);
for (BulletRecord actual : records) {
Assert.assertEquals(actual.fieldCount(), 2);
Integer item = Integer.valueOf(actual.get(FrequentItemsSketch.ITEM_FIELD).toString());
// 1, 2 had the lowest and since our size is 10, we should have not seen them
Assert.assertTrue(item > 2 && item < 13);
Assert.assertEquals(actual.get(FrequentItemsSketch.COUNT_FIELD), Long.valueOf(item));
}
Assert.assertEquals(sketch.getRecords(), result.getRecords());
Assert.assertEquals(sketch.getMetadata("meta", ALL_METADATA).asMap(), result.getMeta().asMap());
}
use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.
the class QuantileSketchTest method testNoDataPMFDistribution.
@Test
public void testNoDataPMFDistribution() {
QuantileSketch sketch = new QuantileSketch(64, 2, Distribution.Type.PMF, 10);
Clip result = sketch.getResult("meta", ALL_METADATA);
Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
Assert.assertEquals(metadata.size(), 7);
Assert.assertFalse((Boolean) metadata.get("isEst"));
Assert.assertEquals(metadata.get("n"), 0L);
Assert.assertEquals(metadata.get("min"), Double.POSITIVE_INFINITY);
Assert.assertEquals(metadata.get("max"), Double.NEGATIVE_INFINITY);
List<BulletRecord> records = result.getRecords();
Assert.assertEquals(records.size(), 2);
BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + Double.POSITIVE_INFINITY + END_EXCLUSIVE).add(PROBABILITY_FIELD, Double.NaN).add(COUNT_FIELD, Double.NaN).getRecord();
BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, START_INCLUSIVE + Double.POSITIVE_INFINITY + SEPARATOR + POSITIVE_INFINITY_END).add(PROBABILITY_FIELD, Double.NaN).add(COUNT_FIELD, Double.NaN).getRecord();
Assert.assertEquals(records.get(0), expectedA);
Assert.assertEquals(records.get(1), expectedB);
Assert.assertEquals(sketch.getRecords(), records);
Assert.assertEquals(sketch.getMetadata("meta", ALL_METADATA).asMap(), result.getMeta().asMap());
}
use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.
the class QuantileSketchTest method testNoDataQuantileDistribution.
@Test
public void testNoDataQuantileDistribution() {
QuantileSketch sketch = new QuantileSketch(64, Distribution.Type.QUANTILE, new double[] { 0, 0.3, 1 });
Clip result = sketch.getResult("meta", ALL_METADATA);
Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
Assert.assertEquals(metadata.size(), 7);
Assert.assertFalse((Boolean) metadata.get("isEst"));
Assert.assertEquals(metadata.get("n"), 0L);
Assert.assertEquals(metadata.get("min"), Double.POSITIVE_INFINITY);
Assert.assertEquals(metadata.get("max"), Double.NEGATIVE_INFINITY);
List<BulletRecord> records = result.getRecords();
Assert.assertEquals(records.size(), 3);
BulletRecord expectedA = RecordBox.get().add(QUANTILE_FIELD, 0.0).add(VALUE_FIELD, Double.POSITIVE_INFINITY).getRecord();
BulletRecord expectedB = RecordBox.get().add(QUANTILE_FIELD, 0.3).add(VALUE_FIELD, Double.NaN).getRecord();
BulletRecord expectedC = RecordBox.get().add(QUANTILE_FIELD, 1.0).add(VALUE_FIELD, Double.NEGATIVE_INFINITY).getRecord();
Assert.assertEquals(records.get(0), expectedA);
Assert.assertEquals(records.get(1), expectedB);
Assert.assertEquals(records.get(2), expectedC);
Assert.assertEquals(sketch.getRecords(), records);
Assert.assertEquals(sketch.getMetadata("meta", ALL_METADATA).asMap(), result.getMeta().asMap());
}
Aggregations