Examples with Clip com.yahoo.bullet.result.Clip used on opensource projects

Example 36 with Clip

use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.

the class FrequentItemsSketchTest method testApproximateCounting.

@Test
public void testApproximateCounting() {
    FrequentItemsSketch sketch = new FrequentItemsSketch(ErrorType.NO_FALSE_POSITIVES, 32, 40);
    IntStream.range(0, 40).forEach(i -> sketch.update(String.valueOf(i)));
    IntStream.of(10, 20, 25, 30).forEach(i -> IntStream.range(0, i).forEach(j -> sketch.update(String.valueOf(i))));
    Clip result = sketch.getResult("meta", ALL_METADATA);
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 5);
    Assert.assertTrue((Boolean) metadata.get("isEst"));
    Long error = (Long) metadata.get("error");
    List<BulletRecord> records = result.getRecords();
    // We should have our four frequent items
    Assert.assertTrue(records.size() >= 4);
    for (BulletRecord actual : records) {
        Assert.assertEquals(actual.fieldCount(), 2);
        Integer item = Integer.valueOf(actual.get(FrequentItemsSketch.ITEM_FIELD).toString());
        Long count = (Long) actual.get(FrequentItemsSketch.COUNT_FIELD);
        if (item == 10 || item == 20 || item == 25 || item == 30) {
            Assert.assertTrue(count >= item + 1);
            Assert.assertTrue(count <= item + 1 + error);
        } else {
            Assert.assertTrue(count >= 1);
            Assert.assertTrue(count <= 1 + error);
        }
    }
    Assert.assertEquals(sketch.getRecords(), result.getRecords());
    Assert.assertEquals(sketch.getMetadata("meta", ALL_METADATA).asMap(), result.getMeta().asMap());
}

Example 37 with Clip

use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.

the class QuantileSketchTest method testUnioning.

@Test
public void testUnioning() {
    QuantileSketch sketch = new QuantileSketch(64, Distribution.Type.CDF, makePoints(0.0, 9.0, 1.0));
    IntStream.range(0, 30).forEach(i -> sketch.update(i % 10));
    QuantileSketch anotherSketch = new QuantileSketch(64, Distribution.Type.CDF, makePoints(0.0, 9.0, 1.0));
    IntStream.range(0, 30).forEach(i -> sketch.update(i % 3));
    QuantileSketch mergedSketch = new QuantileSketch(64, Distribution.Type.CDF, makePoints(0.0, 9.0, 1.0));
    mergedSketch.union(sketch.serialize());
    mergedSketch.union(anotherSketch.serialize());
    Clip result = mergedSketch.getResult(null, null);
    List<BulletRecord> records = result.getRecords();
    for (BulletRecord record : records) {
        String range = (String) record.get(RANGE_FIELD);
        double count = (Double) record.get(COUNT_FIELD);
        double probablity = (Double) record.get(PROBABILITY_FIELD);
        String rangeEnd = getEnd(range);
        if (rangeEnd.equals(POSITIVE_INFINITY)) {
            Assert.assertEquals(count, 60.0);
            Assert.assertEquals(probablity, 1.0);
        } else {
            double end = Double.valueOf(rangeEnd);
            if (end <= 3.0) {
                Assert.assertEquals(count, end * 13.0);
                Assert.assertEquals(probablity, end * 13.0 / 60);
            } else {
                Assert.assertEquals(count, 39.0 + (end - 3) * 3.0);
                Assert.assertEquals(probablity, (39.0 + (end - 3) * 3.0) / 60);
            }
        }
    }
}

Example 38 with Clip

use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.

the class QuantileSketchTest method testExactPMFWithNumberOfPoints.

@Test
public void testExactPMFWithNumberOfPoints() {
    QuantileSketch sketch = new QuantileSketch(64, 2, Distribution.Type.PMF, 10);
    // Insert 0, 1, 2 ... 9 three times
    IntStream.range(0, 30).forEach(i -> sketch.update(i % 10));
    // Insert 0, 1, 2 ten times
    IntStream.range(0, 30).forEach(i -> sketch.update(i % 3));
    Clip result = sketch.getResult("meta", ALL_METADATA);
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 7);
    Assert.assertFalse((Boolean) metadata.get("isEst"));
    Assert.assertEquals((String) metadata.get("family"), Family.QUANTILES.getFamilyName());
    // Size should be at least 10 bytes since we inserted 60 items: 10 uniques
    Assert.assertTrue((Integer) metadata.get("size") >= 10);
    Assert.assertEquals(metadata.get("nre"), DoublesSketch.getNormalizedRankError(64));
    // 60 items
    Assert.assertEquals(metadata.get("n"), 60L);
    Assert.assertEquals(metadata.get("min"), 0.0);
    Assert.assertEquals(metadata.get("max"), 9.0);
    List<BulletRecord> records = result.getRecords();
    for (BulletRecord record : records) {
        String range = (String) record.get(RANGE_FIELD);
        double count = (Double) record.get(COUNT_FIELD);
        double probablity = (Double) record.get(PROBABILITY_FIELD);
        String rangeStart = getStart(range);
        String rangeEnd = getEnd(range);
        if (rangeStart.equals(NEGATIVE_INFINITY)) {
            Assert.assertEquals(count, 0.0);
            Assert.assertEquals(probablity, 0.0);
        } else if (rangeEnd.equals(POSITIVE_INFINITY)) {
            Assert.assertEquals(count, 3.0);
            Assert.assertEquals(probablity, 1.0 / 20);
        } else {
            double start = Double.valueOf(rangeStart);
            if (start <= 2.0) {
                // 0.0 - 1.0, 1.0 - 2.0, 2.0 - 3.0 have 3 from the first, 10 from the second
                Assert.assertEquals(count, 13.0);
                Assert.assertEquals(probablity, 13.0 / 60);
            } else {
                // The rest are all 3
                Assert.assertEquals(count, 3.0);
                Assert.assertEquals(probablity, 1.0 / 20);
            }
        }
    }
    Assert.assertEquals(sketch.getRecords(), records);
    Assert.assertEquals(sketch.getMetadata("meta", ALL_METADATA).asMap(), result.getMeta().asMap());
}

Example 39 with Clip

use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.

the class QuantileSketchTest method testExactCDFWithNumberOfPoints.

@Test
public void testExactCDFWithNumberOfPoints() {
    QuantileSketch sketch = new QuantileSketch(64, 2, Distribution.Type.CDF, 10);
    IntStream.range(0, 30).forEach(i -> sketch.update(i % 10));
    IntStream.range(0, 30).forEach(i -> sketch.update(i % 3));
    Clip result = sketch.getResult(null, null);
    List<BulletRecord> records = result.getRecords();
    for (BulletRecord record : records) {
        String range = (String) record.get(RANGE_FIELD);
        double count = (Double) record.get(COUNT_FIELD);
        double probablity = (Double) record.get(PROBABILITY_FIELD);
        String rangeEnd = getEnd(range);
        if (rangeEnd.equals(POSITIVE_INFINITY)) {
            Assert.assertEquals(count, 60.0);
            Assert.assertEquals(probablity, 1.0);
        } else {
            double end = Double.valueOf(rangeEnd);
            if (end <= 3.0) {
                // -inf - 0.0, -inf - 1.0, -inf - 2.0, -inf - 3.0 are 13 * 0, 13 * 1, ...
                Assert.assertEquals(count, end * 13.0);
                Assert.assertEquals(probablity, end * 13.0 / 60);
            } else {
                // -inf - 4.0, -inf - 5.0 ... -inf - 9.0 are 13*3 + (1*3),  13*3 + (2*3) ...
                Assert.assertEquals(count, 39.0 + (end - 3) * 3.0);
                Assert.assertEquals(probablity, (39.0 + (end - 3) * 3.0) / 60);
            }
        }
    }
}

Example 40 with Clip

use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.

the class QuantileSketchTest method testFetchingDataWithoutResettingAndInsertingMoreData.

@Test
public void testFetchingDataWithoutResettingAndInsertingMoreData() {
    QuantileSketch sketch = new QuantileSketch(64, Distribution.Type.CDF, makePoints(0.0, 9.0, 1.0));
    IntStream.range(0, 30).forEach(i -> sketch.update(i % 10));
    QuantileSketch anotherSketch = new QuantileSketch(64, Distribution.Type.CDF, makePoints(0.0, 9.0, 1.0));
    IntStream.range(0, 30).forEach(i -> sketch.update(i % 3));
    sketch.union(anotherSketch.serialize());
    Clip result = sketch.getResult(null, null);
    List<BulletRecord> records = result.getRecords();
    for (BulletRecord record : records) {
        String range = (String) record.get(RANGE_FIELD);
        double count = (Double) record.get(COUNT_FIELD);
        String rangeEnd = getEnd(range);
        if (rangeEnd.equals(POSITIVE_INFINITY)) {
            Assert.assertEquals(count, 60.0);
        } else {
            double end = Double.valueOf(rangeEnd);
            if (end <= 3.0) {
                Assert.assertEquals(count, end * 13.0);
            } else {
                Assert.assertEquals(count, 39.0 + (end - 3) * 3.0);
            }
        }
    }
    sketch.update(1.0);
    result = sketch.getResult(null, null);
    records = result.getRecords();
    for (BulletRecord record : records) {
        String range = (String) record.get(RANGE_FIELD);
        double count = (Double) record.get(COUNT_FIELD);
        String rangeEnd = getEnd(range);
        if (rangeEnd.equals(POSITIVE_INFINITY)) {
            Assert.assertEquals(count, 61.0);
        } else {
            double end = Double.valueOf(rangeEnd);
            if (end <= 1.0) {
                Assert.assertEquals(count, end * 13.0);
            } else if (end <= 3.0) {
                Assert.assertEquals(count, (end * 13.0) + 1);
            } else {
                Assert.assertEquals(count, 40.0 + (end - 3) * 3.0);
            }
        }
    }
}