Examples with Clip com.yahoo.bullet.result.Clip used on opensource projects

Example 46 with Clip

use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.

the class GroupByTest method testDistincts.

@Test
public void testDistincts() {
    List<String> fields = asList("fieldA", "fieldB", "fieldC");
    GroupBy groupBy = makeDistinct(fields, 3);
    BulletRecord recordA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("fieldC", "baz").getRecord();
    IntStream.range(0, 9).forEach(i -> groupBy.consume(recordA));
    BulletRecord recordB = RecordBox.get().add("fieldA", "1").add("fieldB", "2").getRecord();
    IntStream.range(0, 9).forEach(i -> groupBy.consume(recordB));
    groupBy.consume(RecordBox.get().getRecord());
    Clip aggregate = groupBy.getResult();
    Assert.assertNotNull(aggregate);
    Map<String, Object> meta = aggregate.getMeta().asMap();
    Assert.assertEquals(meta.size(), 1);
    List<BulletRecord> records = aggregate.getRecords();
    Assert.assertEquals(records.size(), 3);
    BulletRecord expectedA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("fieldC", "baz").getRecord();
    BulletRecord expectedB = RecordBox.get().add("fieldA", "1").add("fieldB", "2").add("fieldC", "null").getRecord();
    BulletRecord expectedC = RecordBox.get().add("fieldA", "null").add("fieldB", "null").add("fieldC", "null").getRecord();
    // We have each distinct record exactly once
    assertContains(records, expectedA);
    assertContains(records, expectedB);
    assertContains(records, expectedC);
    Assert.assertEquals(groupBy.getRecords(), aggregate.getRecords());
    Assert.assertEquals(groupBy.getMetadata().asMap(), aggregate.getMeta().asMap());
}

Example 47 with Clip

use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.

the class TopKTest method testApproximateTopK.

@Test
public void testApproximateTopK() {
    TopK topK = makeTopK(asList("A", "B"), 64, 4);
    IntStream.range(0, 128).mapToObj(i -> RecordBox.get().add("A", i).add("B", 128 + i).getRecord()).forEach(topK::consume);
    IntStream.range(0, 60).mapToObj(i -> RecordBox.get().add("A", i % 3).getRecord()).forEach(topK::consume);
    topK.consume(RecordBox.get().add("A", 0).getRecord());
    topK.consume(RecordBox.get().add("A", 0).getRecord());
    topK.consume(RecordBox.get().add("A", 1).getRecord());
    Clip result = topK.getResult();
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 5);
    Assert.assertTrue((Boolean) metadata.get("isEst"));
    List<BulletRecord> records = result.getRecords();
    long error = (Long) metadata.get("error");
    Assert.assertEquals(records.size(), 4);
    for (int i = 0; i < 3; ++i) {
        BulletRecord actual = records.get(i);
        int fieldA = Integer.valueOf(actual.get("A").toString());
        long count = (Long) actual.get(TopK.DEFAULT_NEW_NAME);
        if (fieldA == 0) {
            Assert.assertTrue(count >= 23L);
            Assert.assertTrue(count <= 23L + error);
        } else if (fieldA == 1) {
            Assert.assertTrue(count >= 22L);
            Assert.assertTrue(count <= 22L + error);
        } else if (fieldA == 2) {
            Assert.assertTrue(count >= 21L);
            Assert.assertTrue(count <= 21L + error);
        } else {
            Assert.fail("This case should not exist.");
        }
    }
    // The last one is one of the other records
    BulletRecord actual = records.get(3);
    long count = (Long) actual.get(TopK.DEFAULT_NEW_NAME);
    Assert.assertTrue(count >= 1L);
    Assert.assertTrue(count <= 1L + error);
    Assert.assertEquals(topK.getRecords(), records);
    Assert.assertEquals(topK.getMetadata().asMap(), result.getMeta().asMap());
}

Example 48 with Clip

use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.

the class TopKTest method testNullAttributes.

@Test
public void testNullAttributes() {
    TopK topK = makeTopK(makeConfiguration(ErrorType.NO_FALSE_NEGATIVES, 32), null, singletonMap("A", "foo"), 16, null);
    IntStream.range(0, 16).mapToObj(i -> RecordBox.get().add("A", i).getRecord()).forEach(topK::consume);
    Clip result = topK.getResult();
    Assert.assertNull(result.getMeta().asMap().get("meta"));
    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 16);
    for (BulletRecord actual : records) {
        Assert.assertEquals(actual.fieldCount(), 2);
        int fieldA = Integer.valueOf(actual.get("foo").toString());
        Assert.assertTrue(fieldA < 16);
        Assert.assertEquals(actual.get(TopK.DEFAULT_NEW_NAME), 1L);
    }
    Assert.assertEquals(topK.getRecords(), records);
    Assert.assertEquals(topK.getMetadata().asMap(), result.getMeta().asMap());
}

Example 49 with Clip

use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.

the class TopKTest method testRenaming.

@Test
public void testRenaming() {
    HashMap<String, String> fields = new HashMap<>();
    fields.put("A", "foo");
    fields.put("fieldB", "");
    TopK topK = makeTopK(makeConfiguration(ErrorType.NO_FALSE_NEGATIVES, -1), null, fields, 16, null);
    IntStream.range(0, 16).mapToObj(i -> RecordBox.get().add("A", i).add("fieldB", 32 - i).getRecord()).forEach(topK::consume);
    Clip result = topK.getResult();
    Assert.assertNull(result.getMeta().asMap().get("meta"));
    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 16);
    for (BulletRecord actual : records) {
        Assert.assertEquals(actual.fieldCount(), 3);
        int fieldA = Integer.valueOf(actual.get("foo").toString());
        int fieldB = Integer.valueOf(actual.get("fieldB").toString());
        Assert.assertTrue(fieldA < 16);
        Assert.assertTrue(fieldB > 16);
        Assert.assertEquals(actual.get(TopK.DEFAULT_NEW_NAME), 1L);
    }
    Assert.assertEquals(topK.getRecords(), records);
    Assert.assertEquals(topK.getMetadata().asMap(), result.getMeta().asMap());
}

Example 50 with Clip

use of com.yahoo.bullet.result.Clip in project bullet-core by yahoo.

the class TopKTest method testCombining.

@Test
public void testCombining() {
    TopK topK = makeTopK(asList("A", "B"), 64, 4);
    IntStream.range(0, 60).mapToObj(i -> RecordBox.get().add("A", i % 3).getRecord()).forEach(topK::consume);
    TopK another = makeTopK(asList("A", "B"), 64, 4);
    another.consume(RecordBox.get().add("A", 0).getRecord());
    another.consume(RecordBox.get().add("A", 0).getRecord());
    another.consume(RecordBox.get().add("A", 1).getRecord());
    TopK union = makeTopK(asList("A", "B"), 64, 4);
    union.combine(topK.getData());
    union.combine(another.getData());
    Clip result = union.getResult();
    Map<String, Object> metadata = (Map<String, Object>) result.getMeta().asMap().get("meta");
    Assert.assertEquals(metadata.size(), 5);
    Assert.assertFalse((Boolean) metadata.get("isEst"));
    List<BulletRecord> records = result.getRecords();
    Assert.assertEquals(records.size(), 3);
    BulletRecord expectedA = RecordBox.get().add("A", "0").add("B", "null").add(TopK.DEFAULT_NEW_NAME, 22L).getRecord();
    BulletRecord expectedB = RecordBox.get().add("A", "1").add("B", "null").add(TopK.DEFAULT_NEW_NAME, 21L).getRecord();
    BulletRecord expectedC = RecordBox.get().add("A", "2").add("B", "null").add(TopK.DEFAULT_NEW_NAME, 20L).getRecord();
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);
    Assert.assertEquals(records.get(2), expectedC);
    Assert.assertEquals(union.getRecords(), records);
    Assert.assertEquals(union.getMetadata().asMap(), result.getMeta().asMap());
}