Examples with GroupOperation com.yahoo.bullet.querying.aggregations.grouping.GroupOperation used on opensource projects

Example 1 with GroupOperation

use of com.yahoo.bullet.querying.aggregations.grouping.GroupOperation in project bullet-core by yahoo.

the class TupleSketchingStrategyTest method testGroupByOperations.

@Test
public void testGroupByOperations() {
    List<String> fields = asList("fieldA", "fieldB");
    TupleSketchingStrategy groupBy = makeGroupBy(fields, 3, new GroupOperation(COUNT, null, "count"), new GroupOperation(SUM, "price", "priceSum"));
    BulletRecord recordA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("price", 3).getRecord();
    BulletRecord recordB = RecordBox.get().addNull("fieldA").add("fieldB", "bar").add("price", 1).getRecord();
    IntStream.range(0, 10).forEach(i -> groupBy.consume(recordA));
    IntStream.range(0, 9).forEach(i -> groupBy.consume(recordB));
    IntStream.range(0, 20).forEach(i -> groupBy.consume(recordA));
    groupBy.consume(recordB);
    Clip aggregate = groupBy.getResult();
    Assert.assertNotNull(aggregate);
    Map<String, Object> meta = aggregate.getMeta().asMap();
    Assert.assertEquals(meta.size(), 1);
    List<BulletRecord> records = aggregate.getRecords();
    Assert.assertEquals(records.size(), 2);
    // count = 10 + 20, price = 10*3 + 20*3
    BulletRecord expectedA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("count", 30L).add("priceSum", 90.0).getRecord();
    // count = 9 + 1, price = 9*1 + 1*1
    BulletRecord expectedB = RecordBox.get().add("fieldA", "null").add("fieldB", "bar").add("count", 10L).add("priceSum", 10.0).getRecord();
    assertContains(records, expectedA);
    assertContains(records, expectedB);
    Assert.assertEquals(groupBy.getRecords(), aggregate.getRecords());
    Assert.assertEquals(groupBy.getMetadata().asMap(), aggregate.getMeta().asMap());
}

Example 2 with GroupOperation

use of com.yahoo.bullet.querying.aggregations.grouping.GroupOperation in project bullet-core by yahoo.

the class TupleSketchingStrategyTest method testCombining.

@Test
public void testCombining() {
    List<String> fields = asList("fieldA", "fieldB");
    TupleSketchingStrategy groupBy = makeGroupBy(fields, 5, new GroupOperation(COUNT, null, "count"), new GroupOperation(SUM, "price", "priceSum"));
    BulletRecord recordA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("price", 3).getRecord();
    BulletRecord recordB = RecordBox.get().add("fieldA", "null").add("fieldB", "bar").add("price", 1).getRecord();
    IntStream.range(0, 10).mapToObj(i -> recordA).forEach(groupBy::consume);
    IntStream.range(0, 9).mapToObj(i -> recordB).forEach(groupBy::consume);
    IntStream.range(0, 20).mapToObj(i -> recordA).forEach(groupBy::consume);
    groupBy.consume(recordB);
    byte[] firstSerialized = groupBy.getData();
    // Remake it
    groupBy = makeGroupBy(fields, 5, new GroupOperation(COUNT, null, "count"), new GroupOperation(SUM, "price", "priceSum"));
    BulletRecord recordC = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("price", 3).getRecord();
    BulletRecord recordD = RecordBox.get().addNull("fieldA").addNull("fieldB").add("price", 10).getRecord();
    IntStream.range(0, 30).mapToObj(i -> recordC).forEach(groupBy::consume);
    IntStream.range(0, 10).mapToObj(i -> recordD).forEach(groupBy::consume);
    byte[] secondSerialized = groupBy.getData();
    groupBy = makeGroupBy(fields, 5, new GroupOperation(COUNT, null, "count"), new GroupOperation(SUM, "price", "priceSum"));
    groupBy.combine(firstSerialized);
    groupBy.combine(secondSerialized);
    Clip aggregate = groupBy.getResult();
    Assert.assertNotNull(aggregate);
    List<BulletRecord> records = aggregate.getRecords();
    Assert.assertEquals(records.size(), 3);
    // count = 10 + 20 + 30, price = 10*3 + 20*3 + 30*3
    BulletRecord expectedA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("count", 60L).add("priceSum", 180.0).getRecord();
    // count = 9 + 1, price = 9*1 + 1*1
    BulletRecord expectedB = RecordBox.get().add("fieldA", "null").add("fieldB", "bar").add("count", 10L).add("priceSum", 10.0).getRecord();
    // count = 10, price = 10*10
    BulletRecord expectedC = RecordBox.get().add("fieldA", "null").add("fieldB", "null").add("count", 10L).add("priceSum", 100.0).getRecord();
    assertContains(records, expectedA);
    assertContains(records, expectedB);
    assertContains(records, expectedC);
    Assert.assertEquals(groupBy.getRecords(), aggregate.getRecords());
    Assert.assertEquals(groupBy.getMetadata().asMap(), aggregate.getMeta().asMap());
}

Example 3 with GroupOperation

use of com.yahoo.bullet.querying.aggregations.grouping.GroupOperation in project bullet-core by yahoo.

the class QuerierTest method makeCountQueryWithAllWindow.

private static RunningQuery makeCountQueryWithAllWindow(BulletConfig config, int emitInterval) {
    GroupAll groupAll = new GroupAll(singleton(new GroupOperation(GroupOperation.GroupOperationType.COUNT, null, "COUNT")));
    Window window = WindowUtils.makeWindow(Window.Unit.TIME, emitInterval, Window.Unit.ALL, null);
    Query query = new Query(new Projection(), null, groupAll, null, window, null);
    query.configure(config);
    RunningQuery runningQuery = spy(makeRunningQuery("", query));
    Mockito.doReturn(false).when(runningQuery).isTimedOut();
    return runningQuery;
}

Example 4 with GroupOperation

use of com.yahoo.bullet.querying.aggregations.grouping.GroupOperation in project bullet-core by yahoo.

the class GroupAllStrategyTest method testNullRecordCount.

@Test
public void testNullRecordCount() {
    GroupAllStrategy strategy = makeGroupAll(new GroupOperation(GroupOperation.GroupOperationType.COUNT, null, "count"));
    strategy.consume(RecordBox.get().add("foo", "bar").getRecord());
    Assert.assertNotNull(strategy.getData());
    List<BulletRecord> aggregate = strategy.getResult().getRecords();
    // We do not expect to send in null records so the count is incremented.
    Assert.assertEquals(aggregate.size(), 1);
    BulletRecord actual = aggregate.get(0);
    BulletRecord expected = RecordBox.get().add("count", 1L).getRecord();
    Assert.assertEquals(actual, expected);
    Assert.assertEquals(strategy.getRecords(), aggregate);
    Assert.assertEquals(strategy.getMetadata().asMap(), strategy.getResult().getMeta().asMap());
}

Example 5 with GroupOperation

use of com.yahoo.bullet.querying.aggregations.grouping.GroupOperation in project bullet-core by yahoo.

the class GroupAllStrategyTest method testCountingMoreThanMaximum.

@Test
public void testCountingMoreThanMaximum() {
    GroupAllStrategy strategy = makeGroupAll(new GroupOperation(GroupOperation.GroupOperationType.COUNT, null, "count"));
    BulletRecord someRecord = RecordBox.get().add("foo", 1L).getRecord();
    IntStream.range(0, 2 * BulletConfig.DEFAULT_AGGREGATION_MAX_SIZE).forEach(i -> strategy.consume(someRecord));
    Assert.assertNotNull(strategy.getData());
    List<BulletRecord> aggregate = strategy.getResult().getRecords();
    Assert.assertEquals(aggregate.size(), 1);
    BulletRecord actual = aggregate.get(0);
    BulletRecord expected = RecordBox.get().add("count", 2L * BulletConfig.DEFAULT_AGGREGATION_MAX_SIZE).getRecord();
    Assert.assertEquals(actual, expected);
    Assert.assertEquals(strategy.getRecords(), aggregate);
    Assert.assertEquals(strategy.getMetadata().asMap(), strategy.getResult().getMeta().asMap());
}