Examples with GroupOperation com.yahoo.bullet.aggregations.grouping.GroupOperation used on opensource projects

Example 6 with GroupOperation

use of com.yahoo.bullet.aggregations.grouping.GroupOperation in project bullet-core by yahoo.

the class GroupAllTest method testResetting.

@Test
public void testResetting() {
    List<GroupOperation> operations = asList(new GroupOperation(GroupOperation.GroupOperationType.COUNT, null, "myCount"), new GroupOperation(GroupOperation.GroupOperationType.MIN, "minField", "myMin"), new GroupOperation(GroupOperation.GroupOperationType.AVG, "groupField", "groupAvg"), new GroupOperation(GroupOperation.GroupOperationType.MIN, "groupField", "groupMin"), new GroupOperation(GroupOperation.GroupOperationType.SUM, "groupField", "groupSum"));
    GroupAll groupAll = makeGroupAll(operations);
    groupAll.consume(RecordBox.get().add("minField", -8.8).add("groupField", 3.14).getRecord());
    groupAll.consume(RecordBox.get().add("minField", 0.0).addNull("groupField").getRecord());
    groupAll.consume(RecordBox.get().add("minField", 51.44).add("groupField", -4.88).getRecord());
    groupAll.consume(RecordBox.get().add("minField", -8.8).add("groupField", 12345.67).getRecord());
    groupAll.consume(RecordBox.get().addNull("minField").add("groupField", 2.718).getRecord());
    groupAll.consume(RecordBox.get().add("minField", -51.0).addNull("groupField").getRecord());
    groupAll.consume(RecordBox.get().add("minField", 0).add("groupField", 1).getRecord());
    groupAll.consume(RecordBox.get().add("minField", 44.8).add("groupField", -51.44).getRecord());
    Assert.assertNotNull(groupAll.getData());
    List<BulletRecord> aggregate = groupAll.getResult().getRecords();
    Assert.assertEquals(aggregate.size(), 1);
    BulletRecord actual = aggregate.get(0);
    BulletRecord expected = RecordBox.get().add("myCount", 8L).add("myMin", -51.0).add("groupAvg", 2049.368).add("groupMin", -51.44).add("groupSum", 12296.208).getRecord();
    Assert.assertTrue(actual.equals(expected));
    Assert.assertEquals(groupAll.getRecords(), aggregate);
    Assert.assertEquals(groupAll.getMetadata().asMap(), groupAll.getResult().getMeta().asMap());
    groupAll.reset();
    groupAll.consume(RecordBox.get().add("minField", -8.8).add("groupField", 3.14).getRecord());
    groupAll.consume(RecordBox.get().add("minField", 0.0).addNull("groupField").getRecord());
    Assert.assertNotNull(groupAll.getData());
    aggregate = groupAll.getResult().getRecords();
    Assert.assertEquals(aggregate.size(), 1);
    actual = aggregate.get(0);
    expected = RecordBox.get().add("myCount", 2L).add("myMin", -8.8).add("groupAvg", 3.14).add("groupMin", 3.14).add("groupSum", 3.14).getRecord();
    Assert.assertTrue(actual.equals(expected));
    Assert.assertEquals(groupAll.getRecords(), aggregate);
    Assert.assertEquals(groupAll.getMetadata().asMap(), groupAll.getResult().getMeta().asMap());
}

Example 7 with GroupOperation

use of com.yahoo.bullet.aggregations.grouping.GroupOperation in project bullet-storm by yahoo.

the class JoinBoltTest method getGroupDataWithCount.

private static byte[] getGroupDataWithCount(String countField, int count) {
    GroupData groupData = new GroupData(new HashSet<>(singletonList(new GroupOperation(COUNT, null, countField))));
    IntStream.range(0, count).forEach(i -> groupData.consume(RecordBox.get().getRecord()));
    return SerializerDeserializer.toBytes(groupData);
}

Example 8 with GroupOperation

use of com.yahoo.bullet.aggregations.grouping.GroupOperation in project bullet-storm by yahoo.

the class JoinBoltTest method testGroupBy.

@Test
public void testGroupBy() {
    final int entries = 16;
    BulletConfig bulletConfig = GroupByTest.makeConfiguration(entries);
    GroupBy groupBy = GroupByTest.makeGroupBy(bulletConfig, singletonMap("fieldA", "A"), entries, AggregationUtils.makeGroupOperation(COUNT, null, "cnt"), AggregationUtils.makeGroupOperation(SUM, "fieldB", "sumB"));
    IntStream.range(0, 256).mapToObj(i -> RecordBox.get().add("fieldA", i % 16).add("fieldB", i / 16).getRecord()).forEach(groupBy::consume);
    byte[] first = groupBy.getData();
    groupBy = GroupByTest.makeGroupBy(bulletConfig, singletonMap("fieldA", "A"), entries, AggregationUtils.makeGroupOperation(COUNT, null, "cnt"), AggregationUtils.makeGroupOperation(SUM, "fieldB", "sumB"));
    IntStream.range(256, 1024).mapToObj(i -> RecordBox.get().add("fieldA", i % 16).add("fieldB", i / 16).getRecord()).forEach(groupBy::consume);
    byte[] second = groupBy.getData();
    // Send generated data to JoinBolt
    bolt = new DonableJoinBolt(config, 2, true);
    setup(bolt);
    List<GroupOperation> operations = asList(new GroupOperation(COUNT, null, "cnt"), new GroupOperation(SUM, "fieldB", "sumB"));
    String queryString = makeGroupFilterQuery("ts", singletonList("1"), EQUALS, GROUP, entries, operations, Pair.of("fieldA", "A"));
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", queryString, EMPTY);
    bolt.execute(query);
    sendRawByteTuplesTo(bolt, "42", asList(first, second));
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    bolt.execute(tick);
    for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
        bolt.execute(tick);
        Assert.assertEquals(collector.getEmittedCount(), 0);
    }
    bolt.execute(tick);
    Assert.assertEquals(collector.getEmittedCount(), 2);
    String response = (String) collector.getMthElementFromNthTupleEmittedTo(TopologyConstants.RESULT_STREAM, 1, 1).get();
    JsonParser parser = new JsonParser();
    JsonObject actual = parser.parse(response).getAsJsonObject();
    JsonArray actualRecords = actual.get(Clip.RECORDS_KEY).getAsJsonArray();
    Assert.assertEquals(actualRecords.size(), 16);
}