use of com.yahoo.bullet.aggregations.grouping.GroupOperation.GroupOperationType.SUM in project bullet-core by yahoo.
the class GroupByTest method testResetting.
@Test
public void testResetting() {
List<String> fields = asList("fieldA", "fieldB");
GroupBy groupBy = makeGroupBy(fields, 5, makeGroupOperation(COUNT, null, null), makeGroupOperation(SUM, "price", "priceSum"));
BulletRecord recordA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("price", 3).getRecord();
BulletRecord recordB = RecordBox.get().add("fieldA", "null").add("fieldB", "bar").add("price", 1).getRecord();
IntStream.range(0, 30).mapToObj(i -> recordA).forEach(groupBy::consume);
IntStream.range(0, 10).mapToObj(i -> recordB).forEach(groupBy::consume);
BulletRecord expectedA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add(COUNT.getName(), 30L).add("priceSum", 90.0).getRecord();
BulletRecord expectedB = RecordBox.get().add("fieldA", "null").add("fieldB", "bar").add(COUNT.getName(), 10L).add("priceSum", 10.0).getRecord();
Clip aggregate = groupBy.getResult();
Assert.assertNotNull(aggregate);
List<BulletRecord> records = aggregate.getRecords();
Assert.assertEquals(records.size(), 2);
assertContains(records, expectedA);
assertContains(records, expectedB);
Assert.assertEquals(groupBy.getRecords(), aggregate.getRecords());
Assert.assertEquals(groupBy.getMetadata().asMap(), aggregate.getMeta().asMap());
groupBy.reset();
BulletRecord recordC = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("price", 3).getRecord();
BulletRecord recordD = RecordBox.get().addNull("fieldA").addNull("fieldB").add("price", 10).getRecord();
IntStream.range(0, 10).mapToObj(i -> recordB).forEach(groupBy::consume);
IntStream.range(0, 30).mapToObj(i -> recordC).forEach(groupBy::consume);
IntStream.range(0, 10).mapToObj(i -> recordD).forEach(groupBy::consume);
aggregate = groupBy.getResult();
Assert.assertNotNull(aggregate);
records = aggregate.getRecords();
Assert.assertEquals(records.size(), 3);
expectedA = RecordBox.get().add("fieldA", "null").add("fieldB", "bar").add(COUNT.getName(), 10L).add("priceSum", 10.0).getRecord();
expectedB = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add(COUNT.getName(), 30L).add("priceSum", 90.0).getRecord();
BulletRecord expectedC = RecordBox.get().add("fieldA", "null").add("fieldB", "null").add(COUNT.getName(), 10L).add("priceSum", 100.0).getRecord();
assertContains(records, expectedA);
assertContains(records, expectedB);
assertContains(records, expectedC);
Assert.assertEquals(groupBy.getRecords(), aggregate.getRecords());
Assert.assertEquals(groupBy.getMetadata().asMap(), aggregate.getMeta().asMap());
}
use of com.yahoo.bullet.aggregations.grouping.GroupOperation.GroupOperationType.SUM in project bullet-core by yahoo.
the class GroupByTest method testCombiningAndConsuming.
@Test
public void testCombiningAndConsuming() {
List<String> fields = asList("fieldA", "fieldB");
GroupBy groupBy = makeGroupBy(fields, 5, makeGroupOperation(COUNT, null, null), makeGroupOperation(SUM, "price", "priceSum"));
BulletRecord recordA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("price", 3).getRecord();
BulletRecord recordB = RecordBox.get().add("fieldA", "null").add("fieldB", "bar").add("price", 1).getRecord();
IntStream.range(0, 30).mapToObj(i -> recordA).forEach(groupBy::consume);
IntStream.range(0, 10).mapToObj(i -> recordB).forEach(groupBy::consume);
byte[] serialized = groupBy.getData();
// Remake it
groupBy = makeGroupBy(fields, 5, makeGroupOperation(COUNT, null, null), makeGroupOperation(SUM, "price", "priceSum"));
BulletRecord recordC = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("price", 3).getRecord();
BulletRecord recordD = RecordBox.get().addNull("fieldA").addNull("fieldB").add("price", 10).getRecord();
IntStream.range(0, 30).mapToObj(i -> recordC).forEach(groupBy::consume);
IntStream.range(0, 10).mapToObj(i -> recordD).forEach(groupBy::consume);
groupBy.combine(serialized);
Clip aggregate = groupBy.getResult();
Assert.assertNotNull(aggregate);
List<BulletRecord> records = aggregate.getRecords();
Assert.assertEquals(records.size(), 3);
BulletRecord expectedA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add(COUNT.getName(), 60L).add("priceSum", 180.0).getRecord();
BulletRecord expectedB = RecordBox.get().add("fieldA", "null").add("fieldB", "bar").add(COUNT.getName(), 10L).add("priceSum", 10.0).getRecord();
BulletRecord expectedC = RecordBox.get().add("fieldA", "null").add("fieldB", "null").add(COUNT.getName(), 10L).add("priceSum", 100.0).getRecord();
assertContains(records, expectedA);
assertContains(records, expectedB);
assertContains(records, expectedC);
Assert.assertEquals(groupBy.getRecords(), aggregate.getRecords());
Assert.assertEquals(groupBy.getMetadata().asMap(), aggregate.getMeta().asMap());
}
use of com.yahoo.bullet.aggregations.grouping.GroupOperation.GroupOperationType.SUM in project bullet-core by yahoo.
the class GroupByTest method testCombining.
@Test
public void testCombining() {
List<String> fields = asList("fieldA", "fieldB");
GroupBy groupBy = makeGroupBy(fields, 5, makeGroupOperation(COUNT, null, null), makeGroupOperation(SUM, "price", "priceSum"));
BulletRecord recordA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("price", 3).getRecord();
BulletRecord recordB = RecordBox.get().add("fieldA", "null").add("fieldB", "bar").add("price", 1).getRecord();
IntStream.range(0, 10).mapToObj(i -> recordA).forEach(groupBy::consume);
IntStream.range(0, 9).mapToObj(i -> recordB).forEach(groupBy::consume);
IntStream.range(0, 20).mapToObj(i -> recordA).forEach(groupBy::consume);
groupBy.consume(recordB);
byte[] firstSerialized = groupBy.getData();
// Remake it
groupBy = makeGroupBy(fields, 5, makeGroupOperation(COUNT, null, null), makeGroupOperation(SUM, "price", "priceSum"));
BulletRecord recordC = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add("price", 3).getRecord();
BulletRecord recordD = RecordBox.get().addNull("fieldA").addNull("fieldB").add("price", 10).getRecord();
IntStream.range(0, 30).mapToObj(i -> recordC).forEach(groupBy::consume);
IntStream.range(0, 10).mapToObj(i -> recordD).forEach(groupBy::consume);
byte[] secondSerialized = groupBy.getData();
groupBy = makeGroupBy(fields, 5, makeGroupOperation(COUNT, null, null), makeGroupOperation(SUM, "price", "priceSum"));
groupBy.combine(firstSerialized);
groupBy.combine(secondSerialized);
Clip aggregate = groupBy.getResult();
Assert.assertNotNull(aggregate);
List<BulletRecord> records = aggregate.getRecords();
Assert.assertEquals(records.size(), 3);
// count = 10 + 20 + 30, price = 10*3 + 20*3 + 30*3
BulletRecord expectedA = RecordBox.get().add("fieldA", "foo").add("fieldB", "bar").add(COUNT.getName(), 60L).add("priceSum", 180.0).getRecord();
// count = 9 + 1, price = 9*1 + 1*1
BulletRecord expectedB = RecordBox.get().add("fieldA", "null").add("fieldB", "bar").add(COUNT.getName(), 10L).add("priceSum", 10.0).getRecord();
// count = 10, price = 10*10
BulletRecord expectedC = RecordBox.get().add("fieldA", "null").add("fieldB", "null").add(COUNT.getName(), 10L).add("priceSum", 100.0).getRecord();
assertContains(records, expectedA);
assertContains(records, expectedB);
assertContains(records, expectedC);
Assert.assertEquals(groupBy.getRecords(), aggregate.getRecords());
Assert.assertEquals(groupBy.getMetadata().asMap(), aggregate.getMeta().asMap());
}
use of com.yahoo.bullet.aggregations.grouping.GroupOperation.GroupOperationType.SUM in project bullet-storm by yahoo.
the class JoinBoltTest method testGroupBy.
@Test
public void testGroupBy() {
final int entries = 16;
BulletConfig bulletConfig = GroupByTest.makeConfiguration(entries);
GroupBy groupBy = GroupByTest.makeGroupBy(bulletConfig, singletonMap("fieldA", "A"), entries, AggregationUtils.makeGroupOperation(COUNT, null, "cnt"), AggregationUtils.makeGroupOperation(SUM, "fieldB", "sumB"));
IntStream.range(0, 256).mapToObj(i -> RecordBox.get().add("fieldA", i % 16).add("fieldB", i / 16).getRecord()).forEach(groupBy::consume);
byte[] first = groupBy.getData();
groupBy = GroupByTest.makeGroupBy(bulletConfig, singletonMap("fieldA", "A"), entries, AggregationUtils.makeGroupOperation(COUNT, null, "cnt"), AggregationUtils.makeGroupOperation(SUM, "fieldB", "sumB"));
IntStream.range(256, 1024).mapToObj(i -> RecordBox.get().add("fieldA", i % 16).add("fieldB", i / 16).getRecord()).forEach(groupBy::consume);
byte[] second = groupBy.getData();
// Send generated data to JoinBolt
bolt = new DonableJoinBolt(config, 2, true);
setup(bolt);
List<GroupOperation> operations = asList(new GroupOperation(COUNT, null, "cnt"), new GroupOperation(SUM, "fieldB", "sumB"));
String queryString = makeGroupFilterQuery("ts", singletonList("1"), EQUALS, GROUP, entries, operations, Pair.of("fieldA", "A"));
Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", queryString, EMPTY);
bolt.execute(query);
sendRawByteTuplesTo(bolt, "42", asList(first, second));
Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
bolt.execute(tick);
for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
bolt.execute(tick);
Assert.assertEquals(collector.getEmittedCount(), 0);
}
bolt.execute(tick);
Assert.assertEquals(collector.getEmittedCount(), 2);
String response = (String) collector.getMthElementFromNthTupleEmittedTo(TopologyConstants.RESULT_STREAM, 1, 1).get();
JsonParser parser = new JsonParser();
JsonObject actual = parser.parse(response).getAsJsonObject();
JsonArray actualRecords = actual.get(Clip.RECORDS_KEY).getAsJsonArray();
Assert.assertEquals(actualRecords.size(), 16);
}
Aggregations