use of com.yahoo.bullet.aggregations.TopK in project bullet-storm by yahoo.
the class FilterBoltTest method testTopK.
@Test
public void testTopK() {
// 16 records
BulletStormConfig config = new BulletStormConfig(TopKTest.makeConfiguration(ErrorType.NO_FALSE_NEGATIVES, 32));
bolt = ComponentUtils.prepare(new DonableFilterBolt(16, config), collector);
Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(TOP_K, 5, null, "cnt", Pair.of("A", ""), Pair.of("B", "foo")), METADATA);
bolt.execute(query);
IntStream.range(0, 8).mapToObj(i -> RecordBox.get().add("A", i).getRecord()).map(FilterBoltTest::makeRecordTuple).forEach(bolt::execute);
IntStream.range(0, 6).mapToObj(i -> RecordBox.get().add("A", 0).getRecord()).map(FilterBoltTest::makeRecordTuple).forEach(bolt::execute);
IntStream.range(0, 2).mapToObj(i -> RecordBox.get().add("A", 3).getRecord()).map(FilterBoltTest::makeRecordTuple).forEach(bolt::execute);
Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
bolt.execute(tick);
bolt.execute(tick);
Assert.assertEquals(collector.getEmittedCount(), 1);
byte[] rawData = getRawPayloadOfNthTuple(1);
Assert.assertNotNull(rawData);
Map<String, String> fields = new HashMap<>();
fields.put("A", "");
fields.put("B", "foo");
TopK topK = TopKTest.makeTopK(config, makeAttributes("cnt", null), fields, 2, null);
topK.combine(rawData);
List<BulletRecord> records = topK.getRecords();
Assert.assertEquals(records.size(), 2);
BulletRecord expectedA = RecordBox.get().add("A", "0").add("foo", "null").add("cnt", 7L).getRecord();
BulletRecord expectedB = RecordBox.get().add("A", "3").add("foo", "null").add("cnt", 3L).getRecord();
Assert.assertEquals(records.get(0), expectedA);
Assert.assertEquals(records.get(1), expectedB);
}
use of com.yahoo.bullet.aggregations.TopK in project bullet-storm by yahoo.
the class JoinBoltTest method testTopK.
@Test
public void testTopK() {
BulletConfig bulletConfig = TopKTest.makeConfiguration(ErrorType.NO_FALSE_NEGATIVES, 16);
Map<String, String> fields = new HashMap<>();
fields.put("A", "");
fields.put("B", "foo");
TopK topK = TopKTest.makeTopK(bulletConfig, makeAttributes(null, 5L), fields, 2, null);
IntStream.range(0, 32).mapToObj(i -> RecordBox.get().add("A", i % 8).getRecord()).forEach(topK::consume);
byte[] first = topK.getData();
topK = TopKTest.makeTopK(bulletConfig, makeAttributes(null, 5L), fields, 2, null);
IntStream.range(0, 8).mapToObj(i -> RecordBox.get().add("A", i % 2).getRecord()).forEach(topK::consume);
byte[] second = topK.getData();
bolt = new DonableJoinBolt(config, 2, true);
setup(bolt);
String aggregationQuery = makeAggregationQuery(TOP_K, 2, 5L, "cnt", Pair.of("A", ""), Pair.of("B", "foo"));
Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", aggregationQuery, EMPTY);
bolt.execute(query);
sendRawByteTuplesTo(bolt, "42", asList(first, second));
BulletRecord expectedA = RecordBox.get().add("A", "0").add("foo", "null").add("cnt", 8L).getRecord();
BulletRecord expectedB = RecordBox.get().add("A", "1").add("foo", "null").add("cnt", 8L).getRecord();
List<BulletRecord> results = asList(expectedA, expectedB);
Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(results).asJSON(), COMPLETED);
Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
bolt.execute(tick);
for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
bolt.execute(tick);
Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
}
bolt.execute(tick);
Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}
Aggregations