Examples with BulletRecord com.yahoo.bullet.record.BulletRecord used on opensource projects

Example 76 with BulletRecord

use of com.yahoo.bullet.record.BulletRecord in project bullet-storm by yahoo.

the class FilterBolt method onRecord.

private void onRecord(Tuple tuple) {
    BulletRecord record = (BulletRecord) tuple.getValue(TopologyConstants.RECORD_POSITION);
    handleCategorizedQueries(new QueryCategorizer().categorize(record, queries));
}

Example 77 with BulletRecord

use of com.yahoo.bullet.record.BulletRecord in project bullet-storm by yahoo.

the class FilterBoltTest method testComplexFilterQuery.

@SuppressWarnings("unchecked")
@Test
public void testComplexFilterQuery() {
    Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeFilterQuery(OR, makeClause(AND, getFieldFilter("field", EQUALS, "abc"), makeClause(OR, makeClause(AND, getFieldFilter("experience", EQUALS, "app", "tv"), getFieldFilter("pid", EQUALS, "1", "2")), getFieldFilter("mid", GREATER_THAN, "10"))), makeClause(AND, getFieldFilter("demographic_map.age", GREATER_THAN, "65"), getFieldFilter("filter_map.is_fake_event", EQUALS, "true"))), METADATA);
    bolt.execute(query);
    // first clause is true : field == "abc", experience == "app" or "tv", mid < 10
    BulletRecord recordA = RecordBox.get().add("field", "abc").add("experience", "tv").add("mid", 11).getRecord();
    // second clause is false: age > 65 and is_fake_event == null
    BulletRecord recordB = RecordBox.get().addMap("demographic_map", Pair.of("age", "67")).getRecord();
    Tuple nonMatching = makeRecordTuple(recordB);
    bolt.execute(nonMatching);
    bolt.execute(nonMatching);
    Tuple matching = makeRecordTuple(recordA);
    bolt.execute(matching);
    BulletRecord expectedRecord = RecordBox.get().add("field", "abc").add("experience", "tv").add("mid", 11).getRecord();
    BulletRecord notExpectedRecord = RecordBox.get().addMap("demographic_map", Pair.of("age", "67")).getRecord();
    Tuple expected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", expectedRecord);
    Tuple notExpected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", notExpectedRecord);
    Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 1, expected));
    Assert.assertFalse(wasRawRecordEmitted(notExpected));
}

Example 78 with BulletRecord

use of com.yahoo.bullet.record.BulletRecord in project bullet-storm by yahoo.

the class FilterBoltTest method testDifferentQueryMatchingSameTuple.

@Test
public void testDifferentQueryMatchingSameTuple() {
    Tuple queryA = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeFieldFilterQuery("b235gf23b"), METADATA);
    Tuple queryB = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "43", makeFilterQuery("timestamp", asList("1", "2", "3", "45"), EQUALS), METADATA);
    bolt.execute(queryA);
    bolt.execute(queryB);
    BulletRecord record = RecordBox.get().add("field", "b235gf23b").add("timestamp", 45L).getRecord();
    Tuple matching = makeRecordTuple(record);
    bolt.execute(matching);
    Tuple expectedA = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", record);
    Tuple expectedB = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "43", record);
    Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 1, expectedA));
    Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 1, expectedB));
}

Example 79 with BulletRecord

use of com.yahoo.bullet.record.BulletRecord in project bullet-storm by yahoo.

the class FilterBoltTest method testTuplesCustomSource.

@Test
public void testTuplesCustomSource() {
    bolt = ComponentUtils.prepare(new FilterBolt("CustomSource", oneRecordConfig()), collector);
    Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeFieldFilterQuery("b235gf23b"), METADATA);
    bolt.execute(query);
    BulletRecord record = RecordBox.get().add("field", "b235gf23b").getRecord();
    Tuple matching = TupleUtils.makeRawTuple("CustomSource", TopologyConstants.RECORD_STREAM, record);
    bolt.execute(matching);
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    bolt.execute(tick);
    BulletRecord anotherRecord = RecordBox.get().add("field", "wontmatch").getRecord();
    Tuple nonMatching = TupleUtils.makeRawTuple("CustomSource", TopologyConstants.RECORD_STREAM, anotherRecord);
    bolt.execute(nonMatching);
    Tuple expected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", record);
    Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 1, expected));
    Tuple notExpected = makeDataTuple(TupleClassifier.Type.DATA_TUPLE, "42", anotherRecord);
    Assert.assertFalse(wasRawRecordEmitted(notExpected));
}

Example 80 with BulletRecord

use of com.yahoo.bullet.record.BulletRecord in project bullet-storm by yahoo.

the class FilterBoltTest method testTopK.

@Test
public void testTopK() {
    // 16 records
    BulletStormConfig config = new BulletStormConfig(TopKTest.makeConfiguration(ErrorType.NO_FALSE_NEGATIVES, 32));
    bolt = ComponentUtils.prepare(new DonableFilterBolt(16, config), collector);
    Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(TOP_K, 5, null, "cnt", Pair.of("A", ""), Pair.of("B", "foo")), METADATA);
    bolt.execute(query);
    IntStream.range(0, 8).mapToObj(i -> RecordBox.get().add("A", i).getRecord()).map(FilterBoltTest::makeRecordTuple).forEach(bolt::execute);
    IntStream.range(0, 6).mapToObj(i -> RecordBox.get().add("A", 0).getRecord()).map(FilterBoltTest::makeRecordTuple).forEach(bolt::execute);
    IntStream.range(0, 2).mapToObj(i -> RecordBox.get().add("A", 3).getRecord()).map(FilterBoltTest::makeRecordTuple).forEach(bolt::execute);
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    bolt.execute(tick);
    bolt.execute(tick);
    Assert.assertEquals(collector.getEmittedCount(), 1);
    byte[] rawData = getRawPayloadOfNthTuple(1);
    Assert.assertNotNull(rawData);
    Map<String, String> fields = new HashMap<>();
    fields.put("A", "");
    fields.put("B", "foo");
    TopK topK = TopKTest.makeTopK(config, makeAttributes("cnt", null), fields, 2, null);
    topK.combine(rawData);
    List<BulletRecord> records = topK.getRecords();
    Assert.assertEquals(records.size(), 2);
    BulletRecord expectedA = RecordBox.get().add("A", "0").add("foo", "null").add("cnt", 7L).getRecord();
    BulletRecord expectedB = RecordBox.get().add("A", "3").add("foo", "null").add("cnt", 3L).getRecord();
    Assert.assertEquals(records.get(0), expectedA);
    Assert.assertEquals(records.get(1), expectedB);
}