Search in sources :

Example 11 with CustomCollector

use of com.yahoo.bullet.storm.testing.CustomCollector in project bullet-storm by yahoo.

the class FilterBoltTest method testFilteringLatency.

@Test
public void testFilteringLatency() {
    config = new BulletStormConfig();
    // Don't use the overridden aggregation default size but turn on built in metrics
    config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true);
    collector = new CustomCollector();
    CustomTopologyContext context = new CustomTopologyContext();
    bolt = new FilterBolt(TopologyConstants.RECORD_COMPONENT, config);
    ComponentUtils.prepare(new HashMap<>(), bolt, context, collector);
    Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeFieldFilterQuery("bar"), METADATA);
    bolt.execute(query);
    BulletRecord record = RecordBox.get().add("field", "foo").getRecord();
    long start = System.currentTimeMillis();
    IntStream.range(0, 10).mapToObj(i -> makeRecordTuple(record, System.currentTimeMillis())).forEach(bolt::execute);
    long end = System.currentTimeMillis();
    double actualLatecy = context.getDoubleMetric(TopologyConstants.LATENCY_METRIC);
    Assert.assertTrue(actualLatecy <= end - start);
}
Also used : Arrays(java.util.Arrays) GREATER_THAN(com.yahoo.bullet.parsing.Clause.Operation.GREATER_THAN) GROUP(com.yahoo.bullet.parsing.Aggregation.Type.GROUP) COUNT(com.yahoo.bullet.aggregations.grouping.GroupOperation.GroupOperationType.COUNT) TopK(com.yahoo.bullet.aggregations.TopK) Test(org.testng.annotations.Test) RecordBox(com.yahoo.bullet.result.RecordBox) ErrorType(com.yahoo.sketches.frequencies.ErrorType) EQUALS(com.yahoo.bullet.parsing.Clause.Operation.EQUALS) PROBABILITY_FIELD(com.yahoo.bullet.aggregations.sketches.QuantileSketch.PROBABILITY_FIELD) QueryUtils.makeProjectionQuery(com.yahoo.bullet.parsing.QueryUtils.makeProjectionQuery) QueryUtils.makeSimpleAggregationFilterQuery(com.yahoo.bullet.parsing.QueryUtils.makeSimpleAggregationFilterQuery) Collections.singletonList(java.util.Collections.singletonList) CustomCollector(com.yahoo.bullet.storm.testing.CustomCollector) Pair(org.apache.commons.lang3.tuple.Pair) TupleUtils.makeTuple(com.yahoo.bullet.storm.testing.TupleUtils.makeTuple) Arrays.asList(java.util.Arrays.asList) Map(java.util.Map) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Metadata(com.yahoo.bullet.pubsub.Metadata) Mockito.doReturn(org.mockito.Mockito.doReturn) GroupData(com.yahoo.bullet.aggregations.grouping.GroupData) QueryUtils.makeAggregationQuery(com.yahoo.bullet.parsing.QueryUtils.makeAggregationQuery) ComponentUtils(com.yahoo.bullet.storm.testing.ComponentUtils) BulletRecord(com.yahoo.bullet.record.BulletRecord) BeforeMethod(org.testng.annotations.BeforeMethod) FilterUtils.getFieldFilter(com.yahoo.bullet.parsing.FilterUtils.getFieldFilter) Fields(org.apache.storm.tuple.Fields) CustomTopologyContext(com.yahoo.bullet.storm.testing.CustomTopologyContext) SlidingRecord(com.yahoo.bullet.windowing.SlidingRecord) QueryUtils.makeProjectionFilterQuery(com.yahoo.bullet.parsing.QueryUtils.makeProjectionFilterQuery) DistributionTest(com.yahoo.bullet.aggregations.DistributionTest) NOT_EQUALS(com.yahoo.bullet.parsing.Clause.Operation.NOT_EQUALS) TupleUtils.makeRawTuple(com.yahoo.bullet.storm.testing.TupleUtils.makeRawTuple) RateLimitError(com.yahoo.bullet.querying.RateLimitError) GroupOperation(com.yahoo.bullet.aggregations.grouping.GroupOperation) Querier(com.yahoo.bullet.querying.Querier) QueryUtils.makeFieldFilterQuery(com.yahoo.bullet.parsing.QueryUtils.makeFieldFilterQuery) List(java.util.List) Distribution(com.yahoo.bullet.aggregations.Distribution) NEGATIVE_INFINITY_START(com.yahoo.bullet.aggregations.sketches.QuantileSketch.NEGATIVE_INFINITY_START) BulletConfig(com.yahoo.bullet.common.BulletConfig) AdditionalAnswers.returnsElementsOf(org.mockito.AdditionalAnswers.returnsElementsOf) Window(com.yahoo.bullet.parsing.Window) Optional(java.util.Optional) OR(com.yahoo.bullet.parsing.Clause.Operation.OR) QueryUtils.makeFilterQuery(com.yahoo.bullet.parsing.QueryUtils.makeFilterQuery) END_EXCLUSIVE(com.yahoo.bullet.aggregations.sketches.QuantileSketch.END_EXCLUSIVE) TupleUtils(com.yahoo.bullet.storm.testing.TupleUtils) IntStream(java.util.stream.IntStream) TopKTest(com.yahoo.bullet.aggregations.TopKTest) SerializerDeserializer(com.yahoo.bullet.common.SerializerDeserializer) HashMap(java.util.HashMap) Mockito.spy(org.mockito.Mockito.spy) SEPARATOR(com.yahoo.bullet.aggregations.sketches.QuantileSketch.SEPARATOR) TestHelpers(com.yahoo.bullet.storm.testing.TestHelpers) ArrayList(java.util.ArrayList) Tuple(org.apache.storm.tuple.Tuple) Assert(org.testng.Assert) FilterUtils.makeClause(com.yahoo.bullet.parsing.FilterUtils.makeClause) CountDistinct(com.yahoo.bullet.aggregations.CountDistinct) AggregationUtils.makeAttributes(com.yahoo.bullet.parsing.AggregationUtils.makeAttributes) TOP_K(com.yahoo.bullet.parsing.Aggregation.Type.TOP_K) COUNT_FIELD(com.yahoo.bullet.aggregations.sketches.QuantileSketch.COUNT_FIELD) POSITIVE_INFINITY_END(com.yahoo.bullet.aggregations.sketches.QuantileSketch.POSITIVE_INFINITY_END) AND(com.yahoo.bullet.parsing.Clause.Operation.AND) Utils(org.apache.storm.utils.Utils) COUNT_DISTINCT(com.yahoo.bullet.parsing.Aggregation.Type.COUNT_DISTINCT) CustomOutputFieldsDeclarer(com.yahoo.bullet.storm.testing.CustomOutputFieldsDeclarer) START_INCLUSIVE(com.yahoo.bullet.aggregations.sketches.QuantileSketch.START_INCLUSIVE) CountDistinctTest(com.yahoo.bullet.aggregations.CountDistinctTest) RAW(com.yahoo.bullet.parsing.Aggregation.Type.RAW) DISTRIBUTION(com.yahoo.bullet.parsing.Aggregation.Type.DISTRIBUTION) QueryUtils.makeGroupFilterQuery(com.yahoo.bullet.parsing.QueryUtils.makeGroupFilterQuery) RANGE_FIELD(com.yahoo.bullet.aggregations.sketches.QuantileSketch.RANGE_FIELD) TupleUtils.makeIDTuple(com.yahoo.bullet.storm.testing.TupleUtils.makeIDTuple) BulletRecord(com.yahoo.bullet.record.BulletRecord) CustomCollector(com.yahoo.bullet.storm.testing.CustomCollector) CustomTopologyContext(com.yahoo.bullet.storm.testing.CustomTopologyContext) TupleUtils.makeTuple(com.yahoo.bullet.storm.testing.TupleUtils.makeTuple) TupleUtils.makeRawTuple(com.yahoo.bullet.storm.testing.TupleUtils.makeRawTuple) Tuple(org.apache.storm.tuple.Tuple) TupleUtils.makeIDTuple(com.yahoo.bullet.storm.testing.TupleUtils.makeIDTuple) Test(org.testng.annotations.Test) DistributionTest(com.yahoo.bullet.aggregations.DistributionTest) TopKTest(com.yahoo.bullet.aggregations.TopKTest) CountDistinctTest(com.yahoo.bullet.aggregations.CountDistinctTest)

Example 12 with CustomCollector

use of com.yahoo.bullet.storm.testing.CustomCollector in project bullet-storm by yahoo.

the class FilterBoltTest method setup.

@BeforeMethod
public void setup() {
    collector = new CustomCollector();
    config = oneRecordConfig();
    bolt = ComponentUtils.prepare(new FilterBolt(TopologyConstants.RECORD_COMPONENT, config), collector);
}
Also used : CustomCollector(com.yahoo.bullet.storm.testing.CustomCollector) BeforeMethod(org.testng.annotations.BeforeMethod)

Aggregations

CustomCollector (com.yahoo.bullet.storm.testing.CustomCollector)12 Test (org.testng.annotations.Test)8 TupleUtils.makeIDTuple (com.yahoo.bullet.storm.testing.TupleUtils.makeIDTuple)6 Tuple (org.apache.storm.tuple.Tuple)6 Querier (com.yahoo.bullet.querying.Querier)4 CustomTopologyContext (com.yahoo.bullet.storm.testing.CustomTopologyContext)4 BeforeMethod (org.testng.annotations.BeforeMethod)4 Metadata (com.yahoo.bullet.pubsub.Metadata)3 CountDistinctTest (com.yahoo.bullet.aggregations.CountDistinctTest)2 DistributionTest (com.yahoo.bullet.aggregations.DistributionTest)2 TopKTest (com.yahoo.bullet.aggregations.TopKTest)2 CountDistinct (com.yahoo.bullet.aggregations.CountDistinct)1 Distribution (com.yahoo.bullet.aggregations.Distribution)1 TopK (com.yahoo.bullet.aggregations.TopK)1 GroupData (com.yahoo.bullet.aggregations.grouping.GroupData)1 GroupOperation (com.yahoo.bullet.aggregations.grouping.GroupOperation)1 COUNT (com.yahoo.bullet.aggregations.grouping.GroupOperation.GroupOperationType.COUNT)1 COUNT_FIELD (com.yahoo.bullet.aggregations.sketches.QuantileSketch.COUNT_FIELD)1 END_EXCLUSIVE (com.yahoo.bullet.aggregations.sketches.QuantileSketch.END_EXCLUSIVE)1 NEGATIVE_INFINITY_START (com.yahoo.bullet.aggregations.sketches.QuantileSketch.NEGATIVE_INFINITY_START)1