Search in sources :

Example 16 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class JoinBoltTest method testMultiJoining.

@Test
public void testMultiJoining() {
    DonableJoinBolt donableJoinBolt = new DonableJoinBolt(config, 2, true);
    bolt = donableJoinBolt;
    setup(bolt);
    Tuple queryA = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 3), EMPTY);
    bolt.execute(queryA);
    donableJoinBolt.shouldBuffer = false;
    Tuple queryB = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "43", makeAggregationQuery(RAW, 3), EMPTY);
    bolt.execute(queryB);
    // This will satisfy the query and will be buffered
    List<BulletRecord> sentFirst = sendRawRecordTuplesTo(bolt, "42", 2);
    // This will satisfy the query and will not be buffered
    List<BulletRecord> sentSecond = sendRawRecordTuplesTo(bolt, "43", 3);
    Tuple emittedFirst = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "43", Clip.of(sentSecond).asJSON(), COMPLETED);
    Tuple emittedSecond = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sentFirst).asJSON(), COMPLETED);
    Assert.assertTrue(wasResultEmitted(emittedFirst));
    Assert.assertFalse(wasResultEmitted(emittedSecond));
    // This will force queryA to finish and start buffering
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    bolt.execute(tick);
    // We need to tick the default query tickout to make queryA emit
    for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
        bolt.execute(tick);
        Assert.assertFalse(wasResultEmitted(emittedSecond));
    }
    // This will cause the emission
    bolt.execute(tick);
    Assert.assertTrue(wasResultEmitted(emittedSecond));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "43", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 2);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 2);
}
Also used : BulletRecord(com.yahoo.bullet.record.BulletRecord) Metadata(com.yahoo.bullet.pubsub.Metadata) Tuple(org.apache.storm.tuple.Tuple) Test(org.testng.annotations.Test) GroupByTest(com.yahoo.bullet.aggregations.GroupByTest) DistributionTest(com.yahoo.bullet.aggregations.DistributionTest) TopKTest(com.yahoo.bullet.aggregations.TopKTest) CountDistinctTest(com.yahoo.bullet.aggregations.CountDistinctTest)

Example 17 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class JoinBoltTest method testRateLimitingOnCombine.

@Test
public void testRateLimitingOnCombine() {
    RateLimitError rateLimitError = new RateLimitError(42.0, config);
    bolt = new RateLimitedJoinBolt(2, rateLimitError, config);
    setup(bolt);
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 10));
    bolt.execute(query);
    // After consuming the 3rd one, it is rate limited and the fourth is not consumed
    List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 4);
    Assert.assertEquals(collector.getEmittedCount(), 2);
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent.subList(0, 3)).add(rateLimitError.makeMeta()).asJSON(), new Metadata(Metadata.Signal.FAIL, null));
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
}
Also used : BulletRecord(com.yahoo.bullet.record.BulletRecord) RateLimitError(com.yahoo.bullet.querying.RateLimitError) Metadata(com.yahoo.bullet.pubsub.Metadata) Tuple(org.apache.storm.tuple.Tuple) Test(org.testng.annotations.Test) GroupByTest(com.yahoo.bullet.aggregations.GroupByTest) DistributionTest(com.yahoo.bullet.aggregations.DistributionTest) TopKTest(com.yahoo.bullet.aggregations.TopKTest) CountDistinctTest(com.yahoo.bullet.aggregations.CountDistinctTest)

Example 18 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class JoinBoltTest method testCounting.

@Test
public void testCounting() {
    bolt = new DonableJoinBolt(config, 5, true);
    setup(bolt);
    String filterQuery = makeGroupFilterQuery("timestamp", asList("1", "2"), EQUALS, GROUP, 1, singletonList(new GroupOperation(COUNT, null, "cnt")));
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", filterQuery, EMPTY);
    bolt.execute(query);
    // Send 5 GroupData with counts 1, 2, 3, 4, 5 to the JoinBolt
    IntStream.range(1, 6).forEach(i -> sendRawByteTuplesTo(bolt, "42", singletonList(getGroupDataWithCount("cnt", i))));
    // 1 + 2 + 3 + 4 + 5
    List<BulletRecord> result = singletonList(RecordBox.get().add("cnt", 15L).getRecord());
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(result).asJSON(), COMPLETED);
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    // Should starts buffering the query for the query tickout
    bolt.execute(tick);
    for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
        bolt.execute(tick);
        Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    }
    bolt.execute(tick);
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}
Also used : BulletRecord(com.yahoo.bullet.record.BulletRecord) Metadata(com.yahoo.bullet.pubsub.Metadata) GroupOperation(com.yahoo.bullet.aggregations.grouping.GroupOperation) Tuple(org.apache.storm.tuple.Tuple) Test(org.testng.annotations.Test) GroupByTest(com.yahoo.bullet.aggregations.GroupByTest) DistributionTest(com.yahoo.bullet.aggregations.DistributionTest) TopKTest(com.yahoo.bullet.aggregations.TopKTest) CountDistinctTest(com.yahoo.bullet.aggregations.CountDistinctTest)

Example 19 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class JoinBoltTest method testTopK.

@Test
public void testTopK() {
    BulletConfig bulletConfig = TopKTest.makeConfiguration(ErrorType.NO_FALSE_NEGATIVES, 16);
    Map<String, String> fields = new HashMap<>();
    fields.put("A", "");
    fields.put("B", "foo");
    TopK topK = TopKTest.makeTopK(bulletConfig, makeAttributes(null, 5L), fields, 2, null);
    IntStream.range(0, 32).mapToObj(i -> RecordBox.get().add("A", i % 8).getRecord()).forEach(topK::consume);
    byte[] first = topK.getData();
    topK = TopKTest.makeTopK(bulletConfig, makeAttributes(null, 5L), fields, 2, null);
    IntStream.range(0, 8).mapToObj(i -> RecordBox.get().add("A", i % 2).getRecord()).forEach(topK::consume);
    byte[] second = topK.getData();
    bolt = new DonableJoinBolt(config, 2, true);
    setup(bolt);
    String aggregationQuery = makeAggregationQuery(TOP_K, 2, 5L, "cnt", Pair.of("A", ""), Pair.of("B", "foo"));
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", aggregationQuery, EMPTY);
    bolt.execute(query);
    sendRawByteTuplesTo(bolt, "42", asList(first, second));
    BulletRecord expectedA = RecordBox.get().add("A", "0").add("foo", "null").add("cnt", 8L).getRecord();
    BulletRecord expectedB = RecordBox.get().add("A", "1").add("foo", "null").add("cnt", 8L).getRecord();
    List<BulletRecord> results = asList(expectedA, expectedB);
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(results).asJSON(), COMPLETED);
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    bolt.execute(tick);
    for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
        bolt.execute(tick);
        Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    }
    bolt.execute(tick);
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}
Also used : JsonObject(com.google.gson.JsonObject) GROUP(com.yahoo.bullet.parsing.Aggregation.Type.GROUP) COUNT(com.yahoo.bullet.aggregations.grouping.GroupOperation.GroupOperationType.COUNT) TopK(com.yahoo.bullet.aggregations.TopK) BulletError(com.yahoo.bullet.common.BulletError) Concept(com.yahoo.bullet.result.Meta.Concept) ParsingError(com.yahoo.bullet.parsing.ParsingError) Test(org.testng.annotations.Test) RecordBox(com.yahoo.bullet.result.RecordBox) ErrorType(com.yahoo.sketches.frequencies.ErrorType) EQUALS(com.yahoo.bullet.parsing.Clause.Operation.EQUALS) PROBABILITY_FIELD(com.yahoo.bullet.aggregations.sketches.QuantileSketch.PROBABILITY_FIELD) Collections.singletonList(java.util.Collections.singletonList) CustomCollector(com.yahoo.bullet.storm.testing.CustomCollector) Pair(org.apache.commons.lang3.tuple.Pair) Arrays.asList(java.util.Arrays.asList) Map(java.util.Map) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Metadata(com.yahoo.bullet.pubsub.Metadata) Mockito.doReturn(org.mockito.Mockito.doReturn) GroupData(com.yahoo.bullet.aggregations.grouping.GroupData) QueryUtils.makeAggregationQuery(com.yahoo.bullet.parsing.QueryUtils.makeAggregationQuery) GroupBy(com.yahoo.bullet.aggregations.GroupBy) ComponentUtils(com.yahoo.bullet.storm.testing.ComponentUtils) BulletRecord(com.yahoo.bullet.record.BulletRecord) AggregationUtils(com.yahoo.bullet.parsing.AggregationUtils) BeforeMethod(org.testng.annotations.BeforeMethod) GroupByTest(com.yahoo.bullet.aggregations.GroupByTest) Fields(org.apache.storm.tuple.Fields) CustomTopologyContext(com.yahoo.bullet.storm.testing.CustomTopologyContext) SlidingRecord(com.yahoo.bullet.windowing.SlidingRecord) DistributionTest(com.yahoo.bullet.aggregations.DistributionTest) Serializable(java.io.Serializable) RateLimitError(com.yahoo.bullet.querying.RateLimitError) GroupOperation(com.yahoo.bullet.aggregations.grouping.GroupOperation) Querier(com.yahoo.bullet.querying.Querier) JsonArray(com.google.gson.JsonArray) List(java.util.List) Distribution(com.yahoo.bullet.aggregations.Distribution) NEGATIVE_INFINITY_START(com.yahoo.bullet.aggregations.sketches.QuantileSketch.NEGATIVE_INFINITY_START) BulletConfig(com.yahoo.bullet.common.BulletConfig) AdditionalAnswers.returnsElementsOf(org.mockito.AdditionalAnswers.returnsElementsOf) Window(com.yahoo.bullet.parsing.Window) IRichBolt(org.apache.storm.topology.IRichBolt) END_EXCLUSIVE(com.yahoo.bullet.aggregations.sketches.QuantileSketch.END_EXCLUSIVE) TupleUtils(com.yahoo.bullet.storm.testing.TupleUtils) IntStream(java.util.stream.IntStream) TopKTest(com.yahoo.bullet.aggregations.TopKTest) Setter(lombok.Setter) TestHelpers.assertJSONEquals(com.yahoo.bullet.storm.testing.TestHelpers.assertJSONEquals) SerializerDeserializer(com.yahoo.bullet.common.SerializerDeserializer) HashMap(java.util.HashMap) JsonParser(com.google.gson.JsonParser) Mockito.spy(org.mockito.Mockito.spy) TestHelpers.getListBytes(com.yahoo.bullet.storm.testing.TestHelpers.getListBytes) Clip(com.yahoo.bullet.result.Clip) SUM(com.yahoo.bullet.aggregations.grouping.GroupOperation.GroupOperationType.SUM) SEPARATOR(com.yahoo.bullet.aggregations.sketches.QuantileSketch.SEPARATOR) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Tuple(org.apache.storm.tuple.Tuple) Assert(org.testng.Assert) CountDistinct(com.yahoo.bullet.aggregations.CountDistinct) AggregationUtils.makeAttributes(com.yahoo.bullet.parsing.AggregationUtils.makeAttributes) Collections.singletonMap(java.util.Collections.singletonMap) TOP_K(com.yahoo.bullet.parsing.Aggregation.Type.TOP_K) Aggregation(com.yahoo.bullet.parsing.Aggregation) Meta(com.yahoo.bullet.result.Meta) COUNT_FIELD(com.yahoo.bullet.aggregations.sketches.QuantileSketch.COUNT_FIELD) POSITIVE_INFINITY_END(com.yahoo.bullet.aggregations.sketches.QuantileSketch.POSITIVE_INFINITY_END) COUNT_DISTINCT(com.yahoo.bullet.parsing.Aggregation.Type.COUNT_DISTINCT) CustomOutputFieldsDeclarer(com.yahoo.bullet.storm.testing.CustomOutputFieldsDeclarer) START_INCLUSIVE(com.yahoo.bullet.aggregations.sketches.QuantileSketch.START_INCLUSIVE) CountDistinctTest(com.yahoo.bullet.aggregations.CountDistinctTest) RAW(com.yahoo.bullet.parsing.Aggregation.Type.RAW) DISTRIBUTION(com.yahoo.bullet.parsing.Aggregation.Type.DISTRIBUTION) QueryUtils.makeGroupFilterQuery(com.yahoo.bullet.parsing.QueryUtils.makeGroupFilterQuery) RANGE_FIELD(com.yahoo.bullet.aggregations.sketches.QuantileSketch.RANGE_FIELD) HashMap(java.util.HashMap) Metadata(com.yahoo.bullet.pubsub.Metadata) TopK(com.yahoo.bullet.aggregations.TopK) BulletRecord(com.yahoo.bullet.record.BulletRecord) BulletConfig(com.yahoo.bullet.common.BulletConfig) Tuple(org.apache.storm.tuple.Tuple) Test(org.testng.annotations.Test) GroupByTest(com.yahoo.bullet.aggregations.GroupByTest) DistributionTest(com.yahoo.bullet.aggregations.DistributionTest) TopKTest(com.yahoo.bullet.aggregations.TopKTest) CountDistinctTest(com.yahoo.bullet.aggregations.CountDistinctTest)

Example 20 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class JoinBoltTest method testRateLimitErrorFromUpstream.

@Test
public void testRateLimitErrorFromUpstream() {
    config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true);
    config.validate();
    setup(bolt);
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY);
    bolt.execute(query);
    List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1);
    Assert.assertEquals(collector.getEmittedCount(), 0);
    Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1));
    RateLimitError rateLimitError = new RateLimitError(2000.0, new BulletConfig());
    Tuple error = TupleUtils.makeIDTuple(TupleClassifier.Type.ERROR_TUPLE, "42", rateLimitError);
    bolt.execute(error);
    Assert.assertEquals(collector.getEmittedCount(), 2);
    Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0));
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(rateLimitError.makeMeta()).asJSON(), new Metadata(Metadata.Signal.FAIL, null));
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}
Also used : BulletRecord(com.yahoo.bullet.record.BulletRecord) RateLimitError(com.yahoo.bullet.querying.RateLimitError) Metadata(com.yahoo.bullet.pubsub.Metadata) Tuple(org.apache.storm.tuple.Tuple) BulletConfig(com.yahoo.bullet.common.BulletConfig) Test(org.testng.annotations.Test) GroupByTest(com.yahoo.bullet.aggregations.GroupByTest) DistributionTest(com.yahoo.bullet.aggregations.DistributionTest) TopKTest(com.yahoo.bullet.aggregations.TopKTest) CountDistinctTest(com.yahoo.bullet.aggregations.CountDistinctTest)

Aggregations

Metadata (com.yahoo.bullet.pubsub.Metadata)45 Test (org.testng.annotations.Test)35 Tuple (org.apache.storm.tuple.Tuple)31 CountDistinctTest (com.yahoo.bullet.aggregations.CountDistinctTest)25 DistributionTest (com.yahoo.bullet.aggregations.DistributionTest)25 TopKTest (com.yahoo.bullet.aggregations.TopKTest)25 BulletRecord (com.yahoo.bullet.record.BulletRecord)25 GroupByTest (com.yahoo.bullet.aggregations.GroupByTest)22 PubSubMessage (com.yahoo.bullet.pubsub.PubSubMessage)11 GroupOperation (com.yahoo.bullet.aggregations.grouping.GroupOperation)7 RateLimitError (com.yahoo.bullet.querying.RateLimitError)7 ArrayList (java.util.ArrayList)6 BulletConfig (com.yahoo.bullet.common.BulletConfig)5 Meta (com.yahoo.bullet.result.Meta)5 CountDistinct (com.yahoo.bullet.aggregations.CountDistinct)4 Distribution (com.yahoo.bullet.aggregations.Distribution)4 TopK (com.yahoo.bullet.aggregations.TopK)4 GroupData (com.yahoo.bullet.aggregations.grouping.GroupData)4 COUNT (com.yahoo.bullet.aggregations.grouping.GroupOperation.GroupOperationType.COUNT)4 COUNT_FIELD (com.yahoo.bullet.aggregations.sketches.QuantileSketch.COUNT_FIELD)4