Examples with RateLimitError com.yahoo.bullet.querying.RateLimitError used on opensource projects

Example 1 with RateLimitError

use of com.yahoo.bullet.querying.RateLimitError in project bullet-storm by yahoo.

the class JoinBolt method onError.

private void onError(Tuple tuple) {
    String id = tuple.getString(TopologyConstants.ID_POSITION);
    Querier querier = getQuery(id);
    if (querier == null) {
        log.debug("Received error for {} without the query existing", id);
        // TODO Might later create this query whose error ignored here. This is a leak.
        return;
    }
    RateLimitError error = (RateLimitError) tuple.getValue(TopologyConstants.ERROR_POSITION);
    emitRateLimitError(id, querier, error);
}

Example 2 with RateLimitError

use of com.yahoo.bullet.querying.RateLimitError in project bullet-storm by yahoo.

the class FilterBoltTest method testRateLimiting.

@Test
public void testRateLimiting() {
    config = new BulletStormConfig();
    RateLimitError rateLimitError = new RateLimitError(42.0, config);
    bolt = new RateLimitedFilterBolt(2, rateLimitError, config);
    bolt = ComponentUtils.prepare(new HashMap<>(), bolt, collector);
    Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeSimpleAggregationFilterQuery("field", singletonList("b235gf23b"), EQUALS, RAW, 100, Window.Unit.RECORD, 1, Window.Unit.RECORD, 1), METADATA);
    bolt.execute(query);
    BulletRecord record = RecordBox.get().add("field", "b235gf23b").getRecord();
    Tuple matching = makeRecordTuple(record);
    bolt.execute(matching);
    bolt.execute(matching);
    Tuple expected = makeSlidingTuple(TupleClassifier.Type.DATA_TUPLE, "42", record);
    Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 2, expected));
    bolt.execute(matching);
    Tuple error = TupleUtils.makeIDTuple(TupleClassifier.Type.ERROR_TUPLE, "42", rateLimitError);
    Assert.assertTrue(collector.wasNthEmitted(error, 3));
}

Example 3 with RateLimitError

use of com.yahoo.bullet.querying.RateLimitError in project bullet-storm by yahoo.

the class JoinBoltTest method testRateLimitErrorFromUpstreamWithoutQuery.

@Test
public void testRateLimitErrorFromUpstreamWithoutQuery() {
    RateLimitError rateLimitError = new RateLimitError(2000.0, new BulletConfig());
    Tuple error = TupleUtils.makeIDTuple(TupleClassifier.Type.ERROR_TUPLE, "42", rateLimitError);
    bolt.execute(error);
    Assert.assertEquals(collector.getEmittedCount(), 0);
}

Example 4 with RateLimitError

use of com.yahoo.bullet.querying.RateLimitError in project bullet-storm by yahoo.

the class JoinBoltTest method testRateLimitingOnCombine.

@Test
public void testRateLimitingOnCombine() {
    RateLimitError rateLimitError = new RateLimitError(42.0, config);
    bolt = new RateLimitedJoinBolt(2, rateLimitError, config);
    setup(bolt);
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 10));
    bolt.execute(query);
    // After consuming the 3rd one, it is rate limited and the fourth is not consumed
    List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 4);
    Assert.assertEquals(collector.getEmittedCount(), 2);
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent.subList(0, 3)).add(rateLimitError.makeMeta()).asJSON(), new Metadata(Metadata.Signal.FAIL, null));
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
}

Example 5 with RateLimitError

use of com.yahoo.bullet.querying.RateLimitError in project bullet-storm by yahoo.

the class JoinBoltTest method testRateLimitErrorFromUpstream.

@Test
public void testRateLimitErrorFromUpstream() {
    config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true);
    config.validate();
    setup(bolt);
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY);
    bolt.execute(query);
    List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1);
    Assert.assertEquals(collector.getEmittedCount(), 0);
    Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1));
    RateLimitError rateLimitError = new RateLimitError(2000.0, new BulletConfig());
    Tuple error = TupleUtils.makeIDTuple(TupleClassifier.Type.ERROR_TUPLE, "42", rateLimitError);
    bolt.execute(error);
    Assert.assertEquals(collector.getEmittedCount(), 2);
    Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0));
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(rateLimitError.makeMeta()).asJSON(), new Metadata(Metadata.Signal.FAIL, null));
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}