Examples with BulletConfig com.yahoo.bullet.common.BulletConfig used on opensource projects

Example 51 with BulletConfig

use of com.yahoo.bullet.common.BulletConfig in project bullet-core by yahoo.

the class SlidingRecordTest method setup.

@BeforeMethod
private void setup() {
    strategy = new MockStrategy();
    config = new BulletConfig();
    addMetadata(config, ALL_METADATA);
}

Example 52 with BulletConfig

use of com.yahoo.bullet.common.BulletConfig in project bullet-storm by yahoo.

the class JoinBoltTest method testRateLimitErrorFromUpstreamWithoutQuery.

@Test
public void testRateLimitErrorFromUpstreamWithoutQuery() {
    RateLimitError rateLimitError = new RateLimitError(2000.0, new BulletConfig());
    Tuple error = TupleUtils.makeIDTuple(TupleClassifier.Type.ERROR_TUPLE, "42", rateLimitError);
    bolt.execute(error);
    Assert.assertEquals(collector.getEmittedCount(), 0);
}

Example 53 with BulletConfig

use of com.yahoo.bullet.common.BulletConfig in project bullet-storm by yahoo.

the class JoinBoltTest method testCountDistinct.

@Test
public void testCountDistinct() {
    BulletConfig bulletConfig = CountDistinctTest.makeConfiguration(8, 512);
    CountDistinct distinct = CountDistinctTest.makeCountDistinct(bulletConfig, singletonList("field"));
    IntStream.range(0, 256).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(distinct::consume);
    byte[] first = distinct.getData();
    distinct = CountDistinctTest.makeCountDistinct(bulletConfig, singletonList("field"));
    IntStream.range(128, 256).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(distinct::consume);
    byte[] second = distinct.getData();
    // Send generated data to JoinBolt
    bolt = new DonableJoinBolt(config, 2, true);
    setup(bolt);
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(COUNT_DISTINCT, 1, null, Pair.of("field", "field")), EMPTY);
    bolt.execute(query);
    sendRawByteTuplesTo(bolt, "42", asList(first, second));
    List<BulletRecord> result = singletonList(RecordBox.get().add(CountDistinct.DEFAULT_NEW_NAME, 256.0).getRecord());
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(result).asJSON(), COMPLETED);
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    bolt.execute(tick);
    for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
        bolt.execute(tick);
        Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    }
    bolt.execute(tick);
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}

Example 54 with BulletConfig

use of com.yahoo.bullet.common.BulletConfig in project bullet-storm by yahoo.

the class JoinBoltTest method testTopK.

@Test
public void testTopK() {
    BulletConfig bulletConfig = TopKTest.makeConfiguration(ErrorType.NO_FALSE_NEGATIVES, 16);
    Map<String, String> fields = new HashMap<>();
    fields.put("A", "");
    fields.put("B", "foo");
    TopK topK = TopKTest.makeTopK(bulletConfig, makeAttributes(null, 5L), fields, 2, null);
    IntStream.range(0, 32).mapToObj(i -> RecordBox.get().add("A", i % 8).getRecord()).forEach(topK::consume);
    byte[] first = topK.getData();
    topK = TopKTest.makeTopK(bulletConfig, makeAttributes(null, 5L), fields, 2, null);
    IntStream.range(0, 8).mapToObj(i -> RecordBox.get().add("A", i % 2).getRecord()).forEach(topK::consume);
    byte[] second = topK.getData();
    bolt = new DonableJoinBolt(config, 2, true);
    setup(bolt);
    String aggregationQuery = makeAggregationQuery(TOP_K, 2, 5L, "cnt", Pair.of("A", ""), Pair.of("B", "foo"));
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", aggregationQuery, EMPTY);
    bolt.execute(query);
    sendRawByteTuplesTo(bolt, "42", asList(first, second));
    BulletRecord expectedA = RecordBox.get().add("A", "0").add("foo", "null").add("cnt", 8L).getRecord();
    BulletRecord expectedB = RecordBox.get().add("A", "1").add("foo", "null").add("cnt", 8L).getRecord();
    List<BulletRecord> results = asList(expectedA, expectedB);
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(results).asJSON(), COMPLETED);
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    bolt.execute(tick);
    for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
        bolt.execute(tick);
        Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    }
    bolt.execute(tick);
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}

Example 55 with BulletConfig

use of com.yahoo.bullet.common.BulletConfig in project bullet-storm by yahoo.

the class JoinBoltTest method testRateLimitErrorFromUpstream.

@Test
public void testRateLimitErrorFromUpstream() {
    config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true);
    config.validate();
    setup(bolt);
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY);
    bolt.execute(query);
    List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1);
    Assert.assertEquals(collector.getEmittedCount(), 0);
    Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1));
    RateLimitError rateLimitError = new RateLimitError(2000.0, new BulletConfig());
    Tuple error = TupleUtils.makeIDTuple(TupleClassifier.Type.ERROR_TUPLE, "42", rateLimitError);
    bolt.execute(error);
    Assert.assertEquals(collector.getEmittedCount(), 2);
    Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0));
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).add(rateLimitError.makeMeta()).asJSON(), new Metadata(Metadata.Signal.FAIL, null));
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}