Examples with Metadata com.yahoo.bullet.pubsub.Metadata used on opensource projects

Example 6 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class FilterBoltTest method testKillSignal.

@Test
public void testKillSignal() {
    Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeSimpleAggregationFilterQuery("field", singletonList("b235gf23b"), EQUALS, RAW, 5, Window.Unit.RECORD, 1, Window.Unit.RECORD, 1), METADATA);
    bolt.execute(query);
    BulletRecord record = RecordBox.get().add("field", "b235gf23b").getRecord();
    Tuple matching = makeRecordTuple(record);
    bolt.execute(matching);
    bolt.execute(matching);
    Tuple expected = makeSlidingTuple(TupleClassifier.Type.DATA_TUPLE, "42", record);
    Assert.assertTrue(wasRawRecordEmittedTo(TopologyConstants.DATA_STREAM, 2, expected));
    Assert.assertEquals(collector.getEmittedCount(), 2);
    Tuple kill = makeIDTuple(TupleClassifier.Type.METADATA_TUPLE, "42", new Metadata(Metadata.Signal.KILL, null));
    bolt.execute(kill);
    bolt.execute(matching);
    bolt.execute(matching);
    Assert.assertEquals(collector.getEmittedCount(), 2);
}

Example 7 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class JoinBoltTest method testWindowDoneWhileBuffering.

@Test
public void testWindowDoneWhileBuffering() {
    config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true);
    config.validate();
    bolt = new DonableJoinBolt(config, 2, true);
    setup(bolt);
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 5, Window.Unit.RECORD, 1, Window.Unit.RECORD, 1), EMPTY);
    bolt.execute(query);
    // Begins buffering in window tickout buffer here
    List<BulletRecord> sentFirst = sendSlidingWindowWithRawRecordTuplesTo(bolt, "42", 1);
    Assert.assertEquals(collector.getEmittedCount(), 0);
    List<BulletRecord> sentSecond = sendSlidingWindowWithRawRecordTuplesTo(bolt, "42", 1);
    Assert.assertEquals(collector.getEmittedCount(), 0);
    Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1));
    // Will be isDone() here and will now be moved into query tickout buffer
    List<BulletRecord> sentThird = sendSlidingWindowWithRawRecordTuplesTo(bolt, "42", 3);
    // Wait to tickout
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
        Assert.assertEquals(collector.getEmittedCount(), 0);
        bolt.execute(tick);
    }
    bolt.execute(tick);
    Assert.assertEquals(collector.getEmittedCount(), 2);
    Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0));
    List<BulletRecord> sent = new ArrayList<>();
    sent.addAll(sentFirst);
    sent.addAll(sentSecond);
    sent.addAll(sentThird);
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}

Example 8 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class JoinBoltTest method testCompleteSignal.

@Test
public void testCompleteSignal() {
    config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true);
    config.validate();
    setup(bolt);
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY);
    bolt.execute(query);
    List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1);
    Assert.assertEquals(collector.getEmittedCount(), 0);
    Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(1));
    Tuple complete = TupleUtils.makeIDTuple(TupleClassifier.Type.METADATA_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    bolt.execute(complete);
    Assert.assertEquals(collector.getEmittedCount(), 0);
    Assert.assertEquals(context.getLongMetric(TopologyConstants.ACTIVE_QUERIES_METRIC), Long.valueOf(0));
}

Example 9 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class JoinBoltTest method testJoining.

@Test
public void testJoining() {
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY);
    bolt.execute(query);
    List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42");
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED);
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}

Example 10 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class JoinBoltTest method testJoiningAfterLateArrivalBeforeTickout.

@Test
public void testJoiningAfterLateArrivalBeforeTickout() {
    bolt = new DonableJoinBolt(config, 2, true);
    setup(bolt);
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 3), EMPTY);
    bolt.execute(query);
    // This calls isDone twice. So the query is done on the next tick
    List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 2);
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED);
    // Tick once to get the query done rotated into buffer.
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    bolt.execute(tick);
    // Now we satisfy the aggregation and tick to see if it causes an emission
    List<BulletRecord> sentLate = sendRawRecordTuplesTo(bolt, "42", 1);
    sent.addAll(sentLate);
    for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
        bolt.execute(tick);
        Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    }
    // The expected record now should contain the sentLate ones too
    bolt.execute(tick);
    expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED);
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}