Examples with Metadata com.yahoo.bullet.pubsub.Metadata used on opensource projects

Example 21 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class JoinBoltTest method testDistribution.

@Test
public void testDistribution() {
    BulletConfig bulletConfig = DistributionTest.makeConfiguration(10, 128);
    Distribution distribution = DistributionTest.makeDistribution(bulletConfig, makeAttributes(Distribution.Type.PMF, 3), "field", 10, null);
    IntStream.range(0, 50).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(distribution::consume);
    byte[] first = distribution.getData();
    distribution = DistributionTest.makeDistribution(bulletConfig, makeAttributes(Distribution.Type.PMF, 3), "field", 10, null);
    IntStream.range(50, 101).mapToObj(i -> RecordBox.get().add("field", i).getRecord()).forEach(distribution::consume);
    byte[] second = distribution.getData();
    bolt = new DonableJoinBolt(config, 2, true);
    setup(bolt);
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(DISTRIBUTION, 10, Distribution.Type.PMF, "field", null, null, null, null, 3), EMPTY);
    bolt.execute(query);
    sendRawByteTuplesTo(bolt, "42", asList(first, second));
    BulletRecord expectedA = RecordBox.get().add(RANGE_FIELD, NEGATIVE_INFINITY_START + SEPARATOR + 0.0 + END_EXCLUSIVE).add(COUNT_FIELD, 0.0).add(PROBABILITY_FIELD, 0.0).getRecord();
    BulletRecord expectedB = RecordBox.get().add(RANGE_FIELD, START_INCLUSIVE + 0.0 + SEPARATOR + 50.0 + END_EXCLUSIVE).add(COUNT_FIELD, 50.0).add(PROBABILITY_FIELD, 50.0 / 101).getRecord();
    BulletRecord expectedC = RecordBox.get().add(RANGE_FIELD, START_INCLUSIVE + 50.0 + SEPARATOR + 100.0 + END_EXCLUSIVE).add(COUNT_FIELD, 50.0).add(PROBABILITY_FIELD, 50.0 / 101).getRecord();
    BulletRecord expectedD = RecordBox.get().add(RANGE_FIELD, START_INCLUSIVE + 100.0 + SEPARATOR + POSITIVE_INFINITY_END).add(COUNT_FIELD, 1.0).add(PROBABILITY_FIELD, 1.0 / 101).getRecord();
    List<BulletRecord> results = asList(expectedA, expectedB, expectedC, expectedD);
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(results).asJSON(), COMPLETED);
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    bolt.execute(tick);
    for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
        bolt.execute(tick);
        Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    }
    bolt.execute(tick);
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}

Example 22 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class JoinBoltTest method testQueryNotDoneButIsTimeBased.

@Test
public void testQueryNotDoneButIsTimeBased() {
    // This bolt will be done after combining RAW_MAX_SIZE - 1 times and is a shouldBuffer, so it is buffered.
    bolt = new DonableJoinBolt(config, RAW_MAX_SIZE - 1, true);
    setup(bolt);
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", "{}", EMPTY);
    bolt.execute(query);
    List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", RAW_MAX_SIZE - 1);
    Tuple tick = TupleUtils.makeTuple(TupleClassifier.Type.TICK_TUPLE);
    // Should make isDone true but query is a time based window so it gets buffered.
    bolt.execute(tick);
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(sent).asJSON(), COMPLETED);
    // We need to tick the default query tickout to make the query emit
    for (int i = 0; i < BulletStormConfig.DEFAULT_JOIN_BOLT_QUERY_TICK_TIMEOUT - 1; ++i) {
        bolt.execute(tick);
        Assert.assertFalse(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    }
    // Should emit on the last tick
    bolt.execute(tick);
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}

Example 23 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class JoinBoltTest method testRawQueryDoneButNotTimedOutWithExcessRecords.

@Test
public void testRawQueryDoneButNotTimedOutWithExcessRecords() {
    Tuple query = TupleUtils.makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeAggregationQuery(RAW, 5), EMPTY);
    bolt.execute(query);
    // This will send 2 batches of 3 records each (total of 6 records).
    List<BulletRecord> sent = sendRawRecordTuplesTo(bolt, "42", 5, 3);
    List<BulletRecord> actualSent = sent.subList(0, 5);
    Tuple expected = TupleUtils.makeTuple(TupleClassifier.Type.RESULT_TUPLE, "42", Clip.of(actualSent).asJSON(), COMPLETED);
    Assert.assertTrue(wasResultEmittedTo(TopologyConstants.RESULT_STREAM, expected));
    Tuple metadata = TupleUtils.makeTuple(TupleClassifier.Type.FEEDBACK_TUPLE, "42", new Metadata(Metadata.Signal.COMPLETE, null));
    Assert.assertTrue(wasMetadataEmittedTo(TopologyConstants.FEEDBACK_STREAM, metadata));
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.RESULT_STREAM).count(), 1);
    Assert.assertEquals(collector.getAllEmittedTo(TopologyConstants.FEEDBACK_STREAM).count(), 1);
}

Example 24 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class QuerySpoutTest method testSignalOnlyMessagesAreSentOnTheMetadataStream.

@Test
public void testSignalOnlyMessagesAreSentOnTheMetadataStream() {
    // Add messages to be received from subscriber
    PubSubMessage messageA = new PubSubMessage("42", Metadata.Signal.KILL);
    PubSubMessage messageB = new PubSubMessage("43", Metadata.Signal.COMPLETE);
    PubSubMessage messageC = new PubSubMessage("44", null, new Metadata());
    subscriber.addMessages(messageA, messageB, messageC);
    Assert.assertEquals(subscriber.getReceived().size(), 0);
    Assert.assertEquals(emitter.getEmitted().size(), 0);
    spout.nextTuple();
    spout.nextTuple();
    spout.nextTuple();
    Assert.assertEquals(subscriber.getReceived().size(), 3);
    Assert.assertEquals(subscriber.getReceived().get(0), messageA);
    Assert.assertEquals(subscriber.getReceived().get(1), messageB);
    Assert.assertEquals(subscriber.getReceived().get(2), messageC);
    Assert.assertEquals(emitter.getEmitted().size(), 3);
    Tuple emittedFirst = TupleUtils.makeTuple(TupleClassifier.Type.METADATA_TUPLE, messageA.getId(), messageA.getMetadata());
    Tuple emittedSecond = TupleUtils.makeTuple(TupleClassifier.Type.METADATA_TUPLE, messageB.getId(), messageB.getMetadata());
    Tuple emittedThird = TupleUtils.makeTuple(TupleClassifier.Type.METADATA_TUPLE, messageC.getId(), messageC.getMetadata());
    Assert.assertTrue(emitter.wasTupleEmittedTo(emittedFirst, TopologyConstants.METADATA_STREAM));
    Assert.assertTrue(emitter.wasTupleEmittedTo(emittedSecond, TopologyConstants.METADATA_STREAM));
    Assert.assertTrue(emitter.wasTupleEmittedTo(emittedThird, TopologyConstants.METADATA_STREAM));
    Assert.assertTrue(emitter.wasNthEmitted(emittedFirst, 1));
    Assert.assertTrue(emitter.wasNthEmitted(emittedSecond, 2));
    Assert.assertTrue(emitter.wasNthEmitted(emittedThird, 3));
}

Example 25 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class QuerySpoutTest method testNextTupleMessagesAreReceivedAndTupleIsEmitted.

@Test
public void testNextTupleMessagesAreReceivedAndTupleIsEmitted() {
    // Add messages to be received from subscriber
    PubSubMessage messageA = new PubSubMessage("42", "This is a PubSubMessage", new Metadata());
    PubSubMessage messageB = new PubSubMessage("43", "This is also a PubSubMessage", new Metadata());
    subscriber.addMessages(messageA, messageB);
    Assert.assertEquals(subscriber.getReceived().size(), 0);
    Assert.assertEquals(emitter.getEmitted().size(), 0);
    // subscriber.receive() -> messageA
    spout.nextTuple();
    Assert.assertEquals(subscriber.getReceived().size(), 1);
    Assert.assertEquals(subscriber.getReceived().get(0), messageA);
    Tuple emittedFirst = TupleUtils.makeTuple(TupleClassifier.Type.QUERY_TUPLE, messageA.getId(), messageA.getContent(), messageA.getMetadata());
    Assert.assertEquals(emitter.getEmitted().size(), 1);
    Assert.assertTrue(emitter.wasNthEmitted(emittedFirst, 1));
    // subscriber.receive() -> messageB
    spout.nextTuple();
    Assert.assertEquals(subscriber.getReceived().size(), 2);
    Assert.assertEquals(subscriber.getReceived().get(0), messageA);
    Assert.assertEquals(subscriber.getReceived().get(1), messageB);
    Tuple emittedSecond = TupleUtils.makeTuple(TupleClassifier.Type.QUERY_TUPLE, messageB.getId(), messageB.getContent(), messageB.getMetadata());
    Assert.assertEquals(emitter.getEmitted().size(), 2);
    Assert.assertTrue(emitter.wasNthEmitted(emittedFirst, 1));
    Assert.assertTrue(emitter.wasNthEmitted(emittedSecond, 2));
}