Examples with Metadata com.yahoo.bullet.pubsub.Metadata used on opensource projects

Example 36 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class QueryBoltTest method testRegularMetaTupleIgnored.

@Test
public void testRegularMetaTupleIgnored() {
    CustomCollector collector = new CustomCollector();
    TestQueryBolt bolt = new TestQueryBolt(new BulletStormConfig());
    ComponentUtils.prepare(bolt, collector);
    Map<String, Querier> queries = bolt.getQueries();
    queries.put("foo", null);
    Tuple meta = makeIDTuple(Type.METADATA_TUPLE, "foo", new Metadata(Signal.ACKNOWLEDGE, null));
    bolt.execute(meta);
    Assert.assertTrue(queries.containsKey("foo"));
}

Example 37 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class ResultBoltTest method testExecuteMessagesAreSent.

@Test
public void testExecuteMessagesAreSent() {
    List<PubSubMessage> expected = asList(new PubSubMessage("42", "This is a PubSubMessage", new Metadata()), new PubSubMessage("43", "This is also a PubSubMessage", new Metadata()), new PubSubMessage("44", "This is still a PubSubMessage", new Metadata()));
    List<Tuple> tuples = new ArrayList<>();
    expected.forEach(m -> tuples.add(makeTuple(m.getId(), m.getContent(), m.getMetadata())));
    for (int i = 0; i < tuples.size(); i++) {
        bolt.execute(tuples.get(i));
        Assert.assertEquals(publisher.getSent().get(i).getId(), expected.get(i).getId());
        Assert.assertEquals(publisher.getSent().get(i).getContent(), expected.get(i).getContent());
        Assert.assertEquals(publisher.getSent().get(i).getMetadata(), expected.get(i).getMetadata());
        Assert.assertTrue(collector.wasNthAcked(tuples.get(i), i + 1));
        Assert.assertEquals(collector.getAckedCount(), i + 1);
    }
}

Example 38 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class DRPCResultPublisher method send.

@Override
public void send(PubSubMessage message) throws PubSubException {
    Metadata metadata = message.getMetadata();
    // Remove the content
    String content = metadata.getContent().toString();
    log.debug("Removing metadata {} for result {}@{}: {}", content, message.getId(), message.getSequence(), message.getContent());
    metadata.setContent(null);
    String serializedMessage = message.asJSON();
    Tuple tuple = new DRPCTuple(new Values(serializedMessage, content));
    // This sends the message through DRPC and not to the collector but it acks or fails accordingly.
    bolt.execute(tuple);
    if (!collector.isAcked()) {
        throw new PubSubException("Message not acked. Unable to send message through DRPC:\n " + serializedMessage);
    }
    // Otherwise, we're good to proceed
    collector.reset();
}

Example 39 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class FilterBoltTest method testFilteringLatency.

@Test
public void testFilteringLatency() {
    config = new BulletStormConfig();
    // Don't use the overridden aggregation default size but turn on built in metrics
    config.set(BulletStormConfig.TOPOLOGY_METRICS_BUILT_IN_ENABLE, true);
    collector = new CustomCollector();
    CustomTopologyContext context = new CustomTopologyContext();
    bolt = new FilterBolt(TopologyConstants.RECORD_COMPONENT, config);
    ComponentUtils.prepare(new HashMap<>(), bolt, context, collector);
    Tuple query = makeIDTuple(TupleClassifier.Type.QUERY_TUPLE, "42", makeFieldFilterQuery("bar"), METADATA);
    bolt.execute(query);
    BulletRecord record = RecordBox.get().add("field", "foo").getRecord();
    long start = System.currentTimeMillis();
    IntStream.range(0, 10).mapToObj(i -> makeRecordTuple(record, System.currentTimeMillis())).forEach(bolt::execute);
    long end = System.currentTimeMillis();
    double actualLatecy = context.getDoubleMetric(TopologyConstants.LATENCY_METRIC);
    Assert.assertTrue(actualLatecy <= end - start);
}

Example 40 with Metadata

use of com.yahoo.bullet.pubsub.Metadata in project bullet-storm by yahoo.

the class DRPCQuerySubscriberTest method testReadingFromSpout.

@Test
public void testReadingFromSpout() throws Exception {
    // The MockDRPCSpout makes messages adds 2 zero based sequences with id foo and given string content.
    // It adds to metadata a string JSON with id: "fake" + foo, host: "testHost" and port: a total count of messages.
    injectedMockSpout.addMessageParts("foo", "{}", "{'duration': 2000}");
    injectedMockSpout.addMessageParts("bar", "{}");
    PubSubMessage actual = subscriber.receive();
    Assert.assertEquals(actual.getId(), "foo");
    Assert.assertEquals(actual.getSequence(), 0);
    Assert.assertEquals(actual.getContent(), "{}");
    Assert.assertFalse(actual.hasSignal());
    Assert.assertTrue(actual.hasMetadata());
    Metadata metadata = actual.getMetadata();
    Assert.assertEquals(metadata.getContent(), makeReturnInfo("fakefoo", "testHost", 0));
    actual = subscriber.receive();
    Assert.assertEquals(actual.getId(), "foo");
    Assert.assertEquals(actual.getSequence(), 1);
    Assert.assertEquals(actual.getContent(), "{'duration': 2000}");
    Assert.assertFalse(actual.hasSignal());
    Assert.assertTrue(actual.hasMetadata());
    metadata = actual.getMetadata();
    Assert.assertEquals(metadata.getContent(), makeReturnInfo("fakefoo", "testHost", 1));
    actual = subscriber.receive();
    Assert.assertEquals(actual.getId(), "bar");
    Assert.assertEquals(actual.getSequence(), 0);
    Assert.assertEquals(actual.getContent(), "{}");
    Assert.assertFalse(actual.hasSignal());
    Assert.assertTrue(actual.hasMetadata());
    metadata = actual.getMetadata();
    Assert.assertEquals(metadata.getContent(), makeReturnInfo("fakebar", "testHost", 2));
}