Examples with Fields backtype.storm.tuple.Fields used on opensource projects

Example 36 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class RollingTopWords method wireTopology.

private void wireTopology() throws InterruptedException {
    String spoutId = "wordGenerator";
    String counterId = "counter";
    String intermediateRankerId = "intermediateRanker";
    String totalRankerId = "finalRanker";
    builder.setSpout(spoutId, new TestWordSpout(), 5);
    builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word"));
    builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId, new Fields("obj"));
    builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId);
}

Example 37 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class SequenceTestSplitRecord method declareOutputFields.

@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    declarer.declareStream(SequenceTopologyDef.TRADE_STREAM_ID, new Fields("ID", "TRADE"));
    declarer.declareStream(SequenceTopologyDef.CUSTOMER_STREAM_ID, new Fields("ID", "CUSTOMER"));
}

Example 38 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class TridentReachTest method testTridentReach.

@Test
public void testTridentReach() {
    TridentTopology tridentTopology = new TridentTopology();
    TridentState urlToTweeters = tridentTopology.newStaticState(new TridentReach.StaticSingleKeyMapState.Factory(TWEETERS));
    TridentState tweetersToFollowers = tridentTopology.newStaticState(new TridentReach.StaticSingleKeyMapState.Factory(FOLLOWERS));
    LocalDRPC localDRPC = new LocalDRPC();
    tridentTopology.newDRPCStream("reach", localDRPC).stateQuery(urlToTweeters, new Fields("args"), new MapGet(), new Fields("tweeters")).each(new Fields("tweeters"), new TridentReach.ExpandList(), new Fields("tweeter")).shuffle().stateQuery(tweetersToFollowers, new Fields("tweeter"), new MapGet(), new Fields("followers")).each(new Fields("followers"), new TridentReach.ExpandList(), new Fields("follower")).groupBy(new Fields("follower")).aggregate(new TridentReach.One(), new Fields("one")).aggregate(new Fields("one"), new Sum(), new Fields("reach"));
    Map config = new HashMap();
    config.put(Config.TOPOLOGY_NAME, "TridentReachTest");
    JStormUnitTestDRPCValidator validator = new JStormUnitTestDRPCValidator(localDRPC) {

        @Override
        public boolean validate(Map config) {
            String query = executeLocalDRPC("reach", "aaa");
            assertEquals("[[0]]", query);
            query = executeLocalDRPC("reach", "foo.com/blog/1");
            assertEquals("[[16]]", query);
            query = executeLocalDRPC("reach", "engineering.twitter.com/blog/5");
            assertEquals("[[14]]", query);
            return true;
        }
    };
    try {
        JStormUnitTestRunner.submitTopology(tridentTopology.build(), config, 120, validator);
    } finally {
        localDRPC.shutdown();
    }
}

Example 39 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class TridentWindowingInmemoryStoreTopology method buildTopology.

public static StormTopology buildTopology(WindowsStoreFactory windowStore, WindowConfig windowConfig) throws Exception {
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"), new Values("the man went to the store and bought some candy"), new Values("four score and seven years ago"), new Values("how many apples can you eat"), new Values("to be or not to be the person"));
    spout.setCycle(true);
    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout).parallelismHint(16).each(new Fields("sentence"), new Split(), new Fields("word")).window(windowConfig, windowStore, new Fields("word"), new CountAsAggregator(), new Fields("count")).peek(new Consumer() {

        @Override
        public void accept(TridentTuple input) {
            LOG.info("Received tuple: [{}]", input);
        }
    });
    return topology.build();
}

Example 40 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class InOrderDeliveryTest method testInOrderDelivery.

@Test
public void testInOrderDelivery() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("spout", new InOrderTestSpout(SPOUT_MAX_SEND_NUM), SPOUT_PARALLELISM_HINT);
    // note that here we use fieldsGrouping, so that the tuples with the same "c1" will be sent to the same bolt.
    // as a result, even though we have BOLT_PARALLELISM_HINT bolts, each bolt maintains a order of tuples from
    // spouts that it can receive.
    topologyBuilder.setBolt("bolt", new InOrderTestBolt(), BOLT_PARALLELISM_HINT).fieldsGrouping("spout", new Fields("c1"));
    Map config = new HashMap();
    config.put(Config.TOPOLOGY_NAME, "InOrderDeliveryTest");
    config.put("topology.debug.metric.names", "emit,success,fail");
    config.put("topology.debug", false);
    config.put("topology.enable.metric.debug", true);
    // the following is just for the JStormUnitTestMetricValidator to pick the metric data
    // from all the metrics.If you are not using JStormUnitTestMetricValidator, it is useless.
    // The element is the key map with the metric value as a parameter in the callback
    // function validateMetrics().
    Set<String> userDefineMetrics = new HashSet<String>();
    userDefineMetrics.add(InOrderTestMetricsDef.METRIC_SPOUT_EMIT);
    userDefineMetrics.add(InOrderTestMetricsDef.METRIC_BOLT_SUCCESS);
    userDefineMetrics.add(InOrderTestMetricsDef.METRIC_BOLT_FAIL);
    JStormUnitTestMetricValidator validator = new JStormUnitTestMetricValidator(userDefineMetrics) {

        @Override
        public boolean validateMetrics(Map<String, Double> metrics) {
            int spoutEmit = (int) metrics.get(InOrderTestMetricsDef.METRIC_SPOUT_EMIT).doubleValue();
            int boltSuccess = (int) metrics.get(InOrderTestMetricsDef.METRIC_BOLT_SUCCESS).doubleValue();
            int boltFail = (int) metrics.get(InOrderTestMetricsDef.METRIC_BOLT_FAIL).doubleValue();
            LOG.info("validateMetrics: " + "spout emit = " + spoutEmit + " bolt success = " + boltSuccess);
            assertEquals(SPOUT_MAX_SEND_NUM * SPOUT_PARALLELISM_HINT, spoutEmit);
            // all tuples should be in order
            assertEquals(spoutEmit, boltSuccess);
            assertEquals(0, boltFail);
            return true;
        }
    };
    JStormUnitTestRunner.submitTopology(topologyBuilder.createTopology(), config, 120, validator);
}