Examples with Fields backtype.storm.tuple.Fields used on opensource projects

Example 76 with Fields

use of backtype.storm.tuple.Fields in project storm-hbase by jrkinley.

the class HBaseTridentAggregateTopology method main.

/**
 * @param args
 * @throws InterruptedException
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public static void main(String[] args) throws InterruptedException {
    List<Object> v0 = HBaseCountersBatchTopology.values.get(0).get(0);
    List<Object> v1 = HBaseCountersBatchTopology.values.get(0).get(1);
    List<Object> v2 = HBaseCountersBatchTopology.values.get(0).get(2);
    List<Object> v3 = HBaseCountersBatchTopology.values.get(0).get(3);
    List<Object> v4 = HBaseCountersBatchTopology.values.get(0).get(4);
    List<Object> v5 = HBaseCountersBatchTopology.values.get(1).get(0);
    List<Object> v6 = HBaseCountersBatchTopology.values.get(1).get(1);
    List<Object> v7 = HBaseCountersBatchTopology.values.get(1).get(2);
    List<Object> v8 = HBaseCountersBatchTopology.values.get(2).get(0);
    List<Object> v9 = HBaseCountersBatchTopology.values.get(2).get(1);
    List<Object> v10 = HBaseCountersBatchTopology.values.get(2).get(2);
    HBaseCountersBatchTopology.values.values();
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("shortid", "url", "user", "date"), 3, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10);
    spout.setCycle(false);
    TridentConfig config = new TridentConfig("shorturl", "shortid");
    config.setBatch(false);
    StateFactory state = HBaseAggregateState.transactional(config);
    TridentTopology topology = new TridentTopology();
    topology.newStream("spout", spout).each(new Fields("shortid", "date"), new DatePartitionFunction(), new Fields("cf", "cq")).project(new Fields("shortid", "cf", "cq")).groupBy(new Fields("shortid", "cf", "cq")).persistentAggregate(state, new Count(), new Fields("count"));
    Config conf = new Config();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("hbase-trident-aggregate", conf, topology.build());
    Utils.sleep(5000);
    cluster.shutdown();
}

Example 77 with Fields

use of backtype.storm.tuple.Fields in project ud381 by udacity.

the class WordCountTopology method main.

public static void main(String[] args) throws Exception {
    // create the topology
    TopologyBuilder builder = new TopologyBuilder();
    // attach the word spout to the topology - parallelism of 5
    builder.setSpout("word-spout", new WordSpout(), 5);
    // attach the count bolt using fields grouping - parallelism of 15
    builder.setBolt("count-bolt", new CountBolt(), 15).fieldsGrouping("word-spout", new Fields("word"));
    // attach the report bolt using global grouping - parallelism of 1
    // ***************************************************
    // BEGIN YOUR CODE - part 2
    builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("count-bolt");
    // END YOUR CODE
    // ***************************************************
    // create the default config object
    Config conf = new Config();
    // set the config in debugging mode
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        // run it in a live cluster
        // set the number of workers for running all spout and bolt tasks
        conf.setNumWorkers(3);
        // create the topology and submit with config
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run it in a simulated local cluster
        // set the number of threads to run - similar to setting number of workers in live cluster
        conf.setMaxTaskParallelism(3);
        // create the local cluster instance
        LocalCluster cluster = new LocalCluster();
        // submit the topology to the local cluster
        cluster.submitTopology("word-count", conf, builder.createTopology());
        // **********************************************************************
        // let the topology run for 30 seconds. note topologies never terminate!
        Thread.sleep(30000);
        // **********************************************************************
        // we are done, so shutdown the local cluster
        cluster.shutdown();
    }
}

Example 78 with Fields

use of backtype.storm.tuple.Fields in project ud381 by udacity.

the class SentenceWordCountTopology method main.

public static void main(String[] args) throws Exception {
    // create the topology
    TopologyBuilder builder = new TopologyBuilder();
    // attach the word spout to the topology - parallelism of 5
    // builder.setSpout("word-spout", new WordSpout(), 5);
    // attach sentence spout to the topology - parallelism of 1
    builder.setSpout("sentence-spout", new RandomSentenceSpout(), 1);
    // attach split sentence split bolt to topology - parallelism of 15
    // builder.setBolt("split-sentence-bolt", new SplitSentenceBolt(), 15).fieldsGrouping("sentence-spout", new Fields("sentence"));
    builder.setBolt("split-sentence-bolt", new SplitSentenceBolt(), 15).shuffleGrouping("sentence-spout");
    // attach the count bolt using fields grouping - parallelism of 15
    builder.setBolt("count-bolt", new CountBolt(), 15).fieldsGrouping("split-sentence-bolt", new Fields("sentence-word"));
    // attach the report bolt using global grouping - parallelism of 1
    // ***************************************************
    // BEGIN YOUR CODE - part 2
    builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("count-bolt");
    // END YOUR CODE
    // ***************************************************
    // create the default config object
    Config conf = new Config();
    // set the config in debugging mode
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        // run it in a live cluster
        // set the number of workers for running all spout and bolt tasks
        conf.setNumWorkers(3);
        // create the topology and submit with config
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run it in a simulated local cluster
        // set the number of threads to run - similar to setting number of workers in live cluster
        conf.setMaxTaskParallelism(3);
        // create the local cluster instance
        LocalCluster cluster = new LocalCluster();
        // submit the topology to the local cluster
        // name topology
        cluster.submitTopology("sentence-word-count", conf, builder.createTopology());
        // **********************************************************************
        // let the topology run for 30 seconds. note topologies never terminate!
        Thread.sleep(30000);
        // **********************************************************************
        // we are done, so shutdown the local cluster
        cluster.shutdown();
    }
}

Example 79 with Fields

use of backtype.storm.tuple.Fields in project ud381 by udacity.

the class TweetTopology method main.

public static void main(String[] args) throws Exception {
    // create the topology
    TopologyBuilder builder = new TopologyBuilder();
    /*
     * In order to create the spout, you need to get twitter credentials
     * If you need to use Twitter firehose/Tweet stream for your idea,
     * create a set of credentials by following the instructions at
     *
     * https://dev.twitter.com/discussions/631
     *
     */
    // now create the tweet spout with the credentials
    TweetSpout tweetSpout = new TweetSpout("[Your customer key]", "[Your secret key]", "[Your access token]", "[Your access secret]");
    // attach the tweet spout to the topology - parallelism of 1
    builder.setSpout("tweet-spout", tweetSpout, 1);
    // attach the parse tweet bolt using shuffle grouping
    builder.setBolt("parse-tweet-bolt", new ParseTweetBolt(), 10).shuffleGrouping("tweet-spout");
    // attach the count bolt using fields grouping - parallelism of 15
    // builder.setBolt("count-bolt", new CountBolt(), 15).fieldsGrouping("parse-tweet-bolt", new Fields("tweet-word"));
    // attach rolling count bolt using fields grouping - parallelism of 5
    builder.setBolt("rolling-count-bolt", new RollingCountBolt(30, 10), 1).fieldsGrouping("parse-tweet-bolt", new Fields("tweet-word"));
    // attach the report bolt using global grouping - parallelism of 1
    builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("rolling-count-bolt");
    // create the default config object
    Config conf = new Config();
    // set the config in debugging mode
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        // run it in a live cluster
        // set the number of workers for running all spout and bolt tasks
        conf.setNumWorkers(3);
        // create the topology and submit with config
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run it in a simulated local cluster
        // set the number of threads to run - similar to setting number of workers in live cluster
        conf.setMaxTaskParallelism(3);
        // create the local cluster instance
        LocalCluster cluster = new LocalCluster();
        // submit the topology to the local cluster
        cluster.submitTopology("tweet-word-count", conf, builder.createTopology());
        // let the topology run for 300 seconds. note topologies never terminate!
        Utils.sleep(300000);
        // now kill the topology
        cluster.killTopology("tweet-word-count");
        // we are done, so shutdown the local cluster
        cluster.shutdown();
    }
}

Example 80 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class SkewedRollingTopWords method wireTopology.

private void wireTopology() throws InterruptedException {
    String spoutId = "wordGenerator";
    String counterId = "counter";
    String aggId = "aggregator";
    String intermediateRankerId = "intermediateRanker";
    String totalRankerId = "finalRanker";
    builder.setSpout(spoutId, new TestWordSpout(), 5);
    builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId, new Fields("word"));
    builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId, new Fields("obj"));
    builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(aggId, new Fields("obj"));
    builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId);
}