Examples with Config backtype.storm.Config used on opensource projects

Example 71 with Config

use of backtype.storm.Config in project ud381 by udacity.

the class WordCountTopology method main.

public static void main(String[] args) throws Exception {
    // create the topology
    TopologyBuilder builder = new TopologyBuilder();
    // attach the word spout to the topology - parallelism of 5
    builder.setSpout("word-spout", new WordSpout(), 5);
    // attach the count bolt using fields grouping - parallelism of 15
    builder.setBolt("count-bolt", new CountBolt(), 15).fieldsGrouping("word-spout", new Fields("word"));
    // attach the report bolt using global grouping - parallelism of 1
    //***************************************************
    // BEGIN YOUR CODE - part 2
    builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("count-bolt");
    // END YOUR CODE
    //***************************************************
    // create the default config object
    Config conf = new Config();
    // set the config in debugging mode
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        // run it in a live cluster
        // set the number of workers for running all spout and bolt tasks
        conf.setNumWorkers(3);
        // create the topology and submit with config
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run it in a simulated local cluster
        // set the number of threads to run - similar to setting number of workers in live cluster
        conf.setMaxTaskParallelism(3);
        // create the local cluster instance
        LocalCluster cluster = new LocalCluster();
        // submit the topology to the local cluster
        cluster.submitTopology("word-count", conf, builder.createTopology());
        //**********************************************************************
        // let the topology run for 30 seconds. note topologies never terminate!
        Thread.sleep(30000);
        //**********************************************************************
        // we are done, so shutdown the local cluster
        cluster.shutdown();
    }
}

Example 72 with Config

use of backtype.storm.Config in project ud381 by udacity.

the class ReporterExclamationTopology method main.

public static void main(String[] args) throws Exception {
    // create the topology
    TopologyBuilder builder = new TopologyBuilder();
    //********* BEGIN stage2 exercise part 2-of-2 ***********
    // attach the word spout to the topology - parallelism of 10
    //builder.setSpout("word", new TestWordSpout(), 10);
    builder.setSpout("rand-sentence", new RandomSentenceSpout(), 10);
    // attach the exclamation bolt to the topology - parallelism of 3
    //builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word");
    builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("rand-sentence");
    // attach another exclamation bolt to the topology - parallelism of 2
    //builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");
    builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("rand-sentence");
    //********* END stage2 exercise part 2-of-2 ***********
    // create the default config object
    Config conf = new Config();
    // set the config in debugging mode
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        // run it in a live cluster
        // set the number of workers for running all spout and bolt tasks
        conf.setNumWorkers(3);
        // create the topology and submit with config
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run it in a simulated local cluster
        // create the local cluster instance
        LocalCluster cluster = new LocalCluster();
        // submit the topology to the local cluster
        cluster.submitTopology("exclamation", conf, builder.createTopology());
        // let the topology run for 30 seconds. note topologies never terminate!
        Thread.sleep(30000);
        // kill the topology
        cluster.killTopology("exclamation");
        // we are done, so shutdown the local cluster
        cluster.shutdown();
    }
}

Example 73 with Config

use of backtype.storm.Config in project ud381 by udacity.

the class ExclamationTopology method main.

public static void main(String[] args) throws Exception {
    // create the topology
    TopologyBuilder builder = new TopologyBuilder();
    // attach the word spout to the topology - parallelism of 10
    //builder.setSpout("word", new TestWordSpout(), 10);
    //******** Add MyLikesSpout and MyNamesSpout
    builder.setSpout("my-likes", new MyLikesSpout(), 10);
    builder.setSpout("my-names", new MyNamesSpout(), 10);
    // attach the exclamation bolt to the topology - parallelism of 3
    builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("my-likes").shuffleGrouping("my-names");
    // attach another exclamation bolt to the topology - parallelism of 2
    builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");
    //******* Attach ReportBolt
    builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("exclaim2");
    // create the default config object
    Config conf = new Config();
    // set the config in debugging mode
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        // run it in a live cluster
        // set the number of workers for running all spout and bolt tasks
        conf.setNumWorkers(3);
        // create the topology and submit with config
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run it in a simulated local cluster
        // create the local cluster instance
        LocalCluster cluster = new LocalCluster();
        // submit the topology to the local cluster
        cluster.submitTopology("exclamation", conf, builder.createTopology());
        // let the topology run for 30 seconds. note topologies never terminate!
        Thread.sleep(30000);
        // kill the topology
        cluster.killTopology("exclamation");
        // we are done, so shutdown the local cluster
        cluster.shutdown();
    }
}

Example 74 with Config

use of backtype.storm.Config in project ud381 by udacity.

the class TweetSpout method getComponentConfiguration.

/**
   * Component specific configuration
   */
@Override
public Map<String, Object> getComponentConfiguration() {
    // create the component config
    Config ret = new Config();
    // set the parallelism for this spout to be 1
    ret.setMaxTaskParallelism(1);
    return ret;
}

Example 75 with Config

use of backtype.storm.Config in project ud381 by udacity.

the class TweetTopology method main.

public static void main(String[] args) throws Exception {
    // create the topology
    TopologyBuilder builder = new TopologyBuilder();
    /*
     * In order to create the spout, you need to get twitter credentials
     * If you need to use Twitter firehose/Tweet stream for your idea,
     * create a set of credentials by following the instructions at
     *
     * https://dev.twitter.com/discussions/631
     *
     */
    // now create the tweet spout with the credentials
    TweetSpout tweetSpout = new TweetSpout("[Your customer key]", "[Your secret key]", "[Your access token]", "[Your access secret]");
    // attach the tweet spout to the topology - parallelism of 1
    builder.setSpout("tweet-spout", tweetSpout, 1);
    // attach the parse tweet bolt using shuffle grouping
    builder.setBolt("parse-tweet-bolt", new ParseTweetBolt(), 10).shuffleGrouping("tweet-spout");
    // attach the count bolt using fields grouping - parallelism of 15
    //builder.setBolt("count-bolt", new CountBolt(), 15).fieldsGrouping("parse-tweet-bolt", new Fields("tweet-word"));
    // attach rolling count bolt using fields grouping - parallelism of 5
    builder.setBolt("rolling-count-bolt", new RollingCountBolt(30, 10), 1).fieldsGrouping("parse-tweet-bolt", new Fields("tweet-word"));
    // attach the report bolt using global grouping - parallelism of 1
    builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("rolling-count-bolt");
    // create the default config object
    Config conf = new Config();
    // set the config in debugging mode
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        // run it in a live cluster
        // set the number of workers for running all spout and bolt tasks
        conf.setNumWorkers(3);
        // create the topology and submit with config
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run it in a simulated local cluster
        // set the number of threads to run - similar to setting number of workers in live cluster
        conf.setMaxTaskParallelism(3);
        // create the local cluster instance
        LocalCluster cluster = new LocalCluster();
        // submit the topology to the local cluster
        cluster.submitTopology("tweet-word-count", conf, builder.createTopology());
        // let the topology run for 300 seconds. note topologies never terminate!
        Utils.sleep(300000);
        // now kill the topology
        cluster.killTopology("tweet-word-count");
        // we are done, so shutdown the local cluster
        cluster.shutdown();
    }
}