Examples with LocalCluster backtype.storm.LocalCluster used on opensource projects

Example 21 with LocalCluster

use of backtype.storm.LocalCluster in project ud381 by udacity.

the class ExclamationTopology method main.

public static void main(String[] args) throws Exception {
    // create the topology
    TopologyBuilder builder = new TopologyBuilder();
    // attach the word spout to the topology - parallelism of 10
    // builder.setSpout("word", new TestWordSpout(), 10);
    // ******** Add MyLikesSpout and MyNamesSpout
    builder.setSpout("my-likes", new MyLikesSpout(), 10);
    builder.setSpout("my-names", new MyNamesSpout(), 10);
    // attach the exclamation bolt to the topology - parallelism of 3
    builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("my-likes").shuffleGrouping("my-names");
    // attach another exclamation bolt to the topology - parallelism of 2
    builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");
    // ******* Attach ReportBolt
    builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("exclaim2");
    // create the default config object
    Config conf = new Config();
    // set the config in debugging mode
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        // run it in a live cluster
        // set the number of workers for running all spout and bolt tasks
        conf.setNumWorkers(3);
        // create the topology and submit with config
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run it in a simulated local cluster
        // create the local cluster instance
        LocalCluster cluster = new LocalCluster();
        // submit the topology to the local cluster
        cluster.submitTopology("exclamation", conf, builder.createTopology());
        // let the topology run for 30 seconds. note topologies never terminate!
        Thread.sleep(30000);
        // kill the topology
        cluster.killTopology("exclamation");
        // we are done, so shutdown the local cluster
        cluster.shutdown();
    }
}

Example 22 with LocalCluster

use of backtype.storm.LocalCluster in project ud381 by udacity.

the class TweetTopology method main.

public static void main(String[] args) throws Exception {
    // create the topology
    TopologyBuilder builder = new TopologyBuilder();
    /*
     * In order to create the spout, you need to get twitter credentials
     * If you need to use Twitter firehose/Tweet stream for your idea,
     * create a set of credentials by following the instructions at
     *
     * https://dev.twitter.com/discussions/631
     *
     */
    // now create the tweet spout with the credentials
    TweetSpout tweetSpout = new TweetSpout("[Your customer key]", "[Your secret key]", "[Your access token]", "[Your access secret]");
    // attach the tweet spout to the topology - parallelism of 1
    builder.setSpout("tweet-spout", tweetSpout, 1);
    // attach the parse tweet bolt using shuffle grouping
    builder.setBolt("parse-tweet-bolt", new ParseTweetBolt(), 10).shuffleGrouping("tweet-spout");
    // attach the count bolt using fields grouping - parallelism of 15
    // builder.setBolt("count-bolt", new CountBolt(), 15).fieldsGrouping("parse-tweet-bolt", new Fields("tweet-word"));
    // attach rolling count bolt using fields grouping - parallelism of 5
    builder.setBolt("rolling-count-bolt", new RollingCountBolt(30, 10), 1).fieldsGrouping("parse-tweet-bolt", new Fields("tweet-word"));
    // attach the report bolt using global grouping - parallelism of 1
    builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("rolling-count-bolt");
    // create the default config object
    Config conf = new Config();
    // set the config in debugging mode
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        // run it in a live cluster
        // set the number of workers for running all spout and bolt tasks
        conf.setNumWorkers(3);
        // create the topology and submit with config
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run it in a simulated local cluster
        // set the number of threads to run - similar to setting number of workers in live cluster
        conf.setMaxTaskParallelism(3);
        // create the local cluster instance
        LocalCluster cluster = new LocalCluster();
        // submit the topology to the local cluster
        cluster.submitTopology("tweet-word-count", conf, builder.createTopology());
        // let the topology run for 300 seconds. note topologies never terminate!
        Utils.sleep(300000);
        // now kill the topology
        cluster.killTopology("tweet-word-count");
        // we are done, so shutdown the local cluster
        cluster.shutdown();
    }
}

Example 23 with LocalCluster

use of backtype.storm.LocalCluster in project jstorm by alibaba.

the class BasicDRPCTopology method testDrpc.

public static void testDrpc() {
    LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("exclamation");
    builder.addBolt(new ExclaimBolt(), 3);
    Config conf = new Config();
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("drpc-demo", conf, builder.createLocalTopology(drpc));
    try {
        Thread.sleep(30000);
    } catch (InterruptedException e) {
    // TODO Auto-generated catch block
    }
    try {
        for (String word : new String[] { "hello", "goodbye" }) {
            System.out.println("Result for \"" + word + "\": " + drpc.execute("exclamation", word));
        }
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("Failed to run DRPC Test");
    }
    drpc.shutdown();
    cluster.shutdown();
}

Example 24 with LocalCluster

use of backtype.storm.LocalCluster in project jstorm by alibaba.

the class ManualDRPC method testDrpc.

public static void testDrpc() {
    TopologyBuilder builder = new TopologyBuilder();
    LocalDRPC drpc = new LocalDRPC();
    DRPCSpout spout = new DRPCSpout("exclamation", drpc);
    builder.setSpout("drpc", spout);
    builder.setBolt("exclaim", new ExclamationBolt(), 3).shuffleGrouping("drpc");
    builder.setBolt("return", new ReturnResults(), 3).shuffleGrouping("exclaim");
    LocalCluster cluster = new LocalCluster();
    Config conf = new Config();
    cluster.submitTopology("exclaim", conf, builder.createTopology());
    JStormUtils.sleepMs(30 * 1000);
    try {
        System.out.println(drpc.execute("exclamation", "aaa"));
        System.out.println(drpc.execute("exclamation", "bbb"));
    } catch (Exception e) {
        Assert.fail("Failed to test drpc");
    }
    drpc.shutdown();
    cluster.shutdown();
}

Example 25 with LocalCluster

use of backtype.storm.LocalCluster in project jstorm by alibaba.

the class SequenceTopology method SetLocalTopology.

public static void SetLocalTopology() throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    conf.put(TOPOLOGY_BOLT_PARALLELISM_HINT, 1);
    SetBuilder(builder, conf);
    LOG.debug("test");
    LOG.info("Submit log");
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("SplitMerge", conf, builder.createTopology());
    Thread.sleep(60000);
    cluster.killTopology("SplitMerge");
    cluster.shutdown();
}