Examples with Config backtype.storm.Config used on opensource projects

Example 26 with Config

use of backtype.storm.Config in project jstorm by alibaba.

the class TestTridentTopology method main.

public static void main(String[] args) throws Exception {
    if (args.length == 0) {
        System.err.println("Please input configuration file");
        System.exit(-1);
    }
    Map conf = LoadConfig.LoadConf(args[0]);
    if (conf == null) {
        LOG.error("Failed to load config");
    } else {
        Config config = new Config();
        config.putAll(conf);
        config.setMaxSpoutPending(10);
        config.put(LoadConfig.TOPOLOGY_TYPE, "Trident");
        StormSubmitter.submitTopology("WordCount", config, buildTopology());
    }
}

Example 27 with Config

use of backtype.storm.Config in project storm-lib by xumingming.

the class TestingApiDemo method testBasicTopology.

public void testBasicTopology() {
    MkClusterParam mkClusterParam = new MkClusterParam();
    mkClusterParam.setSupervisors(4);
    Config daemonConf = new Config();
    daemonConf.put(Config.STORM_LOCAL_MODE_ZMQ, false);
    mkClusterParam.setDaemonConf(daemonConf);
    /**
		 * This is a combination of <code>Testing.withLocalCluster</code> and <code>Testing.withSimulatedTime</code>.
		 */
    Testing.withSimulatedTimeLocalCluster(mkClusterParam, new TestJob() {

        @Override
        public void run(ILocalCluster cluster) {
            // build the test topology
            TopologyBuilder builder = new TopologyBuilder();
            builder.setSpout("1", new TestWordSpout(true), 3);
            builder.setBolt("2", new TestWordCounter(), 4).fieldsGrouping("1", new Fields("word"));
            builder.setBolt("3", new TestGlobalCount()).globalGrouping("1");
            builder.setBolt("4", new TestAggregatesCounter()).globalGrouping("2");
            StormTopology topology = builder.createTopology();
            // complete the topology
            // prepare the mock data
            MockedSources mockedSources = new MockedSources();
            mockedSources.addMockData("1", new Values("nathan"), new Values("bob"), new Values("joey"), new Values("nathan"));
            // prepare the config
            Config conf = new Config();
            conf.setNumWorkers(2);
            CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam();
            completeTopologyParam.setMockedSources(mockedSources);
            completeTopologyParam.setStormConf(conf);
            /**
				 * TODO
				 */
            Map result = Testing.completeTopology(cluster, topology, completeTopologyParam);
            // check whether the result is right
            assertTrue(Testing.multiseteq(new Values(new Values("nathan"), new Values("bob"), new Values("joey"), new Values("nathan")), Testing.readTuples(result, "1")));
            assertTrue(Testing.multiseteq(new Values(new Values("nathan", 1), new Values("nathan", 2), new Values("bob", 1), new Values("joey", 1)), Testing.readTuples(result, "2")));
            assertTrue(Testing.multiseteq(new Values(new Values(1), new Values(2), new Values(3), new Values(4)), Testing.readTuples(result, "3")));
            assertTrue(Testing.multiseteq(new Values(new Values(1), new Values(2), new Values(3), new Values(4)), Testing.readTuples(result, "4")));
        }
    });
}

Example 28 with Config

use of backtype.storm.Config in project storm-hbase by ypf412.

the class OutputTopology method main.

/**
	 * HBase Data Output Topology
	 * @param args
	 * @throws Exception
	 */
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    PropConfig pc = new PropConfig("storm.properties");
    int topoWorkers = Integer.valueOf(pc.getProperty("storm.topolopy.workers"));
    int spoutTasks = Integer.valueOf(pc.getProperty("storm.spout.tasks"));
    builder.setSpout("hbaseSpout", new HBaseSpout(), spoutTasks);
    int boltTasks = spoutTasks;
    builder.setBolt("outputBolt", new OutputBolt(), boltTasks).fieldsGrouping("hbaseSpout", new Fields("sharding"));
    Config conf = new Config();
    conf.put(Constants.STORM_PROP_CONF_FILE, "storm.properties");
    conf.put(Constants.HBASE_PROP_CONF_FILE, "hbase.properties");
    if (args != null && args.length > 0) {
        // run on storm cluster
        conf.setNumAckers(1);
        conf.setNumWorkers(topoWorkers);
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run on local cluster
        conf.setMaxTaskParallelism(3);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("test", conf, builder.createTopology());
        Utils.sleep(10000);
        cluster.killTopology("test");
        cluster.shutdown();
    }
}

Example 29 with Config

use of backtype.storm.Config in project ud381 by udacity.

the class TopNTweetTopology method main.

public static void main(String[] args) throws Exception {
    //Variable TOP_N number of words
    int TOP_N = 5;
    // create the topology
    TopologyBuilder builder = new TopologyBuilder();
    /*
     * In order to create the spout, you need to get twitter credentials
     * If you need to use Twitter firehose/Tweet stream for your idea,
     * create a set of credentials by following the instructions at
     *
     * https://dev.twitter.com/discussions/631
     *
     */
    // now create the tweet spout with the credentials
    // credential
    TweetSpout tweetSpout = new TweetSpout("", "", "", "");
    // attach the tweet spout to the topology - parallelism of 1
    builder.setSpout("tweet-spout", tweetSpout, 1);
    // attach the parse tweet bolt using shuffle grouping
    builder.setBolt("parse-tweet-bolt", new ParseTweetBolt(), 10).shuffleGrouping("tweet-spout");
    builder.setBolt("infoBolt", new InfoBolt(), 10).fieldsGrouping("parse-tweet-bolt", new Fields("county_id"));
    builder.setBolt("top-words", new TopWords(), 10).fieldsGrouping("infoBolt", new Fields("county_id"));
    builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("top-words");
    // attach rolling count bolt using fields grouping - parallelism of 5
    //builder.setBolt("rolling-count-bolt", new RollingCountBolt(1000, 10), 1).fieldsGrouping("parse-tweet-bolt", new Fields("tweet-word"));
    //from incubator-storm/.../storm/starter/RollingTopWords.java
    //builder.setBolt("intermediate-ranker", new IntermediateRankingsBolt(TOP_N, 10), 2).fieldsGrouping("rolling-count-bolt", new Fields("obj"));
    //builder.setBolt("total-ranker", new TotalRankingsBolt(TOP_N, 2)).globalGrouping("intermediate-ranker");
    /*
     * total-ranker bolt output is broadcast (allGrouping) to all the top-tweets bolt instances so
     * that every one of them have access to the top hashtags
     * tweet-spout tweet stream will be distributed randomly to the top-tweets bolt instances
     */
    //builder.setBolt("top-tweets", new TweetsWithTopHashtagsBolt(), 4)
    //    .allGrouping("total-ranker")
    //    .shuffleGrouping("tweet-spout");
    // attach the report bolt using global grouping - parallelism of 1
    //builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("top-tweets");
    // create the default config object
    Config conf = new Config();
    // set the config in debugging mode
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        // run it in a live cluster
        // set the number of workers for running all spout and bolt tasks
        conf.setNumWorkers(3);
        // create the topology and submit with config
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run it in a simulated local cluster
        // set the number of threads to run - similar to setting number of workers in live cluster
        conf.setMaxTaskParallelism(4);
        // create the local cluster instance
        LocalCluster cluster = new LocalCluster();
        // submit the topology to the local cluster
        cluster.submitTopology("tweet-word-count", conf, builder.createTopology());
        // let the topology run for 300 seconds. note topologies never terminate!
        Utils.sleep(300000000);
        // now kill the topology
        cluster.killTopology("tweet-word-count");
        // we are done, so shutdown the local cluster
        cluster.shutdown();
    }
}

Example 30 with Config

use of backtype.storm.Config in project ud381 by udacity.

the class SentenceCountTopology method main.

public static void main(String[] args) throws Exception {
    // create the topology
    TopologyBuilder builder = new TopologyBuilder();
    // attach the word spout to the topology - parallelism of 5
    //builder.setSpout("word-spout", new WordSpout(), 5);
    // attach sentence spout to the topology - parallelism of 1
    builder.setSpout("sentence-spout", new RandomSentenceSpout(), 1);
    // attach the count bolt using fields grouping - parallelism of 15
    builder.setBolt("count-bolt", new CountBolt(), 15).fieldsGrouping("sentence-spout", new Fields("sentence"));
    // attach the report bolt using global grouping - parallelism of 1
    //***************************************************
    // BEGIN YOUR CODE - part 2
    builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("count-bolt");
    // END YOUR CODE
    //***************************************************
    // create the default config object
    Config conf = new Config();
    // set the config in debugging mode
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        // run it in a live cluster
        // set the number of workers for running all spout and bolt tasks
        conf.setNumWorkers(3);
        // create the topology and submit with config
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run it in a simulated local cluster
        // set the number of threads to run - similar to setting number of workers in live cluster
        conf.setMaxTaskParallelism(3);
        // create the local cluster instance
        LocalCluster cluster = new LocalCluster();
        // submit the topology to the local cluster
        // name topology
        cluster.submitTopology("sentence-count", conf, builder.createTopology());
        //**********************************************************************
        // let the topology run for 30 seconds. note topologies never terminate!
        Thread.sleep(30000);
        //**********************************************************************
        // we are done, so shutdown the local cluster
        cluster.shutdown();
    }
}