Examples with TopologyBuilder backtype.storm.topology.TopologyBuilder used on opensource projects

Example 51 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project ud381 by udacity.

the class SentenceWordCountTopology method main.

public static void main(String[] args) throws Exception {
    // create the topology
    TopologyBuilder builder = new TopologyBuilder();
    // attach the word spout to the topology - parallelism of 5
    //builder.setSpout("word-spout", new WordSpout(), 5);
    // attach sentence spout to the topology - parallelism of 1
    builder.setSpout("sentence-spout", new RandomSentenceSpout(), 1);
    // attach split sentence split bolt to topology - parallelism of 15
    //builder.setBolt("split-sentence-bolt", new SplitSentenceBolt(), 15).fieldsGrouping("sentence-spout", new Fields("sentence"));
    builder.setBolt("split-sentence-bolt", new SplitSentenceBolt(), 15).shuffleGrouping("sentence-spout");
    // attach the count bolt using fields grouping - parallelism of 15
    builder.setBolt("count-bolt", new CountBolt(), 15).fieldsGrouping("split-sentence-bolt", new Fields("sentence-word"));
    // attach the report bolt using global grouping - parallelism of 1
    //***************************************************
    // BEGIN YOUR CODE - part 2
    builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("count-bolt");
    // END YOUR CODE
    //***************************************************
    // create the default config object
    Config conf = new Config();
    // set the config in debugging mode
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        // run it in a live cluster
        // set the number of workers for running all spout and bolt tasks
        conf.setNumWorkers(3);
        // create the topology and submit with config
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run it in a simulated local cluster
        // set the number of threads to run - similar to setting number of workers in live cluster
        conf.setMaxTaskParallelism(3);
        // create the local cluster instance
        LocalCluster cluster = new LocalCluster();
        // submit the topology to the local cluster
        // name topology
        cluster.submitTopology("sentence-word-count", conf, builder.createTopology());
        //**********************************************************************
        // let the topology run for 30 seconds. note topologies never terminate!
        Thread.sleep(30000);
        //**********************************************************************
        // we are done, so shutdown the local cluster
        cluster.shutdown();
    }
}

Example 52 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class DeployTopology method realMain.

public void realMain(String[] args) throws Exception {
    String _name = "MetricTest";
    /*        if (args.length > 0){
            conf = Utils.loadConf(args[0]);
        }*/
    int _killTopologyTimeout = JStormUtils.parseInt(conf.get(ConfigExtension.TASK_CLEANUP_TIMEOUT_SEC), 180);
    conf.put(ConfigExtension.TASK_CLEANUP_TIMEOUT_SEC, _killTopologyTimeout);
    int _numWorkers = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS), 6);
    int _numTopologies = JStormUtils.parseInt(conf.get(TOPOLOGY_NUMS), 1);
    int _spoutParallel = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 2);
    int _boltParallel = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 4);
    int _messageSize = JStormUtils.parseInt(conf.get(TOPOLOGY_MESSAGE_SIZES), 10);
    int _numAcker = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 2);
    int _boltNum = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLTS_NUMS), 3);
    boolean _ackEnabled = false;
    if (_numAcker > 0)
        _ackEnabled = true;
    for (int topoNum = 0; topoNum < _numTopologies; topoNum++) {
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("messageSpout", new DeploySpoult(_messageSize, _ackEnabled), _spoutParallel);
        builder.setBolt("messageBolt1", new DeployBolt(), _boltParallel).shuffleGrouping("messageSpout");
        for (int levelNum = 2; levelNum <= _boltNum; levelNum++) {
            builder.setBolt("messageBolt" + levelNum, new DeployBolt(), _boltParallel).shuffleGrouping("messageBolt" + (levelNum - 1));
        }
        conf.put(Config.TOPOLOGY_WORKERS, _numWorkers);
        conf.put(Config.TOPOLOGY_ACKER_EXECUTORS, _numAcker);
        StormSubmitter.submitTopology(_name + "_" + topoNum, conf, builder.createTopology());
    }
}

Example 53 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class WordCountTopology method test.

public static void test() {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new RandomSentenceSpout(), 5);
    builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
    builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));
    Config conf = new Config();
    conf.setDebug(true);
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    try {
        JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        Assert.fail("Failed");
    }
}

Example 54 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class SequenceTopology method SetLocalTopology.

public static void SetLocalTopology() throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    conf.put(TOPOLOGY_BOLT_PARALLELISM_HINT, 1);
    SetBuilder(builder, conf);
    LOG.debug("test");
    LOG.info("Submit log");
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("SplitMerge", conf, builder.createTopology());
    Thread.sleep(60000);
    cluster.killTopology("SplitMerge");
    cluster.shutdown();
}

Example 55 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.

the class TaskInDifferentNodeTopology method test.

public static void test() throws Exception {
    JStormHelper.cleanCluster();
    hosts = JStormHelper.getSupervisorHosts();
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new TestWordSpout(), 1);
    /*********
         * 
         * This make sure the tasks will run on different nodes
         * 
         * 
         * 
         */
    Map<String, Object> componentMap = new HashMap<>();
    ConfigExtension.setTaskOnDifferentNode(componentMap, true);
    builder.setBolt(BOLT_NAME, new ExclamationLoggingBolt(), hosts.size()).localFirstGrouping("word").addConfigurations(componentMap);
    if (isLocal == false) {
        if (spoutSingle == true) {
            conf.setNumWorkers(hosts.size() + 3);
        } else {
            conf.setNumWorkers(hosts.size());
        }
    }
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    try {
        JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 180, new Validator(conf), isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        Assert.fail("Failed");
    }
}