use of backtype.storm.topology.TopologyBuilder in project ud381 by udacity.
the class SentenceWordCountTopology method main.
public static void main(String[] args) throws Exception {
// create the topology
TopologyBuilder builder = new TopologyBuilder();
// attach the word spout to the topology - parallelism of 5
//builder.setSpout("word-spout", new WordSpout(), 5);
// attach sentence spout to the topology - parallelism of 1
builder.setSpout("sentence-spout", new RandomSentenceSpout(), 1);
// attach split sentence split bolt to topology - parallelism of 15
//builder.setBolt("split-sentence-bolt", new SplitSentenceBolt(), 15).fieldsGrouping("sentence-spout", new Fields("sentence"));
builder.setBolt("split-sentence-bolt", new SplitSentenceBolt(), 15).shuffleGrouping("sentence-spout");
// attach the count bolt using fields grouping - parallelism of 15
builder.setBolt("count-bolt", new CountBolt(), 15).fieldsGrouping("split-sentence-bolt", new Fields("sentence-word"));
// attach the report bolt using global grouping - parallelism of 1
//***************************************************
// BEGIN YOUR CODE - part 2
builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("count-bolt");
// END YOUR CODE
//***************************************************
// create the default config object
Config conf = new Config();
// set the config in debugging mode
conf.setDebug(true);
if (args != null && args.length > 0) {
// run it in a live cluster
// set the number of workers for running all spout and bolt tasks
conf.setNumWorkers(3);
// create the topology and submit with config
StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
} else {
// run it in a simulated local cluster
// set the number of threads to run - similar to setting number of workers in live cluster
conf.setMaxTaskParallelism(3);
// create the local cluster instance
LocalCluster cluster = new LocalCluster();
// submit the topology to the local cluster
// name topology
cluster.submitTopology("sentence-word-count", conf, builder.createTopology());
//**********************************************************************
// let the topology run for 30 seconds. note topologies never terminate!
Thread.sleep(30000);
//**********************************************************************
// we are done, so shutdown the local cluster
cluster.shutdown();
}
}
use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.
the class DeployTopology method realMain.
public void realMain(String[] args) throws Exception {
String _name = "MetricTest";
/* if (args.length > 0){
conf = Utils.loadConf(args[0]);
}*/
int _killTopologyTimeout = JStormUtils.parseInt(conf.get(ConfigExtension.TASK_CLEANUP_TIMEOUT_SEC), 180);
conf.put(ConfigExtension.TASK_CLEANUP_TIMEOUT_SEC, _killTopologyTimeout);
int _numWorkers = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS), 6);
int _numTopologies = JStormUtils.parseInt(conf.get(TOPOLOGY_NUMS), 1);
int _spoutParallel = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 2);
int _boltParallel = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 4);
int _messageSize = JStormUtils.parseInt(conf.get(TOPOLOGY_MESSAGE_SIZES), 10);
int _numAcker = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 2);
int _boltNum = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLTS_NUMS), 3);
boolean _ackEnabled = false;
if (_numAcker > 0)
_ackEnabled = true;
for (int topoNum = 0; topoNum < _numTopologies; topoNum++) {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("messageSpout", new DeploySpoult(_messageSize, _ackEnabled), _spoutParallel);
builder.setBolt("messageBolt1", new DeployBolt(), _boltParallel).shuffleGrouping("messageSpout");
for (int levelNum = 2; levelNum <= _boltNum; levelNum++) {
builder.setBolt("messageBolt" + levelNum, new DeployBolt(), _boltParallel).shuffleGrouping("messageBolt" + (levelNum - 1));
}
conf.put(Config.TOPOLOGY_WORKERS, _numWorkers);
conf.put(Config.TOPOLOGY_ACKER_EXECUTORS, _numAcker);
StormSubmitter.submitTopology(_name + "_" + topoNum, conf, builder.createTopology());
}
}
use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.
the class WordCountTopology method test.
public static void test() {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("spout", new RandomSentenceSpout(), 5);
builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));
Config conf = new Config();
conf.setDebug(true);
String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
String topologyName = className[className.length - 1];
try {
JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
Assert.fail("Failed");
}
}
use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.
the class SequenceTopology method SetLocalTopology.
public static void SetLocalTopology() throws Exception {
TopologyBuilder builder = new TopologyBuilder();
conf.put(TOPOLOGY_BOLT_PARALLELISM_HINT, 1);
SetBuilder(builder, conf);
LOG.debug("test");
LOG.info("Submit log");
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("SplitMerge", conf, builder.createTopology());
Thread.sleep(60000);
cluster.killTopology("SplitMerge");
cluster.shutdown();
}
use of backtype.storm.topology.TopologyBuilder in project jstorm by alibaba.
the class TaskInDifferentNodeTopology method test.
public static void test() throws Exception {
JStormHelper.cleanCluster();
hosts = JStormHelper.getSupervisorHosts();
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("word", new TestWordSpout(), 1);
/*********
*
* This make sure the tasks will run on different nodes
*
*
*
*/
Map<String, Object> componentMap = new HashMap<>();
ConfigExtension.setTaskOnDifferentNode(componentMap, true);
builder.setBolt(BOLT_NAME, new ExclamationLoggingBolt(), hosts.size()).localFirstGrouping("word").addConfigurations(componentMap);
if (isLocal == false) {
if (spoutSingle == true) {
conf.setNumWorkers(hosts.size() + 3);
} else {
conf.setNumWorkers(hosts.size());
}
}
String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
String topologyName = className[className.length - 1];
try {
JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 180, new Validator(conf), isLocal);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
Assert.fail("Failed");
}
}
Aggregations