Examples with Config backtype.storm.Config used on opensource projects

Example 51 with Config

use of backtype.storm.Config in project heron by twitter.

the class MultiSpoutExclamationTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word0", new TestWordSpout(), 2);
    builder.setSpout("word1", new TestWordSpout(), 2);
    builder.setSpout("word2", new TestWordSpout(), 2);
    builder.setBolt("exclaim1", new ExclamationBolt(), 2).shuffleGrouping("word0").shuffleGrouping("word1").shuffleGrouping("word2");
    //builder.setBolt("exclaim2", new ExclamationBolt(), 2)
    //        .shuffleGrouping("exclaim1");
    Config conf = new Config();
    conf.setDebug(true);
    conf.setMaxSpoutPending(10);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    conf.setComponentRam("word0", ByteAmount.fromMegabytes(500));
    conf.setComponentRam("word1", ByteAmount.fromMegabytes(500));
    conf.setComponentRam("word2", ByteAmount.fromMegabytes(500));
    conf.setComponentRam("exclaim1", ByteAmount.fromGigabytes(1));
    if (args != null && args.length > 0) {
        conf.setNumStmgrs(1);
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("test", conf, builder.createTopology());
        Utils.sleep(10000);
        cluster.killTopology("test");
        cluster.shutdown();
    }
}

Example 52 with Config

use of backtype.storm.Config in project ud381 by udacity.

the class SentenceWordCountTopology method main.

public static void main(String[] args) throws Exception {
    // create the topology
    TopologyBuilder builder = new TopologyBuilder();
    // attach the word spout to the topology - parallelism of 5
    //builder.setSpout("word-spout", new WordSpout(), 5);
    // attach sentence spout to the topology - parallelism of 1
    builder.setSpout("sentence-spout", new RandomSentenceSpout(), 1);
    // attach split sentence split bolt to topology - parallelism of 15
    //builder.setBolt("split-sentence-bolt", new SplitSentenceBolt(), 15).fieldsGrouping("sentence-spout", new Fields("sentence"));
    builder.setBolt("split-sentence-bolt", new SplitSentenceBolt(), 15).shuffleGrouping("sentence-spout");
    // attach the count bolt using fields grouping - parallelism of 15
    builder.setBolt("count-bolt", new CountBolt(), 15).fieldsGrouping("split-sentence-bolt", new Fields("sentence-word"));
    // attach the report bolt using global grouping - parallelism of 1
    //***************************************************
    // BEGIN YOUR CODE - part 2
    builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("count-bolt");
    // END YOUR CODE
    //***************************************************
    // create the default config object
    Config conf = new Config();
    // set the config in debugging mode
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        // run it in a live cluster
        // set the number of workers for running all spout and bolt tasks
        conf.setNumWorkers(3);
        // create the topology and submit with config
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run it in a simulated local cluster
        // set the number of threads to run - similar to setting number of workers in live cluster
        conf.setMaxTaskParallelism(3);
        // create the local cluster instance
        LocalCluster cluster = new LocalCluster();
        // submit the topology to the local cluster
        // name topology
        cluster.submitTopology("sentence-word-count", conf, builder.createTopology());
        //**********************************************************************
        // let the topology run for 30 seconds. note topologies never terminate!
        Thread.sleep(30000);
        //**********************************************************************
        // we are done, so shutdown the local cluster
        cluster.shutdown();
    }
}

Example 53 with Config

use of backtype.storm.Config in project jstorm by alibaba.

the class ReachTopology method main.

public static void main(String[] args) throws Exception {
    LinearDRPCTopologyBuilder builder = construct();
    Config conf = new Config();
    conf.setNumWorkers(6);
    if (args.length != 0) {
        try {
            Map yamlConf = LoadConf.LoadYaml(args[0]);
            if (yamlConf != null) {
                conf.putAll(yamlConf);
            }
        } catch (Exception e) {
            System.out.println("Input " + args[0] + " isn't one yaml ");
        }
        StormSubmitter.submitTopology(TOPOLOGY_NAME, conf, builder.createRemoteTopology());
    } else {
        conf.setMaxTaskParallelism(3);
        LocalDRPC drpc = new LocalDRPC();
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(TOPOLOGY_NAME, conf, builder.createLocalTopology(drpc));
        JStormUtils.sleepMs(10000);
        String[] urlsToTry = new String[] { "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" };
        for (String url : urlsToTry) {
            System.out.println("Reach of " + url + ": " + drpc.execute(TOPOLOGY_NAME, url));
        }
        cluster.shutdown();
        drpc.shutdown();
    }
}

Example 54 with Config

use of backtype.storm.Config in project jstorm by alibaba.

the class WordCountTopology method test.

public static void test() {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new RandomSentenceSpout(), 5);
    builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
    builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));
    Config conf = new Config();
    conf.setDebug(true);
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    try {
        JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        Assert.fail("Failed");
    }
}

Example 55 with Config

use of backtype.storm.Config in project jstorm by alibaba.

the class SequenceTopologyTool method SetRemoteTopology.

public void SetRemoteTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    Config conf = getConf();
    StormTopology topology = buildTopology();
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        streamName = "SequenceTest";
    }
    if (streamName.contains("zeromq")) {
        conf.put(Config.STORM_MESSAGING_TRANSPORT, "com.alibaba.jstorm.message.zeroMq.MQContext");
    } else {
        conf.put(Config.STORM_MESSAGING_TRANSPORT, "com.alibaba.jstorm.message.netty.NettyContext");
    }
    StormSubmitter.submitTopology(streamName, conf, topology);
}