Examples with TopologyBuilder backtype.storm.topology.TopologyBuilder used on opensource projects

Example 6 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project storm-lib by xumingming.

the class TestingApiDemo method testBasicTopology.

public void testBasicTopology() {
    MkClusterParam mkClusterParam = new MkClusterParam();
    mkClusterParam.setSupervisors(4);
    Config daemonConf = new Config();
    daemonConf.put(Config.STORM_LOCAL_MODE_ZMQ, false);
    mkClusterParam.setDaemonConf(daemonConf);
    /**
     * This is a combination of <code>Testing.withLocalCluster</code> and <code>Testing.withSimulatedTime</code>.
     */
    Testing.withSimulatedTimeLocalCluster(mkClusterParam, new TestJob() {

        @Override
        public void run(ILocalCluster cluster) {
            // build the test topology
            TopologyBuilder builder = new TopologyBuilder();
            builder.setSpout("1", new TestWordSpout(true), 3);
            builder.setBolt("2", new TestWordCounter(), 4).fieldsGrouping("1", new Fields("word"));
            builder.setBolt("3", new TestGlobalCount()).globalGrouping("1");
            builder.setBolt("4", new TestAggregatesCounter()).globalGrouping("2");
            StormTopology topology = builder.createTopology();
            // complete the topology
            // prepare the mock data
            MockedSources mockedSources = new MockedSources();
            mockedSources.addMockData("1", new Values("nathan"), new Values("bob"), new Values("joey"), new Values("nathan"));
            // prepare the config
            Config conf = new Config();
            conf.setNumWorkers(2);
            CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam();
            completeTopologyParam.setMockedSources(mockedSources);
            completeTopologyParam.setStormConf(conf);
            /**
             * TODO
             */
            Map result = Testing.completeTopology(cluster, topology, completeTopologyParam);
            // check whether the result is right
            assertTrue(Testing.multiseteq(new Values(new Values("nathan"), new Values("bob"), new Values("joey"), new Values("nathan")), Testing.readTuples(result, "1")));
            assertTrue(Testing.multiseteq(new Values(new Values("nathan", 1), new Values("nathan", 2), new Values("bob", 1), new Values("joey", 1)), Testing.readTuples(result, "2")));
            assertTrue(Testing.multiseteq(new Values(new Values(1), new Values(2), new Values(3), new Values(4)), Testing.readTuples(result, "3")));
            assertTrue(Testing.multiseteq(new Values(new Values(1), new Values(2), new Values(3), new Values(4)), Testing.readTuples(result, "4")));
        }
    });
}

Example 7 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project visitante by pranab.

the class VisitTopology method main.

/**
 * @param args
 * @throws Exception
 */
public static void main(String[] args) throws Exception {
    if (args.length != 2) {
        throw new IllegalArgumentException("Need two arguments: topology name and config file path");
    }
    String topologyName = args[0];
    String configFilePath = args[1];
    Config conf = RealtimeUtil.buildStormConfig(configFilePath);
    boolean debugOn = ConfigUtility.getBoolean(conf, "debug.on", false);
    System.out.println("config file:" + configFilePath + " debugOn:" + debugOn);
    conf.put(Config.TOPOLOGY_DEBUG, debugOn);
    // spout
    TopologyBuilder builder = new TopologyBuilder();
    int spoutThreads = ConfigUtility.getInt(conf, "spout.threads", 1);
    VisitDepthSpout spout = new VisitDepthSpout();
    spout.withTupleFields(VisitTopology.SESSION_ID, VisitTopology.VISIT_TIME, VisitTopology.VISIT_URL);
    builder.setSpout("visitDepthRedisSpout", spout, spoutThreads);
    // visit session bolt
    int visSessTickFreqInSec = ConfigUtility.getInt(conf, "visit.session.tick.freq.sec", 1);
    VisitSessionBolt viSessBolt = new VisitSessionBolt(visSessTickFreqInSec);
    viSessBolt.withTupleFields(VisitTopology.PAGE_ID, VisitTopology.PAGE_COUNT);
    int boltThreads = ConfigUtility.getInt(conf, "visit.session.bolt.threads", 1);
    builder.setBolt("visitSessionBolt", viSessBolt, boltThreads).fieldsGrouping("visitDepthRedisSpout", new Fields(VisitTopology.SESSION_ID));
    // visit depth bolt
    int visDepthTickFreqInSec = ConfigUtility.getInt(conf, "visit.depth.tick.freq.sec", 1);
    VisitDepthBolt viDepthBolt = new VisitDepthBolt(visDepthTickFreqInSec);
    boltThreads = ConfigUtility.getInt(conf, "visit.depth.bolt.threads", 1);
    builder.setBolt("visitDepthBolt", viDepthBolt, boltThreads).shuffleGrouping("visitSessionBolt");
    // submit
    RealtimeUtil.submitStormTopology(topologyName, conf, builder);
}

Example 8 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project incubator-heron by apache.

the class TaskHookTopology method main.

public static void main(String[] args) throws Exception {
    if (args.length != 1) {
        throw new RuntimeException("Specify topology name");
    }
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new AckingTestWordSpout(), 2);
    builder.setBolt("count", new CountBolt(), 2).shuffleGrouping("word");
    Config conf = new Config();
    conf.setDebug(true);
    // Put an arbitrary large number here if you don't want to slow the topology down
    conf.setMaxSpoutPending(1000 * 1000 * 1000);
    // To enable acking, we need to setEnableAcking true
    conf.setNumAckers(1);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    // Set the task hook
    List<String> taskHooks = new LinkedList<>();
    taskHooks.add("com.twitter.heron.examples.TaskHookTopology$TestTaskHook");
    com.twitter.heron.api.Config.setAutoTaskHooks(conf, taskHooks);
    // component resource configuration
    com.twitter.heron.api.Config.setComponentRam(conf, "word", ByteAmount.fromMegabytes(512));
    com.twitter.heron.api.Config.setComponentRam(conf, "count", ByteAmount.fromMegabytes(512));
    // container resource configuration
    com.twitter.heron.api.Config.setContainerDiskRequested(conf, ByteAmount.fromGigabytes(2));
    com.twitter.heron.api.Config.setContainerRamRequested(conf, ByteAmount.fromGigabytes(2));
    com.twitter.heron.api.Config.setContainerCpuRequested(conf, 2);
    conf.setNumWorkers(2);
    StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example 9 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project incubator-heron by apache.

the class WordCountTopology method main.

/**
 * Main method
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    if (args.length < 1) {
        throw new RuntimeException("Specify topology name");
    }
    int parallelism = 1;
    if (args.length > 1) {
        parallelism = Integer.parseInt(args[1]);
    }
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new WordSpout(), parallelism);
    builder.setBolt("consumer", new ConsumerBolt(), parallelism).fieldsGrouping("word", new Fields("word"));
    Config conf = new Config();
    conf.setNumWorkers(parallelism);
    StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example 10 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project ud381 by udacity.

the class TopNTweetTopology method main.

public static void main(String[] args) throws Exception {
    // Variable TOP_N number of words
    int TOP_N = 5;
    // create the topology
    TopologyBuilder builder = new TopologyBuilder();
    /*
     * In order to create the spout, you need to get twitter credentials
     * If you need to use Twitter firehose/Tweet stream for your idea,
     * create a set of credentials by following the instructions at
     *
     * https://dev.twitter.com/discussions/631
     *
     */
    // now create the tweet spout with the credentials
    // credential
    TweetSpout tweetSpout = new TweetSpout("", "", "", "");
    // attach the tweet spout to the topology - parallelism of 1
    builder.setSpout("tweet-spout", tweetSpout, 1);
    // attach the parse tweet bolt using shuffle grouping
    builder.setBolt("parse-tweet-bolt", new ParseTweetBolt(), 10).shuffleGrouping("tweet-spout");
    builder.setBolt("infoBolt", new InfoBolt(), 10).fieldsGrouping("parse-tweet-bolt", new Fields("county_id"));
    builder.setBolt("top-words", new TopWords(), 10).fieldsGrouping("infoBolt", new Fields("county_id"));
    builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("top-words");
    // attach rolling count bolt using fields grouping - parallelism of 5
    // builder.setBolt("rolling-count-bolt", new RollingCountBolt(1000, 10), 1).fieldsGrouping("parse-tweet-bolt", new Fields("tweet-word"));
    // from incubator-storm/.../storm/starter/RollingTopWords.java
    // builder.setBolt("intermediate-ranker", new IntermediateRankingsBolt(TOP_N, 10), 2).fieldsGrouping("rolling-count-bolt", new Fields("obj"));
    // builder.setBolt("total-ranker", new TotalRankingsBolt(TOP_N, 2)).globalGrouping("intermediate-ranker");
    /*
     * total-ranker bolt output is broadcast (allGrouping) to all the top-tweets bolt instances so
     * that every one of them have access to the top hashtags
     * tweet-spout tweet stream will be distributed randomly to the top-tweets bolt instances
     */
    // builder.setBolt("top-tweets", new TweetsWithTopHashtagsBolt(), 4)
    // .allGrouping("total-ranker")
    // .shuffleGrouping("tweet-spout");
    // attach the report bolt using global grouping - parallelism of 1
    // builder.setBolt("report-bolt", new ReportBolt(), 1).globalGrouping("top-tweets");
    // create the default config object
    Config conf = new Config();
    // set the config in debugging mode
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        // run it in a live cluster
        // set the number of workers for running all spout and bolt tasks
        conf.setNumWorkers(3);
        // create the topology and submit with config
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run it in a simulated local cluster
        // set the number of threads to run - similar to setting number of workers in live cluster
        conf.setMaxTaskParallelism(4);
        // create the local cluster instance
        LocalCluster cluster = new LocalCluster();
        // submit the topology to the local cluster
        cluster.submitTopology("tweet-word-count", conf, builder.createTopology());
        // let the topology run for 300 seconds. note topologies never terminate!
        Utils.sleep(300000000);
        // now kill the topology
        cluster.killTopology("tweet-word-count");
        // we are done, so shutdown the local cluster
        cluster.shutdown();
    }
}