Examples with TopologyBuilder backtype.storm.topology.TopologyBuilder used on opensource projects

Example 91 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project storm-hbase by ypf412.

the class DumpToHBaseTopology method main.

/**
 * HBase Data Dump to Another HBase Table Topology
 * @param args
 * @throws Exception
 */
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    PropConfig pc = new PropConfig("storm.properties");
    int topoWorkers = Integer.valueOf(pc.getProperty("storm.topolopy.workers"));
    int spoutTasks = Integer.valueOf(pc.getProperty("storm.spout.tasks"));
    builder.setSpout("hbaseSpout", new HBaseSpout(), spoutTasks);
    int boltTasks = spoutTasks;
    builder.setBolt("dumpBolt", new DumpToHBaseBolt(), boltTasks).fieldsGrouping("hbaseSpout", new Fields("sharding"));
    Config conf = new Config();
    conf.put(Constants.STORM_PROP_CONF_FILE, "storm.properties");
    conf.put(Constants.HBASE_PROP_CONF_FILE, "hbase.properties");
    if (args != null && args.length > 0) {
        // run on storm cluster
        conf.setNumAckers(1);
        conf.setNumWorkers(topoWorkers);
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        // run on local cluster
        conf.setMaxTaskParallelism(3);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("test", conf, builder.createTopology());
        Utils.sleep(10000);
        cluster.killTopology("test");
        cluster.shutdown();
    }
}

Example 92 with TopologyBuilder

use of backtype.storm.topology.TopologyBuilder in project visitante by pranab.

the class UniqueVisitorTopology method main.

/**
 * @param args
 * @throws Exception
 */
public static void main(String[] args) throws Exception {
    if (args.length != 2) {
        throw new IllegalArgumentException("Need two arguments: topology name and config file path");
    }
    String topologyName = args[0];
    String configFilePath = args[1];
    Config conf = RealtimeUtil.buildStormConfig(configFilePath);
    boolean debugOn = ConfigUtility.getBoolean(conf, "debug.on", false);
    System.out.println("config file:" + configFilePath + " debugOn:" + debugOn);
    conf.put(Config.TOPOLOGY_DEBUG, debugOn);
    // spout
    TopologyBuilder builder = new TopologyBuilder();
    int spoutThreads = ConfigUtility.getInt(conf, "spout.threads", 1);
    UniqueVisitorSpout spout = new UniqueVisitorSpout();
    spout.withTupleFields(PART_ID, USER_ID);
    builder.setSpout("unqueVisitorSpout", spout, spoutThreads);
    // unique count bolt
    int uniqueCountTickFreqInSec = ConfigUtility.getInt(conf, "unique.user.tick.freq.sec", 1);
    UniqueVisitorCounterBolt uniqueCountBolt = new UniqueVisitorCounterBolt(uniqueCountTickFreqInSec);
    uniqueCountBolt.withTupleFields(BOLT_ID, UNIQUE_COUNT);
    int boltThreads = ConfigUtility.getInt(conf, "unique.count.bolt.threads", 1);
    builder.setBolt("uniqueCountBolt", uniqueCountBolt, boltThreads).fieldsGrouping("unqueVisitorSpout", new Fields(PART_ID));
    // unique count aggregation bolt
    UniqueVisitorAggregatorBolt uniqueCountAggrBolt = new UniqueVisitorAggregatorBolt();
    boltThreads = ConfigUtility.getInt(conf, "unique.count.aggr.bolt.threads", 1);
    builder.setBolt("uniqueCountAggrBolt", uniqueCountAggrBolt, boltThreads).shuffleGrouping("uniqueCountBolt");
    // submit
    RealtimeUtil.submitStormTopology(topologyName, conf, builder);
}