Examples with LocalCluster backtype.storm.LocalCluster used on opensource projects

Example 6 with LocalCluster

use of backtype.storm.LocalCluster in project storm-hbase by ypf412.

the class StormTestUtil method createLocalStormCluster.

/**
 * Create Storm local cluster
 *
 * @param conf
 * @return
 * @throws Exception
 */
public static StormTestUtil createLocalStormCluster(Config conf) throws Exception {
    conf.setDebug(true);
    conf.setMaxTaskParallelism(4);
    LocalCluster cluster = new LocalCluster();
    StormTestUtil info = new StormTestUtil(conf, cluster);
    return info;
}

Example 7 with LocalCluster

use of backtype.storm.LocalCluster in project avro-kafka-storm by ransilberman.

the class AvroTopology method main.

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    // Add those lines to prevent too much logging noise in the console
    Logger.getLogger("storm.kafka.PartitionManager").setLevel(Level.ERROR);
    Logger.getLogger("backtype.storm").setLevel(Level.ERROR);
    Logger.getLogger("storm.kafka").setLevel(Level.ERROR);
    TopologyBuilder builder = new TopologyBuilder();
    int partitions = 1;
    final String offsetPath = "/liveperson-avro-test";
    final String consumerId = "v1";
    final String topic = "avro-test";
    List<String> hosts = new ArrayList<String>();
    hosts.add("tlvwhale1");
    hosts.add("tlvwhale2");
    hosts.add("tlvwhale3");
    SpoutConfig kafkaConfig = new SpoutConfig(KafkaConfig.StaticHosts.fromHostString(hosts, partitions), topic, offsetPath, consumerId);
    KafkaSpout kafkaSpout = new KafkaSpout(kafkaConfig);
    builder.setSpout("spout", kafkaSpout);
    builder.setBolt("bolt", new AvroBolt()).shuffleGrouping("spout");
    Config conf = new Config();
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("avroTopology", conf, builder.createTopology());
        Utils.sleep(100000);
        cluster.killTopology("avroTopology");
        cluster.shutdown();
    }
}

Example 8 with LocalCluster

use of backtype.storm.LocalCluster in project avro-kafka-storm by ransilberman.

the class AvroTopology2 method main.

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    // Add those lines to prevent too much logging noise in the console
    Logger.getLogger("storm.kafka.PartitionManager").setLevel(Level.ERROR);
    Logger.getLogger("backtype.storm").setLevel(Level.ERROR);
    Logger.getLogger("storm.kafka").setLevel(Level.ERROR);
    TopologyBuilder builder = new TopologyBuilder();
    int partitions = 1;
    final String offsetPath = "/liveperson-avro-test";
    final String consumerId = "v1";
    final String topic = "avro-test";
    List<String> hosts = new ArrayList<String>();
    hosts.add("tlvwhale1");
    hosts.add("tlvwhale2");
    hosts.add("tlvwhale3");
    SpoutConfig kafkaConfig = new SpoutConfig(KafkaConfig.StaticHosts.fromHostString(hosts, partitions), topic, offsetPath, consumerId);
    KafkaSpout kafkaSpout = new KafkaSpout(kafkaConfig);
    builder.setSpout("spout", kafkaSpout);
    builder.setBolt("bolt", new AvroBolt()).shuffleGrouping("spout");
    Config conf = new Config();
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("avroTopology", conf, builder.createTopology());
        Utils.sleep(1000000);
        cluster.killTopology("avroTopology");
        cluster.shutdown();
    }
}

Example 9 with LocalCluster

use of backtype.storm.LocalCluster in project storm-hbase by jrkinley.

the class HBaseExampleTopology method main.

/**
 * @param args
 */
public static void main(String[] args) {
    TopologyBuilder builder = new TopologyBuilder();
    // Add test spout
    builder.setSpout("spout", new TestSpout(), 1);
    // Build TupleTableConifg
    TupleTableConfig config = new TupleTableConfig("shorturl", "shortid");
    config.setBatch(false);
    config.addColumn("data", "url");
    config.addColumn("data", "user");
    config.addColumn("data", "date");
    // Add HBaseBolt
    builder.setBolt("hbase", new HBaseBolt(config), 1).shuffleGrouping("spout");
    Config stormConf = new Config();
    stormConf.setDebug(true);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("hbase-example", stormConf, builder.createTopology());
    Utils.sleep(10000);
    cluster.shutdown();
}

Example 10 with LocalCluster

use of backtype.storm.LocalCluster in project storm-hbase by jrkinley.

the class HBaseCountersTopology method main.

/**
 * @param args
 */
public static void main(String[] args) {
    TopologyBuilder builder = new TopologyBuilder();
    // Add test spout
    builder.setSpout("spout", new TestSpout(), 1);
    // Build TupleTableConifg
    TupleTableConfig config = new TupleTableConfig("shorturl", "shortid");
    config.setBatch(false);
    /*
     * By default the HBaseCountersBolt will use the tuple output fields value
     * to set the CQ name. For example if the 'date' output field exists in the
     * tuple then its value (e.g. "YYYYMMDD") will be used to set the counters
     * CQ. However, the 'clicks' output field does not exist in the tuple so in
     * this case the counters CQ will be set to the given field name 'clicks'.
     */
    config.addColumn("data", "clicks");
    config.addColumn("daily", "date");
    // Add HBaseBolt
    builder.setBolt("hbase-counters", new HBaseCountersBolt(config), 1).shuffleGrouping("spout");
    Config stormConf = new Config();
    stormConf.setDebug(true);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("hbase-example", stormConf, builder.createTopology());
    Utils.sleep(10000);
    cluster.shutdown();
}