Examples with KafkaSpout storm.kafka.KafkaSpout used on opensource projects

Example 1 with KafkaSpout

use of storm.kafka.KafkaSpout in project avro-kafka-storm by ransilberman.

the class AvroTopology method main.

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    //Add those lines to prevent too much logging noise in the console
    Logger.getLogger("storm.kafka.PartitionManager").setLevel(Level.ERROR);
    Logger.getLogger("backtype.storm").setLevel(Level.ERROR);
    Logger.getLogger("storm.kafka").setLevel(Level.ERROR);
    TopologyBuilder builder = new TopologyBuilder();
    int partitions = 1;
    final String offsetPath = "/liveperson-avro-test";
    final String consumerId = "v1";
    final String topic = "avro-test";
    List<String> hosts = new ArrayList<String>();
    hosts.add("tlvwhale1");
    hosts.add("tlvwhale2");
    hosts.add("tlvwhale3");
    SpoutConfig kafkaConfig = new SpoutConfig(KafkaConfig.StaticHosts.fromHostString(hosts, partitions), topic, offsetPath, consumerId);
    KafkaSpout kafkaSpout = new KafkaSpout(kafkaConfig);
    builder.setSpout("spout", kafkaSpout);
    builder.setBolt("bolt", new AvroBolt()).shuffleGrouping("spout");
    Config conf = new Config();
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("avroTopology", conf, builder.createTopology());
        Utils.sleep(100000);
        cluster.killTopology("avroTopology");
        cluster.shutdown();
    }
}

Example 2 with KafkaSpout

use of storm.kafka.KafkaSpout in project avro-kafka-storm by ransilberman.

the class AvroTopology2 method main.

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    //Add those lines to prevent too much logging noise in the console
    Logger.getLogger("storm.kafka.PartitionManager").setLevel(Level.ERROR);
    Logger.getLogger("backtype.storm").setLevel(Level.ERROR);
    Logger.getLogger("storm.kafka").setLevel(Level.ERROR);
    TopologyBuilder builder = new TopologyBuilder();
    int partitions = 1;
    final String offsetPath = "/liveperson-avro-test";
    final String consumerId = "v1";
    final String topic = "avro-test";
    List<String> hosts = new ArrayList<String>();
    hosts.add("tlvwhale1");
    hosts.add("tlvwhale2");
    hosts.add("tlvwhale3");
    SpoutConfig kafkaConfig = new SpoutConfig(KafkaConfig.StaticHosts.fromHostString(hosts, partitions), topic, offsetPath, consumerId);
    KafkaSpout kafkaSpout = new KafkaSpout(kafkaConfig);
    builder.setSpout("spout", kafkaSpout);
    builder.setBolt("bolt", new AvroBolt()).shuffleGrouping("spout");
    Config conf = new Config();
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("avroTopology", conf, builder.createTopology());
        Utils.sleep(1000000);
        cluster.killTopology("avroTopology");
        cluster.shutdown();
    }
}