Examples with KafkaSpout org.apache.storm.kafka.KafkaSpout used on opensource projects

Example 1 with KafkaSpout

use of org.apache.storm.kafka.KafkaSpout in project storm by apache.

the class KafkaHdfsTopo method getTopology.

public static StormTopology getTopology(Map config) {
    final int spoutNum = getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
    final int boltNum = getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM);
    final int hdfsBatch = getInt(config, HDFS_BATCH, DEFAULT_HDFS_BATCH);
    // 1 -  Setup Kafka Spout   --------
    String zkConnString = getStr(config, ZOOKEEPER_URI);
    String topicName = getStr(config, KAFKA_TOPIC);
    BrokerHosts brokerHosts = new ZkHosts(zkConnString);
    SpoutConfig spoutConfig = new SpoutConfig(brokerHosts, topicName, "/" + topicName, UUID.randomUUID().toString());
    spoutConfig.scheme = new StringMultiSchemeWithTopic();
    spoutConfig.ignoreZkOffsets = true;
    KafkaSpout spout = new KafkaSpout(spoutConfig);
    // 2 -  Setup HFS Bolt   --------
    String Hdfs_url = getStr(config, HDFS_URI);
    RecordFormat format = new LineWriter("str");
    SyncPolicy syncPolicy = new CountSyncPolicy(hdfsBatch);
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f, FileSizeRotationPolicy.Units.GB);
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(getStr(config, HDFS_PATH));
    // Instantiate the HdfsBolt
    HdfsBolt bolt = new HdfsBolt().withFsUrl(Hdfs_url).withFileNameFormat(fileNameFormat).withRecordFormat(format).withRotationPolicy(rotationPolicy).withSyncPolicy(syncPolicy);
    // 3 - Setup Topology  --------
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT_ID, spout, spoutNum);
    builder.setBolt(BOLT_ID, bolt, boltNum).localOrShuffleGrouping(SPOUT_ID);
    return builder.createTopology();
}

Example 2 with KafkaSpout

use of org.apache.storm.kafka.KafkaSpout in project storm by apache.

the class KafkaSpoutNullBoltTopo method getTopology.

public static StormTopology getTopology(Map config) {
    final int spoutNum = getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
    final int boltNum = getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM);
    // 1 -  Setup Kafka Spout   --------
    String zkConnString = getStr(config, ZOOKEEPER_URI);
    String topicName = getStr(config, KAFKA_TOPIC);
    BrokerHosts brokerHosts = new ZkHosts(zkConnString);
    SpoutConfig spoutConfig = new SpoutConfig(brokerHosts, topicName, "/" + topicName, UUID.randomUUID().toString());
    spoutConfig.scheme = new StringMultiSchemeWithTopic();
    spoutConfig.ignoreZkOffsets = true;
    KafkaSpout spout = new KafkaSpout(spoutConfig);
    // 2 -   DevNull Bolt   --------
    DevNullBolt bolt = new DevNullBolt();
    // 3 - Setup Topology  --------
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT_ID, spout, spoutNum);
    builder.setBolt(BOLT_ID, bolt, boltNum).localOrShuffleGrouping(SPOUT_ID);
    return builder.createTopology();
}

Example 3 with KafkaSpout

use of org.apache.storm.kafka.KafkaSpout in project pancm_project by xuwujing.

the class MykafkaSpout method main.

/**
 * The entry point of application.
 *
 * @param args the input arguments
 */
/*
	 * 通过zookeeper进行获取kafka的数据
	 */
public static void main(String[] args) {
    String topic = "pcm_test1";
    ZkHosts zkHosts = new ZkHosts("192.169.0.23:2181");
    SpoutConfig spoutConfig = new SpoutConfig(zkHosts, topic, "", "MyTrack");
    List<String> zkServers = new ArrayList<String>();
    zkServers.add("192.169.0.23");
    spoutConfig.zkServers = zkServers;
    spoutConfig.zkPort = 2181;
    spoutConfig.socketTimeoutMs = 60 * 1000;
    spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
    TopologyBuilder builder = new TopologyBuilder();
    // 设置1个Executeor(线程)，默认一个
    builder.setSpout("spout", new KafkaSpout(spoutConfig), 1);
    // 设置storm 设置1个Executeor(线程) 没有设置Task，默认一个
    builder.setBolt("bolt1", new MyKafkaBolt(), 1).shuffleGrouping("spout");
    Config conf = new Config();
    conf.setDebug(false);
    if (args.length > 0) {
        System.out.println("远程模式");
        try {
            StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
        } catch (AlreadyAliveException e) {
            e.printStackTrace();
        } catch (InvalidTopologyException e) {
            e.printStackTrace();
        } catch (org.apache.storm.generated.AuthorizationException e) {
            e.printStackTrace();
        }
    } else {
        System.out.println("本地模式");
        LocalCluster localCluster = new LocalCluster();
        localCluster.submitTopology("mytopology", conf, builder.createTopology());
    }
}