Search in sources :

Example 1 with ZkHosts

use of org.apache.storm.kafka.ZkHosts in project storm by apache.

the class TridentKafkaWordCount method newTridentKafkaConfig.

private static TridentKafkaConfig newTridentKafkaConfig(String zkUrl) {
    ZkHosts hosts = new ZkHosts(zkUrl);
    TridentKafkaConfig config = new TridentKafkaConfig(hosts, "test");
    config.scheme = new SchemeAsMultiScheme(new StringScheme());
    // Consume new data from the topic
    config.startOffsetTime = kafka.api.OffsetRequest.LatestTime();
    return config;
}
Also used : ZkHosts(org.apache.storm.kafka.ZkHosts) SchemeAsMultiScheme(org.apache.storm.spout.SchemeAsMultiScheme) StringScheme(org.apache.storm.kafka.StringScheme)

Example 2 with ZkHosts

use of org.apache.storm.kafka.ZkHosts in project storm by apache.

the class KafkaDataSourcesProvider method constructTrident.

@Override
public ISqlTridentDataSource constructTrident(URI uri, String inputFormatClass, String outputFormatClass, Properties properties, List<FieldInfo> fields) {
    int port = uri.getPort() != -1 ? uri.getPort() : DEFAULT_ZK_PORT;
    ZkHosts zk = new ZkHosts(uri.getHost() + ":" + port, uri.getPath());
    Map<String, String> values = parseURIParams(uri.getQuery());
    String topic = values.get("topic");
    Preconditions.checkNotNull(topic, "No topic of the spout is specified");
    TridentKafkaConfig conf = new TridentKafkaConfig(zk, topic);
    List<String> fieldNames = new ArrayList<>();
    int primaryIndex = -1;
    for (int i = 0; i < fields.size(); ++i) {
        FieldInfo f = fields.get(i);
        fieldNames.add(f.name());
        if (f.isPrimary()) {
            primaryIndex = i;
        }
    }
    Preconditions.checkState(primaryIndex != -1, "Kafka stream table must have a primary key");
    Scheme scheme = SerdeUtils.getScheme(inputFormatClass, properties, fieldNames);
    conf.scheme = new SchemeAsMultiScheme(scheme);
    IOutputSerializer serializer = SerdeUtils.getSerializer(outputFormatClass, properties, fieldNames);
    return new KafkaTridentDataSource(conf, topic, primaryIndex, properties, serializer);
}
Also used : IOutputSerializer(org.apache.storm.sql.runtime.IOutputSerializer) SchemeAsMultiScheme(org.apache.storm.spout.SchemeAsMultiScheme) Scheme(org.apache.storm.spout.Scheme) ZkHosts(org.apache.storm.kafka.ZkHosts) ArrayList(java.util.ArrayList) TridentKafkaConfig(org.apache.storm.kafka.trident.TridentKafkaConfig) SchemeAsMultiScheme(org.apache.storm.spout.SchemeAsMultiScheme) FieldInfo(org.apache.storm.sql.runtime.FieldInfo)

Example 3 with ZkHosts

use of org.apache.storm.kafka.ZkHosts in project storm by apache.

the class KafkaHdfsTopo method getTopology.

public static StormTopology getTopology(Map config) {
    final int spoutNum = getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
    final int boltNum = getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM);
    final int hdfsBatch = getInt(config, HDFS_BATCH, DEFAULT_HDFS_BATCH);
    // 1 -  Setup Kafka Spout   --------
    String zkConnString = getStr(config, ZOOKEEPER_URI);
    String topicName = getStr(config, KAFKA_TOPIC);
    BrokerHosts brokerHosts = new ZkHosts(zkConnString);
    SpoutConfig spoutConfig = new SpoutConfig(brokerHosts, topicName, "/" + topicName, UUID.randomUUID().toString());
    spoutConfig.scheme = new StringMultiSchemeWithTopic();
    spoutConfig.ignoreZkOffsets = true;
    KafkaSpout spout = new KafkaSpout(spoutConfig);
    // 2 -  Setup HFS Bolt   --------
    String Hdfs_url = getStr(config, HDFS_URI);
    RecordFormat format = new LineWriter("str");
    SyncPolicy syncPolicy = new CountSyncPolicy(hdfsBatch);
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f, FileSizeRotationPolicy.Units.GB);
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(getStr(config, HDFS_PATH));
    // Instantiate the HdfsBolt
    HdfsBolt bolt = new HdfsBolt().withFsUrl(Hdfs_url).withFileNameFormat(fileNameFormat).withRecordFormat(format).withRotationPolicy(rotationPolicy).withSyncPolicy(syncPolicy);
    // 3 - Setup Topology  --------
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT_ID, spout, spoutNum);
    builder.setBolt(BOLT_ID, bolt, boltNum).localOrShuffleGrouping(SPOUT_ID);
    return builder.createTopology();
}
Also used : TopologyBuilder(org.apache.storm.topology.TopologyBuilder) RecordFormat(org.apache.storm.hdfs.bolt.format.RecordFormat) SpoutConfig(org.apache.storm.kafka.SpoutConfig) ZkHosts(org.apache.storm.kafka.ZkHosts) CountSyncPolicy(org.apache.storm.hdfs.bolt.sync.CountSyncPolicy) CountSyncPolicy(org.apache.storm.hdfs.bolt.sync.CountSyncPolicy) SyncPolicy(org.apache.storm.hdfs.bolt.sync.SyncPolicy) DefaultFileNameFormat(org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat) FileNameFormat(org.apache.storm.hdfs.bolt.format.FileNameFormat) StringMultiSchemeWithTopic(org.apache.storm.kafka.StringMultiSchemeWithTopic) FileRotationPolicy(org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy) DefaultFileNameFormat(org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat) BrokerHosts(org.apache.storm.kafka.BrokerHosts) HdfsBolt(org.apache.storm.hdfs.bolt.HdfsBolt) KafkaSpout(org.apache.storm.kafka.KafkaSpout) FileSizeRotationPolicy(org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy)

Example 4 with ZkHosts

use of org.apache.storm.kafka.ZkHosts in project storm by apache.

the class KafkaSpoutNullBoltTopo method getTopology.

public static StormTopology getTopology(Map config) {
    final int spoutNum = getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
    final int boltNum = getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM);
    // 1 -  Setup Kafka Spout   --------
    String zkConnString = getStr(config, ZOOKEEPER_URI);
    String topicName = getStr(config, KAFKA_TOPIC);
    BrokerHosts brokerHosts = new ZkHosts(zkConnString);
    SpoutConfig spoutConfig = new SpoutConfig(brokerHosts, topicName, "/" + topicName, UUID.randomUUID().toString());
    spoutConfig.scheme = new StringMultiSchemeWithTopic();
    spoutConfig.ignoreZkOffsets = true;
    KafkaSpout spout = new KafkaSpout(spoutConfig);
    // 2 -   DevNull Bolt   --------
    DevNullBolt bolt = new DevNullBolt();
    // 3 - Setup Topology  --------
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT_ID, spout, spoutNum);
    builder.setBolt(BOLT_ID, bolt, boltNum).localOrShuffleGrouping(SPOUT_ID);
    return builder.createTopology();
}
Also used : TopologyBuilder(org.apache.storm.topology.TopologyBuilder) SpoutConfig(org.apache.storm.kafka.SpoutConfig) DevNullBolt(org.apache.storm.perf.bolt.DevNullBolt) ZkHosts(org.apache.storm.kafka.ZkHosts) StringMultiSchemeWithTopic(org.apache.storm.kafka.StringMultiSchemeWithTopic) KafkaSpout(org.apache.storm.kafka.KafkaSpout) BrokerHosts(org.apache.storm.kafka.BrokerHosts)

Aggregations

ZkHosts (org.apache.storm.kafka.ZkHosts)4 BrokerHosts (org.apache.storm.kafka.BrokerHosts)2 KafkaSpout (org.apache.storm.kafka.KafkaSpout)2 SpoutConfig (org.apache.storm.kafka.SpoutConfig)2 StringMultiSchemeWithTopic (org.apache.storm.kafka.StringMultiSchemeWithTopic)2 SchemeAsMultiScheme (org.apache.storm.spout.SchemeAsMultiScheme)2 TopologyBuilder (org.apache.storm.topology.TopologyBuilder)2 ArrayList (java.util.ArrayList)1 HdfsBolt (org.apache.storm.hdfs.bolt.HdfsBolt)1 DefaultFileNameFormat (org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat)1 FileNameFormat (org.apache.storm.hdfs.bolt.format.FileNameFormat)1 RecordFormat (org.apache.storm.hdfs.bolt.format.RecordFormat)1 FileRotationPolicy (org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy)1 FileSizeRotationPolicy (org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy)1 CountSyncPolicy (org.apache.storm.hdfs.bolt.sync.CountSyncPolicy)1 SyncPolicy (org.apache.storm.hdfs.bolt.sync.SyncPolicy)1 StringScheme (org.apache.storm.kafka.StringScheme)1 TridentKafkaConfig (org.apache.storm.kafka.trident.TridentKafkaConfig)1 DevNullBolt (org.apache.storm.perf.bolt.DevNullBolt)1 Scheme (org.apache.storm.spout.Scheme)1