use of org.apache.storm.kafka.ZkHosts in project storm by apache.
the class TridentKafkaWordCount method newTridentKafkaConfig.
private static TridentKafkaConfig newTridentKafkaConfig(String zkUrl) {
ZkHosts hosts = new ZkHosts(zkUrl);
TridentKafkaConfig config = new TridentKafkaConfig(hosts, "test");
config.scheme = new SchemeAsMultiScheme(new StringScheme());
// Consume new data from the topic
config.startOffsetTime = kafka.api.OffsetRequest.LatestTime();
return config;
}
use of org.apache.storm.kafka.ZkHosts in project storm by apache.
the class KafkaDataSourcesProvider method constructTrident.
@Override
public ISqlTridentDataSource constructTrident(URI uri, String inputFormatClass, String outputFormatClass, Properties properties, List<FieldInfo> fields) {
int port = uri.getPort() != -1 ? uri.getPort() : DEFAULT_ZK_PORT;
ZkHosts zk = new ZkHosts(uri.getHost() + ":" + port, uri.getPath());
Map<String, String> values = parseURIParams(uri.getQuery());
String topic = values.get("topic");
Preconditions.checkNotNull(topic, "No topic of the spout is specified");
TridentKafkaConfig conf = new TridentKafkaConfig(zk, topic);
List<String> fieldNames = new ArrayList<>();
int primaryIndex = -1;
for (int i = 0; i < fields.size(); ++i) {
FieldInfo f = fields.get(i);
fieldNames.add(f.name());
if (f.isPrimary()) {
primaryIndex = i;
}
}
Preconditions.checkState(primaryIndex != -1, "Kafka stream table must have a primary key");
Scheme scheme = SerdeUtils.getScheme(inputFormatClass, properties, fieldNames);
conf.scheme = new SchemeAsMultiScheme(scheme);
IOutputSerializer serializer = SerdeUtils.getSerializer(outputFormatClass, properties, fieldNames);
return new KafkaTridentDataSource(conf, topic, primaryIndex, properties, serializer);
}
use of org.apache.storm.kafka.ZkHosts in project storm by apache.
the class KafkaHdfsTopo method getTopology.
public static StormTopology getTopology(Map config) {
final int spoutNum = getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
final int boltNum = getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM);
final int hdfsBatch = getInt(config, HDFS_BATCH, DEFAULT_HDFS_BATCH);
// 1 - Setup Kafka Spout --------
String zkConnString = getStr(config, ZOOKEEPER_URI);
String topicName = getStr(config, KAFKA_TOPIC);
BrokerHosts brokerHosts = new ZkHosts(zkConnString);
SpoutConfig spoutConfig = new SpoutConfig(brokerHosts, topicName, "/" + topicName, UUID.randomUUID().toString());
spoutConfig.scheme = new StringMultiSchemeWithTopic();
spoutConfig.ignoreZkOffsets = true;
KafkaSpout spout = new KafkaSpout(spoutConfig);
// 2 - Setup HFS Bolt --------
String Hdfs_url = getStr(config, HDFS_URI);
RecordFormat format = new LineWriter("str");
SyncPolicy syncPolicy = new CountSyncPolicy(hdfsBatch);
FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f, FileSizeRotationPolicy.Units.GB);
FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(getStr(config, HDFS_PATH));
// Instantiate the HdfsBolt
HdfsBolt bolt = new HdfsBolt().withFsUrl(Hdfs_url).withFileNameFormat(fileNameFormat).withRecordFormat(format).withRotationPolicy(rotationPolicy).withSyncPolicy(syncPolicy);
// 3 - Setup Topology --------
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(SPOUT_ID, spout, spoutNum);
builder.setBolt(BOLT_ID, bolt, boltNum).localOrShuffleGrouping(SPOUT_ID);
return builder.createTopology();
}
use of org.apache.storm.kafka.ZkHosts in project storm by apache.
the class KafkaSpoutNullBoltTopo method getTopology.
public static StormTopology getTopology(Map config) {
final int spoutNum = getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
final int boltNum = getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM);
// 1 - Setup Kafka Spout --------
String zkConnString = getStr(config, ZOOKEEPER_URI);
String topicName = getStr(config, KAFKA_TOPIC);
BrokerHosts brokerHosts = new ZkHosts(zkConnString);
SpoutConfig spoutConfig = new SpoutConfig(brokerHosts, topicName, "/" + topicName, UUID.randomUUID().toString());
spoutConfig.scheme = new StringMultiSchemeWithTopic();
spoutConfig.ignoreZkOffsets = true;
KafkaSpout spout = new KafkaSpout(spoutConfig);
// 2 - DevNull Bolt --------
DevNullBolt bolt = new DevNullBolt();
// 3 - Setup Topology --------
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(SPOUT_ID, spout, spoutNum);
builder.setBolt(BOLT_ID, bolt, boltNum).localOrShuffleGrouping(SPOUT_ID);
return builder.createTopology();
}
Aggregations