use of org.apache.storm.sql.runtime.IOutputSerializer in project storm by apache.
the class KafkaDataSourcesProvider method constructTrident.
@Override
public ISqlTridentDataSource constructTrident(URI uri, String inputFormatClass, String outputFormatClass, Properties properties, List<FieldInfo> fields) {
int port = uri.getPort() != -1 ? uri.getPort() : DEFAULT_ZK_PORT;
ZkHosts zk = new ZkHosts(uri.getHost() + ":" + port, uri.getPath());
Map<String, String> values = parseURIParams(uri.getQuery());
String topic = values.get("topic");
Preconditions.checkNotNull(topic, "No topic of the spout is specified");
TridentKafkaConfig conf = new TridentKafkaConfig(zk, topic);
List<String> fieldNames = new ArrayList<>();
int primaryIndex = -1;
for (int i = 0; i < fields.size(); ++i) {
FieldInfo f = fields.get(i);
fieldNames.add(f.name());
if (f.isPrimary()) {
primaryIndex = i;
}
}
Preconditions.checkState(primaryIndex != -1, "Kafka stream table must have a primary key");
Scheme scheme = SerdeUtils.getScheme(inputFormatClass, properties, fieldNames);
conf.scheme = new SchemeAsMultiScheme(scheme);
IOutputSerializer serializer = SerdeUtils.getSerializer(outputFormatClass, properties, fieldNames);
return new KafkaTridentDataSource(conf, topic, primaryIndex, properties, serializer);
}
use of org.apache.storm.sql.runtime.IOutputSerializer in project storm by apache.
the class MongoDataSourcesProvider method constructStreams.
@Override
public ISqlStreamsDataSource constructStreams(URI uri, String inputFormatClass, String outputFormatClass, Properties properties, List<FieldInfo> fields) {
List<String> fieldNames = FieldInfoUtils.getFieldNames(fields);
IOutputSerializer serializer = SerdeUtils.getSerializer(outputFormatClass, properties, fieldNames);
return new MongoStreamsDataSource(uri.toString(), properties, serializer);
}
use of org.apache.storm.sql.runtime.IOutputSerializer in project storm by apache.
the class RedisDataSourcesProvider method constructTrident.
@Override
public ISqlTridentDataSource constructTrident(URI uri, String inputFormatClass, String outputFormatClass, Properties props, List<FieldInfo> fields) {
Preconditions.checkArgument(JedisURIHelper.isValid(uri), "URI is not valid for Redis: " + uri);
String host = uri.getHost();
int port = uri.getPort() != -1 ? uri.getPort() : DEFAULT_REDIS_PORT;
int dbIdx = JedisURIHelper.getDBIndex(uri);
String password = JedisURIHelper.getPassword(uri);
int timeout = Integer.parseInt(props.getProperty("redis.timeout", String.valueOf(DEFAULT_TIMEOUT)));
boolean clusterMode = Boolean.valueOf(props.getProperty("use.redis.cluster", "false"));
List<String> fieldNames = FieldInfoUtils.getFieldNames(fields);
IOutputSerializer serializer = SerdeUtils.getSerializer(outputFormatClass, props, fieldNames);
if (clusterMode) {
JedisClusterConfig config = new JedisClusterConfig.Builder().setNodes(Collections.singleton(new InetSocketAddress(host, port))).setTimeout(timeout).build();
return new RedisClusterTridentDataSource(config, props, fields, serializer);
} else {
JedisPoolConfig config = new JedisPoolConfig(host, port, timeout, password, dbIdx);
return new RedisTridentDataSource(config, props, fields, serializer);
}
}
use of org.apache.storm.sql.runtime.IOutputSerializer in project storm by apache.
the class SocketDataSourcesProvider method constructTrident.
@Override
public ISqlTridentDataSource constructTrident(URI uri, String inputFormatClass, String outputFormatClass, Properties properties, List<FieldInfo> fields) {
String host = uri.getHost();
int port = uri.getPort();
if (port == -1) {
throw new RuntimeException("Port information is not available. URI: " + uri);
}
List<String> fieldNames = FieldInfoUtils.getFieldNames(fields);
Scheme scheme = SerdeUtils.getScheme(inputFormatClass, properties, fieldNames);
IOutputSerializer serializer = SerdeUtils.getSerializer(outputFormatClass, properties, fieldNames);
return new SocketTridentDataSource(scheme, serializer, host, port);
}
use of org.apache.storm.sql.runtime.IOutputSerializer in project storm by apache.
the class HdfsDataSourcesProvider method constructTrident.
@Override
public ISqlTridentDataSource constructTrident(URI uri, String inputFormatClass, String outputFormatClass, Properties properties, List<FieldInfo> fields) {
List<String> fieldNames = FieldInfoUtils.getFieldNames(fields);
IOutputSerializer serializer = SerdeUtils.getSerializer(outputFormatClass, properties, fieldNames);
return new HdfsTridentDataSource(uri.toString(), properties, serializer);
}
Aggregations