Examples with Scheme org.apache.storm.spout.Scheme used on opensource projects

Example 1 with Scheme

use of org.apache.storm.spout.Scheme in project storm by apache.

the class KafkaDataSourcesProvider method constructTrident.

@Override
public ISqlTridentDataSource constructTrident(URI uri, String inputFormatClass, String outputFormatClass, Properties properties, List<FieldInfo> fields) {
    int port = uri.getPort() != -1 ? uri.getPort() : DEFAULT_ZK_PORT;
    ZkHosts zk = new ZkHosts(uri.getHost() + ":" + port, uri.getPath());
    Map<String, String> values = parseURIParams(uri.getQuery());
    String topic = values.get("topic");
    Preconditions.checkNotNull(topic, "No topic of the spout is specified");
    TridentKafkaConfig conf = new TridentKafkaConfig(zk, topic);
    List<String> fieldNames = new ArrayList<>();
    int primaryIndex = -1;
    for (int i = 0; i < fields.size(); ++i) {
        FieldInfo f = fields.get(i);
        fieldNames.add(f.name());
        if (f.isPrimary()) {
            primaryIndex = i;
        }
    }
    Preconditions.checkState(primaryIndex != -1, "Kafka stream table must have a primary key");
    Scheme scheme = SerdeUtils.getScheme(inputFormatClass, properties, fieldNames);
    conf.scheme = new SchemeAsMultiScheme(scheme);
    IOutputSerializer serializer = SerdeUtils.getSerializer(outputFormatClass, properties, fieldNames);
    return new KafkaTridentDataSource(conf, topic, primaryIndex, properties, serializer);
}

Example 2 with Scheme

use of org.apache.storm.spout.Scheme in project kafka-spout by HolmesNL.

the class KafkaSpoutBufferBehaviourTest method setup.

@Before
public void setup() {
    final Scheme scheme = spy(new RawScheme());
    // main test subject
    _subject = new KafkaSpout(scheme);
    // assign the topic to be used for stream retrieval
    _subject._topic = "test-topic";
    // use a buffer size higher than the expected amount of messages available
    _subject._bufSize = 4;
    // assign the default FailHandler
    _subject._failHandler = ConfigUtils.DEFAULT_FAIL_HANDLER;
    // mocked consumer to avoid actually contacting zookeeper
    _consumer = mock(ConsumerConnector.class);
    // ... but make sure it will return a valid (empty) stream
    when(_consumer.createMessageStreams(any(Map.class))).thenReturn(EMPTY_STREAM);
    // assign the consumer to the test subject
    _subject._consumer = _consumer;
    // provide a mocked collector to be able to check for emitted values
    _subject._collector = mock(SpoutOutputCollector.class);
}

Example 3 with Scheme

use of org.apache.storm.spout.Scheme in project storm by apache.

the class SocketDataSourcesProvider method constructTrident.

@Override
public ISqlTridentDataSource constructTrident(URI uri, String inputFormatClass, String outputFormatClass, Properties properties, List<FieldInfo> fields) {
    String host = uri.getHost();
    int port = uri.getPort();
    if (port == -1) {
        throw new RuntimeException("Port information is not available. URI: " + uri);
    }
    List<String> fieldNames = FieldInfoUtils.getFieldNames(fields);
    Scheme scheme = SerdeUtils.getScheme(inputFormatClass, properties, fieldNames);
    IOutputSerializer serializer = SerdeUtils.getSerializer(outputFormatClass, properties, fieldNames);
    return new SocketTridentDataSource(scheme, serializer, host, port);
}

Example 4 with Scheme

use of org.apache.storm.spout.Scheme in project storm by apache.

the class KafkaDataSourcesProvider method constructStreams.

@Override
public ISqlStreamsDataSource constructStreams(URI uri, String inputFormatClass, String outputFormatClass, Properties properties, List<FieldInfo> fields) {
    List<String> fieldNames = new ArrayList<>();
    int primaryIndex = -1;
    for (int i = 0; i < fields.size(); ++i) {
        FieldInfo f = fields.get(i);
        fieldNames.add(f.name());
        if (f.isPrimary()) {
            primaryIndex = i;
        }
    }
    Preconditions.checkState(primaryIndex != -1, "Kafka stream table must have a primary key");
    Scheme scheme = SerdeUtils.getScheme(inputFormatClass, properties, fieldNames);
    Map<String, String> values = parseUriParams(uri.getQuery());
    String bootstrapServers = values.get(URI_PARAMS_BOOTSTRAP_SERVERS);
    Preconditions.checkNotNull(bootstrapServers, "bootstrap-servers must be specified");
    String topic = uri.getHost();
    KafkaSpoutConfig<ByteBuffer, ByteBuffer> kafkaSpoutConfig = new KafkaSpoutConfig.Builder<ByteBuffer, ByteBuffer>(bootstrapServers, topic).setProp(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteBufferDeserializer.class).setProp(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteBufferDeserializer.class).setProp(ConsumerConfig.GROUP_ID_CONFIG, "storm-sql-kafka-" + UUID.randomUUID().toString()).setRecordTranslator(new RecordTranslatorSchemeAdapter(scheme)).build();
    IOutputSerializer serializer = SerdeUtils.getSerializer(outputFormatClass, properties, fieldNames);
    return new KafkaStreamsDataSource(kafkaSpoutConfig, bootstrapServers, topic, properties, serializer);
}

Example 5 with Scheme

use of org.apache.storm.spout.Scheme in project storm by apache.

the class SocketDataSourcesProvider method constructStreams.

@Override
public ISqlStreamsDataSource constructStreams(URI uri, String inputFormatClass, String outputFormatClass, Properties properties, List<FieldInfo> fields) {
    String host = uri.getHost();
    int port = uri.getPort();
    if (port == -1) {
        throw new RuntimeException("Port information is not available. URI: " + uri);
    }
    List<String> fieldNames = FieldInfoUtils.getFieldNames(fields);
    Scheme scheme = SerdeUtils.getScheme(inputFormatClass, properties, fieldNames);
    IOutputSerializer serializer = SerdeUtils.getSerializer(outputFormatClass, properties, fieldNames);
    return new SocketDataSourcesProvider.SocketStreamsDataSource(host, port, scheme, serializer);
}